gammasimtools 0.21.0__py3-none-any.whl → 0.22.0__py3-none-any.whl

simtools/reporting/docs_read_parameters.py

@@ -16,9 +16,11 @@ from simtools.db import db_handler
 from simtools.io import ascii_handler, io_handler
 from simtools.model.telescope_model import TelescopeModel
 from simtools.utils import names
+from simtools.version import sort_versions
 from simtools.visualization import plot_pixels, plot_tables
 
 logger = logging.getLogger()
+MARKDOWN_LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)]+)\)")
 
 
 class ReadParameters:
@@ -64,7 +66,7 @@ class ReadParameters:
             )
         self._model_version = model_version
 
-    def _generate_plots(self, parameter, parameter_version, input_file, outpath, design_type):
+    def _generate_plots(self, parameter, parameter_version, input_file, outpath):
         """Generate plots based on the parameter type."""
         plot_names = []
 
@@ -72,7 +74,9 @@ class ReadParameters:
             plot_names = self._plot_camera_config(parameter, parameter_version, input_file, outpath)
         elif parameter_version:
             plot_names = self._plot_parameter_tables(
-                parameter, parameter_version, outpath, design_type
+                parameter,
+                parameter_version,
+                outpath,
             )
 
         return plot_names
@@ -111,18 +115,9 @@ class ReadParameters:
 
         return plot_names
 
-    def _plot_parameter_tables(self, parameter, parameter_version, outpath, design_type):
+    def _plot_parameter_tables(self, parameter, parameter_version, outpath):
         """Generate plots for parameter tables."""
-        telescope_design = self.db.get_design_model(
-            self.model_version, self.array_element, collection="telescopes"
-        )
-
-        if not self.array_element:
-            tel = None
-        elif not design_type:
-            tel = telescope_design
-        else:
-            tel = self.array_element
+        tel = self._get_telescope_identifier()
 
         config_data = plot_tables.generate_plot_configurations(
             parameter=parameter,
@@ -152,7 +147,20 @@ class ReadParameters:
 
         return plot_names
 
-    def _convert_to_md(self, parameter, parameter_version, input_file, design_type=False):
+    def _get_telescope_identifier(self, model_version=None):
+        """Get the appropriate telescope design type for file naming (e.g., LSTN-design)."""
+        model_version = model_version or self.model_version
+        telescope_design = self.db.get_design_model(
+            model_version, self.array_element, collection="telescopes"
+        )
+
+        if not self.array_element:
+            return None
+        if not names.is_design_type(self.array_element):
+            return telescope_design
+        return self.array_element
+
+    def _convert_to_md(self, parameter, parameter_version, input_file):
         """Convert a file to a Markdown file, preserving formatting."""
         input_file = Path(input_file)
 
@@ -170,9 +178,7 @@ class ReadParameters:
             outpath = Path(io_handler.IOHandler().get_output_directory().parent / "_images")
             outpath.mkdir(parents=True, exist_ok=True)
 
-            plot_names = self._generate_plots(
-                parameter, parameter_version, input_file, outpath, design_type
-            )
+            plot_names = self._generate_plots(parameter, parameter_version, input_file, outpath)
             # Write markdown file using the stored path
             file_contents = ascii_handler.read_file_encoded_in_utf_or_latin(input_file)
 
@@ -198,7 +204,7 @@ class ReadParameters:
         return relative_path
 
     def _format_parameter_value(
-        self, parameter, value_data, unit, file_flag, parameter_version=None, design_type=False
+        self, parameter, value_data, unit, file_flag, parameter_version=None
     ):
         """Format parameter value based on type."""
         if file_flag:
@@ -209,9 +215,7 @@ class ReadParameters:
                     "cta-science/simulations/simulation-model/simulation-models/-/blob/main/"
                     f"simulation-models/model_parameters/Files/{value_data})"
                 ).strip()
-            output_file_name = self._convert_to_md(
-                parameter, parameter_version, input_file_name, design_type
-            )
+            output_file_name = self._convert_to_md(parameter, parameter_version, input_file_name)
             return f"[{Path(value_data).name}]({output_file_name})".strip()
         if isinstance(value_data, (str | int | float)):
             return f"{value_data} {unit}".strip()
@@ -223,26 +227,6 @@ class ReadParameters:
             else ", ".join(f"{v} {unit}" for v in value_data)
         ).strip()
 
-    def _wrap_at_underscores(self, text, max_width):
-        """Wrap text at underscores to fit within a specified width."""
-        parts = text.split("_")
-        lines = []
-        current = []
-
-        for part in parts:
-            # Predict the new length if we add this part
-            next_line = "_".join([*current, part])
-            if len(next_line) <= max_width:
-                current.append(part)
-            else:
-                lines.append("_".join(current))
-                current = [part]
-
-        if current:
-            lines.append("_".join(current))
-
-        return " ".join(lines)
-
     def _group_model_versions_by_parameter_version(self, grouped_data):
         """Group model versions by parameter version and track the parameter values."""
         result = {}
@@ -265,7 +249,7 @@ class ReadParameters:
                     "value": data["value"],
                     "parameter_version": param_version,
                     "file_flag": data["file_flag"],
-                    "model_version": ", ".join(data["model_versions"]),
+                    "model_version": ", ".join(sort_versions(data["model_versions"])),
                 }
                 for param_version, data in version_grouped.items()
             ]
@@ -397,10 +381,9 @@ class ReadParameters:
             if value_data is None:
                 continue
 
-            design_type = names.is_design_type(telescope_model.name)
             file_flag = parameter_data.get("file", False)
             value = self._format_parameter_value(
-                parameter, value_data, unit, file_flag, parameter_version, design_type
+                parameter, value_data, unit, file_flag, parameter_version
             )
 
             description = parameter_descriptions.get(parameter).get("description")
@@ -410,7 +393,7 @@ class ReadParameters:
             inst_class = parameter_descriptions.get(parameter).get("inst_class")
 
             matching_instrument = parameter_data["instrument"] == telescope_model.name
-            if not design_type and matching_instrument:
+            if not names.is_design_type(telescope_model.name) and matching_instrument:
                 parameter = f"***{parameter}***"
                 parameter_version = f"***{parameter_version}***"
                 if not self.is_markdown_link(value):
@@ -453,7 +436,6 @@ class ReadParameters:
             text = short_description if short_description else description
             wrapped_text = textwrap.fill(str(text), column_widths[3]).split("\n")
             wrapped_text = " ".join(wrapped_text)
-            parameter_name = self._wrap_at_underscores(parameter_name, column_widths[0])
 
             file.write(
                 f"| {parameter_name:{column_widths[0]}} |"
@@ -628,33 +610,82 @@ class ReadParameters:
 
             description = parameter_descriptions.get("description")
             with output_filename.open("w", encoding="utf-8") as file:
-                # Write header
-                file.write(
-                    f"# {parameter}\n\n"
-                    f"**Telescope**: {self.array_element}\n\n"
-                    f"**Description**: {description}\n\n"
-                    "\n"
-                )
+                # Write header and table
+                self._write_parameter_header(file, parameter, description)
+                self._write_table_rows(file, parameter, comparison_data)
 
-                # Write table header
-                file.write(
-                    "| Parameter Version      | Model Version(s)      "
-                    "| Value                |\n"
-                    "|------------------------|--------------------"
-                    "|----------------------|\n"
-                )
+                # If entries reference files, write the image/plot section
+                if comparison_data.get(parameter)[0]["file_flag"]:
+                    self._write_file_flag_section(file, parameter, comparison_data)
 
-                # Write table rows
-                for item in comparison_data.get(parameter):
-                    file.write(
-                        f"| {item['parameter_version']} |"
-                        f" {item['model_version']} |"
-                        f"{item['value']} |\n"
-                    )
+    def _write_parameter_header(self, file, parameter, description):
+        """Write the markdown header for a parameter file."""
+        file.write(
+            f"# {parameter}\n\n"
+            f"**Telescope**: {self.array_element}\n\n"
+            f"**Description**: {description}\n\n"
+            "\n"
+        )
 
-                file.write("\n")
-                if comparison_data.get(parameter)[0]["file_flag"]:
-                    file.write(f"![Parameter plot.](/_images/{self.array_element}_{parameter}.png)")
+    def _write_table_rows(self, file, parameter, comparison_data):
+        """Write the comparison table header and rows for a parameter."""
+        # Write table header
+        file.write(
+            "| Parameter Version      | Model Version(s)      "
+            "| Value                |\n"
+            "|------------------------|--------------------"
+            "|----------------------|\n"
+        )
+
+        # Write table rows
+        for item in comparison_data.get(parameter):
+            file.write(
+                f"| {item['parameter_version']} | {item['model_version']} | {item['value']} |\n"
+            )
+
+        file.write("\n")
+
+    def _write_file_flag_section(self, file, parameter, comparison_data):
+        """Write image/plot references when parameter entries include files."""
+        outpath = Path(io_handler.IOHandler().get_output_directory().parent / "_images")
+        latest_parameter_version = max(
+            comparison_data.get(parameter),
+            key=lambda x: tuple(map(int, x["parameter_version"].split("."))),
+        )["parameter_version"]
+
+        all_model_versions = []
+        for item in comparison_data.get(parameter):
+            model_versions = item["model_version"].split(", ")
+            all_model_versions.extend(model_versions)
+
+        latest_model_version = max(all_model_versions, key=lambda x: tuple(map(int, x.split("."))))
+        tel = self._get_telescope_identifier(latest_model_version)
+
+        file.write("The latest parameter version is plotted below.\n\n")
+
+        if parameter != "camera_config_file":
+            plot_name = f"{parameter}_{latest_parameter_version}_{self.site}_{tel}"
+            image_path = outpath / f"{plot_name}.png"
+            file.write(f"![Parameter plot.]({image_path.as_posix()})")
+            return
+
+        # camera_config_file: find latest value and convert markdown link to png filename
+        latest_value = None
+        for item in comparison_data.get(parameter):
+            if latest_model_version in item["model_version"].split(", "):
+                latest_value = item["value"]
+                break
+
+        if latest_value is None:
+            return
+
+        match = MARKDOWN_LINK_RE.search(latest_value)
+        if not match:
+            return
+
+        filename_png = Path(match.group(1)).with_suffix(".png").name
+        image_path = outpath / filename_png
+        file.write(f"![Camera configuration plot.]({image_path.as_posix()})")
 
     def _write_array_layouts_section(self, file, layouts):
         """Write the array layouts section of the report."""
@@ -840,20 +871,14 @@ class ReadParameters:
 
     def produce_calibration_reports(self):
         """Write calibration reports."""
-        calibration_array_elements = self.db.get_array_elements(
-            self.model_version, collection="calibration_devices"
-        )
-        array_elements = calibration_array_elements.copy()
-        for element in calibration_array_elements:
-            design_model = self.db.get_design_model(
-                self.model_version, element, "calibration_devices"
-            )
-            if design_model and design_model not in array_elements:
-                array_elements.append(design_model)
+        array_elements = self._collect_calibration_array_elements()
 
         for calibration_device in array_elements:
+            device_sites = names.get_site_from_array_element_name(calibration_device)
+            site = device_sites[0] if isinstance(device_sites, list) else device_sites
+
             all_parameter_data = self.db.get_model_parameters(
-                site=names.get_site_from_array_element_name(calibration_device),
+                site=site,
                 array_element_name=calibration_device,
                 collection="calibration_devices",
                 model_version=self.model_version,
@@ -867,32 +892,77 @@ class ReadParameters:
                 self.model_version, calibration_device, "calibration_devices"
             )
 
-            with output_filename.open("w", encoding="utf-8") as file:
-                file.write(f"# {calibration_device}\n")
-                file.write("\n\n")
+            self._write_single_calibration_report(
+                output_filename, calibration_device, data, design_model
+            )
 
-                if not names.is_design_type(calibration_device):
-                    file.write(
-                        "The design model can be found here: "
-                        f"[{design_model}]"
-                        f"({design_model}.md).\n\n"
-                    )
-                    file.write(
-                        "Parameters shown in ***bold and italics*** are specific"
-                        " to each array element.\n"
-                        "Parameters without emphasis are inherited from the design model.\n"
-                    )
-                    file.write("\n\n")
+        # produce parameter comparison reports (site-independent)
+        self._generate_model_parameter_reports_for_devices(array_elements)
 
-                for class_name, group in groupby(data, key=lambda x: x[0]):
-                    group = sorted(group, key=lambda x: x[1])
-                    file.write(f"## {class_name}\n\n")
-                    self._write_to_file(group, file)
+    def _collect_calibration_array_elements(self):
+        """Return a list of calibration devices including their design models."""
+        calibration_array_elements = self.db.get_array_elements(
+            self.model_version, collection="calibration_devices"
+        )
+        array_elements = calibration_array_elements.copy()
+        for element in calibration_array_elements:
+            design_model = self.db.get_design_model(
+                self.model_version, element, "calibration_devices"
+            )
+            if design_model and design_model not in array_elements:
+                array_elements.append(design_model)
+        return array_elements
+
+    def _write_single_calibration_report(
+        self, output_filename, calibration_device, data, design_model
+    ):
+        """Write a single calibration device markdown report file."""
+        with output_filename.open("w", encoding="utf-8") as file:
+            file.write(f"# {calibration_device}\n")
+            file.write("\n\n")
 
+            if not names.is_design_type(calibration_device):
+                file.write(
+                    f"The design model can be found here: [{design_model}]({design_model}.md).\n\n"
+                )
+                file.write(
+                    "Parameters shown in ***bold and italics*** are specific"
+                    " to each array element.\n"
+                    "Parameters without emphasis are inherited from the design model.\n"
+                )
+                file.write("\n\n")
+
+            for class_name, group in groupby(data, key=lambda x: x[0]):
+                group = sorted(group, key=lambda x: x[1])
+                file.write(f"## {class_name}\n\n")
+                # Transform parameter display names for human-readable report
+                display_group = []
+                for row in group:
+                    param = row[1]
+                    # Preserve markdown emphasis (***param***) if present
+                    if isinstance(param, str) and param.startswith("***") and param.endswith("***"):
+                        inner = param.strip("*")
+                        new_inner = inner.replace("_", " ")
+                        new_param = f"***{new_inner}***"
+                    elif isinstance(param, str):
+                        new_param = param.replace("_", " ")
+                    else:
+                        new_param = param
+
+                    new_row = [row[0], new_param, row[2], row[3], row[4], row[5]]
+                    display_group.append(new_row)
+
+                self._write_to_file(display_group, file)
+
+    def _generate_model_parameter_reports_for_devices(self, array_elements):
+        """Create model-parameter comparison reports for calibration devices."""
         new_output_path = Path(self.output_path).parent.parent / "parameters"
         new_output_path.mkdir(parents=True, exist_ok=True)
         self.output_path = new_output_path
         for calibration_device in array_elements:
-            self.site = names.get_site_from_array_element_name(calibration_device)
+            device_sites = names.get_site_from_array_element_name(calibration_device)
+            # parameters are site independent so just take the first site to read from db
+            site = device_sites[0] if isinstance(device_sites, list) else device_sites
+            self.site = site
             self.array_element = calibration_device
             self.produce_model_parameter_reports(collection="calibration_devices")

simtools/resources/array_elements.yml

@@ -131,6 +131,32 @@ data:
     observatory: "CTAO"
     site: "South"
     description: "Lightning sensor"
+  MSFx:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: ["North", "South"]
+    design_types: ["FlashCam", "NectarCam", "test"]
+    description: "Flat-fielding device (MST)"
+  LSFN:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "North"
+    description: "Flat-fielding device (LST)"
+  LSFS:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "South"
+    description: "Flat-fielding device (LST)"
+  SSFS:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "South"
+    description: "Flat-fielding device (SST)"
+  SCFS:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "South"
+    description: "Flat-fielding device (SCT)"
   HESS:
     collection: "telescopes"
     site: "South"

simtools/runners/corsika_simtel_runner.py

@@ -4,6 +4,7 @@ import logging
 import stat
 from pathlib import Path
 
+import simtools.utils.general as gen
 from simtools.runners.corsika_runner import CorsikaRunner
 from simtools.simtel.simulator_array import SimulatorArray
 
@@ -32,6 +33,8 @@ class CorsikaSimtelRunner:
         Use multipipe to run CORSIKA and sim_telarray.
     sim_telarray_seeds : dict
         Dictionary with configuration for sim_telarray random instrument setup.
+    calibration_config : dict
+        Configuration for the calibration of the sim_telarray data.
     """
 
     def __init__(
@@ -43,12 +46,10 @@ class CorsikaSimtelRunner:
         use_multipipe=False,
         sim_telarray_seeds=None,
         sequential=False,
-        calibration_runner_args=None,
+        calibration_config=None,
     ):
         self._logger = logging.getLogger(__name__)
-        self.corsika_config = (
-            corsika_config if isinstance(corsika_config, list) else [corsika_config]
-        )
+        self.corsika_config = gen.ensure_iterable(corsika_config)
         # the base corsika config is the one used to define the CORSIKA specific parameters.
         # The others are used for the array configurations.
         self.base_corsika_config = self.corsika_config[0]
@@ -56,7 +57,6 @@ class CorsikaSimtelRunner:
         self.sim_telarray_seeds = sim_telarray_seeds
         self.label = label
         self.sequential = "--sequential" if sequential else ""
-        self.calibration_runner_args = calibration_runner_args
 
         self.base_corsika_config.set_output_file_and_directory(use_multipipe)
         self.corsika_runner = CorsikaRunner(
@@ -77,6 +77,7 @@ class CorsikaSimtelRunner:
                     label=label,
                     use_multipipe=use_multipipe,
                     sim_telarray_seeds=sim_telarray_seeds,
+                    calibration_config=calibration_config,
                 )
             )
 
@@ -126,18 +127,11 @@ class CorsikaSimtelRunner:
 
         with open(multipipe_file, "w", encoding="utf-8") as file:
             for simulator_array in self.simulator_array:
-                if self.calibration_runner_args:
-                    run_command = simulator_array.make_run_command_for_calibration_simulations(
-                        run_number=run_number,
-                        input_file="-",  # instruct sim_telarray to take input from standard output
-                        calibration_runner_args=self.calibration_runner_args,
-                    )
-                else:
-                    run_command = simulator_array.make_run_command(
-                        run_number=run_number,
-                        input_file="-",  # instruct sim_telarray to take input from standard output
-                        weak_pointing=self._determine_pointing_option(self.label),
-                    )
+                run_command = simulator_array.make_run_command(
+                    run_number=run_number,
+                    input_file="-",  # instruct sim_telarray to take input from standard output
+                    weak_pointing=self._determine_pointing_option(self.label),
+                )
                 file.write(f"{run_command}")
                 file.write("\n")
         self._logger.info(f"Multipipe script: {multipipe_file}")

simtools/runners/runner_services.py

@@ -90,17 +90,16 @@ class RunnerServices:
         dict
             Dictionary containing paths requires for simulation configuration.
         """
-        self.directory["output"] = io_handler.IOHandler().get_output_directory(self.label)
+        ioh = io_handler.IOHandler()
+        self.directory["output"] = ioh.get_output_directory()
         _logger.debug(f"Creating output dir {self.directory['output']}")
         for dir_name in ["sub_scripts", "sub_logs"]:
-            self.directory[dir_name] = self.directory["output"].joinpath(dir_name)
-            self.directory[dir_name].mkdir(parents=True, exist_ok=True)
+            self.directory[dir_name] = ioh.get_output_directory(dir_name)
         for _simulation_software in self._get_simulation_software_list(simulation_software):
             for dir_name in ["data", "inputs", "logs"]:
-                self.directory[dir_name] = self.directory["output"].joinpath(
-                    _simulation_software, dir_name
+                self.directory[dir_name] = ioh.get_output_directory(
+                    [_simulation_software, dir_name]
                 )
-                self.directory[dir_name].mkdir(parents=True, exist_ok=True)
         self._logger.debug(f"Data directories for {simulation_software}: {self.directory}")
         return self.directory
 

simtools/runners/simtools_runner.py

@@ -122,7 +122,6 @@ def _read_application_configuration(configuration_file, steps, logger):
     - output files will be written out to './output/<workflow directory>/'
 
     Replaces the placeholders in the configuration file with the actual values.
-    Sets 'USE_PLAIN_OUTPUT_PATH' to True for all applications.
 
     Parameters
     ----------
@@ -195,7 +194,6 @@ def _replace_placeholders_in_configuration(
                 for item in value
             ]
     if output_path:
-        configuration["use_plain_output_path"] = True
         configuration["output_path"] = str(output_path)
 
     return configuration

simtools/schemas/application_workflow.metaschema.yml

@@ -148,7 +148,7 @@ definitions:
         type: object
         additionalProperties: false
         patternProperties:
-          "^[0-9]+\\.[0-9]+\\.[0-9]+$":  # Semantic versioning pattern (e.g., "5.0.0", "6.0.0")
+          "^[0-9]+\\.[0-9]+(?:\\.[0-9]+)?$":  # Version pattern (e.g., "5.0", "6.0", "5.0.0", "6.0.0")
             type: string
             description: Path to the configuration file for the given version.
         minProperties: 1

simtools/schemas/common_definitions.schema.yml

@@ -0,0 +1,39 @@
+---
+$schema: http://json-schema.org/draft-06/schema#
+$id: common_definitions.schema.yml
+title: Common Schema Definitions
+description: |
+  Common definitions and patterns used across multiple schema files
+schema_version: 0.1.0
+schema_name: common_definitions
+type: object
+
+$defs:
+  telescope_name_pattern:
+    type: string
+    description: |
+      Pattern for valid telescope names including:
+      - Standard telescope types (LST, MST, SST, CT) with site (N/S/x) and number
+      - Design telescopes with specific camera types (FlashCam, NectarCam)
+      - Observatory sites (North/South)
+      - Dummy telescopes for testing
+    pattern: '^([A-Za-z](ST|LL|CT|SF)[N,S,x]-\d{2,3}|[A-Za-z](ST|LL|CT|SF)[N,S,x]-(design|FlashCam|NectarCam)|OBS-(North|South)|Dummy-Telescope)$'
+    examples:
+      - "LSTN-01"
+      - "MSTN-02"
+      - "SSTN-03"
+      - "LSTN-design"
+      - "MSTx-FlashCam"
+      - "SSTx-NectarCam"
+      - "OBS-North"
+      - "OBS-South"
+      - "Dummy-Telescope"
+
+  semantic_version_pattern:
+    type: string
+    description: Semantic version pattern (MAJOR.MINOR.PATCH)
+    pattern: '^\d+\.\d+\.\d+$'
+    examples:
+      - "1.0.0"
+      - "2.1.3"
+      - "0.5.12"

simtools/schemas/model_parameter.metaschema.yml

@@ -51,20 +51,24 @@ definitions:
                 - South
         description: "Associated CTAO site."
       type:
-        type: string
         description: "Data type"
-        enum:
-          - boolean
-          - dict
-          - double
-          - file
-          - float64
-          - int
-          - int64
-          - string
-          - uint
-          - uint32
-          - uint64
+        oneOf:
+          - type: string
+            enum: &data_types
+              - boolean
+              - dict
+              - double
+              - file
+              - float64
+              - int
+              - int64
+              - string
+              - uint
+              - uint32
+              - uint64
+          - type: array
+            items:
+              enum: *data_types
       unit:
         anyOf:
           - type: string
@@ -82,6 +86,7 @@ definitions:
           - type: string
           - type: "null"
           - type: array
+          - type: object  # allow dictionary
         description: "Value of the parameter."
       parameter_version:
         anyOf:
@@ -178,6 +183,7 @@ definitions:
           - type: string
           - type: "null"
           - type: array
+          - type: object
         description: "Value of the parameter."
       parameter_version:
         anyOf: