gammasimtools 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

simtools/reporting/docs_read_parameters.py

@@ -0,0 +1,310 @@
+#!/usr/bin/python3
+
+r"""Class to read and manage relevant model parameters for a given telescope model."""
+
+import logging
+import textwrap
+from itertools import groupby
+from pathlib import Path
+
+from simtools.io_operations import io_handler
+from simtools.model.telescope_model import TelescopeModel
+from simtools.utils import names
+
+logger = logging.getLogger()
+
+
+class ReadParameters:
+    """Read and manage model parameter data."""
+
+    def __init__(self, db_config, telescope_model, output_path):
+        """Initialise class with a telescope model."""
+        self._logger = logging.getLogger(__name__)
+        self.db_config = db_config
+        self.telescope_model = telescope_model
+        self.output_path = output_path
+
+    def _convert_to_md(self, input_file):
+        """
+        Convert a '.dat' or '.ecsv' file to a Markdown file, preserving formatting.
+
+        Parameters
+        ----------
+        input_file: Path, str
+           Simulation data file (in '.dat' or '.ecsv' format).
+
+        Returns
+        -------
+        - Path to the created Markdown file.
+        """
+        input_file = Path(input_file)
+        output_data_path = Path(self.output_path / "_data_files")
+        output_data_path.mkdir(parents=True, exist_ok=True)
+        output_file_name = Path(input_file.stem + ".md")
+        output_file = output_data_path / output_file_name
+
+        try:
+            with (
+                input_file.open("r", encoding="utf-8") as infile,
+                output_file.open("w", encoding="utf-8") as outfile,
+            ):
+                outfile.write(f"# {input_file.stem}")
+                outfile.write("\n")
+                outfile.write("```")
+                outfile.write("\n")
+                file_contents = infile.read()
+                outfile.write(file_contents)
+                outfile.write("\n")
+                outfile.write("```")
+
+        except FileNotFoundError as exc:
+            logger.exception(f"Data file not found: {input_file}.")
+            raise FileNotFoundError(f"Data file not found: {input_file}.") from exc
+
+        return f"_data_files/{output_file_name}"
+
+    def get_all_parameter_descriptions(self):
+        """
+        Get descriptions for all model parameters.
+
+        Returns
+        -------
+            tuple: A tuple containing two dictionaries:
+                - parameter_description: Maps parameter names to their descriptions.
+                - short_description: Maps parameter names to their short descriptions.
+                - inst_class: Maps parameter names to their respective class.
+        """
+        parameter_description, short_description, inst_class = {}, {}, {}
+
+        for instrument_class in names.db_collection_to_instrument_class_key("telescopes"):
+            for parameter, details in names.model_parameters(instrument_class).items():
+                parameter_description[parameter] = details.get("description")
+                short_description[parameter] = details.get("short_description")
+                inst_class[parameter] = instrument_class
+
+        return parameter_description, short_description, inst_class
+
+    def get_array_element_parameter_data(self, telescope_model, collection="telescopes"):
+        """
+        Get model parameter data for a given array element.
+
+        Currently only configures for telescope.
+
+        Parameters
+        ----------
+        telescope_model : TelescopeModel
+            The telescope model instance.
+
+        Returns
+        -------
+        list: A list of lists containing parameter names, values with units,
+                descriptions, and short descriptions.
+        """
+        all_params = telescope_model.db.get_model_parameters(
+            site=telescope_model.site,
+            array_element_name=telescope_model.name,
+            collection=collection,
+        )
+
+        telescope_model.export_model_files()
+        parameter_descriptions = self.get_all_parameter_descriptions()
+        data = []
+
+        for parameter in all_params:
+            if all_params[parameter]["instrument"] != telescope_model.name:
+                continue
+            parameter_version = telescope_model.get_parameter_version(parameter)
+            value = telescope_model.get_parameter_value_with_unit(parameter)
+            if telescope_model.get_parameter_file_flag(parameter) and value:
+                input_file_name = telescope_model.config_file_directory / Path(value)
+                output_file_name = self._convert_to_md(input_file_name)
+                value = f"[{Path(value).name}]({output_file_name})"
+            elif isinstance(value, list):
+                value = ", ".join(str(q) for q in value)
+            else:
+                value = str(value)
+
+            description = parameter_descriptions[0].get(parameter)
+            short_description = parameter_descriptions[1].get(parameter, description)
+            inst_class = parameter_descriptions[2].get(parameter)
+            data.append(
+                [
+                    inst_class,
+                    parameter,
+                    parameter_version,
+                    value,
+                    description,
+                    short_description,
+                ]
+            )
+
+        return data
+
+    def _compare_parameter_across_versions(self, parameter_name):
+        """
+        Compare a parameter's value across different model versions.
+
+        Parameters
+        ----------
+        parameter_name : str
+            The name of the parameter to compare.
+
+        Returns
+        -------
+        list
+            A list of dictionaries containing model version, parameter value, and description.
+        """
+        all_versions = self.telescope_model.db.get_model_versions()
+        all_versions.reverse()
+        comparison_data = []
+
+        for model_version in all_versions:
+            telescope_model = TelescopeModel(
+                site=self.telescope_model.site,
+                telescope_name=self.telescope_model.name,
+                model_version=model_version,
+                label="reports",
+                mongo_db_config=self.db_config,
+            )
+
+            if not telescope_model.has_parameter(parameter_name):
+                return comparison_data
+
+            parameter_data = self.get_array_element_parameter_data(telescope_model)
+            for param in parameter_data:
+                if param[1] == parameter_name:
+                    comparison_data.append(
+                        {
+                            "model_version": model_version,
+                            "parameter_version": param[2],
+                            "value": param[3],
+                            "description": param[4],
+                        }
+                    )
+                    break
+        return comparison_data
+
+    def produce_array_element_report(self):
+        """
+        Produce a markdown report of all model parameters per array element.
+
+        Output
+        ----------
+        One markdown report of a given array element listing parameter values,
+        versions, and descriptions.
+        """
+        output_filename = Path(self.output_path / (self.telescope_model.name + ".md"))
+        output_filename.parent.mkdir(parents=True, exist_ok=True)
+        data = self.get_array_element_parameter_data(self.telescope_model)
+        # Sort data by class to prepare for grouping
+        if not isinstance(data, str):
+            data.sort(key=lambda x: (x[0], x[1]), reverse=True)
+
+        with output_filename.open("w", encoding="utf-8") as file:
+            # Group by class and write sections
+            file.write(f"# {self.telescope_model.name}\n")
+
+            if self.telescope_model.name != self.telescope_model.design_model:
+                file.write(
+                    "The design model can be found here: "
+                    f"[{self.telescope_model.design_model}]"
+                    f"({self.telescope_model.design_model}.md).\n"
+                )
+                file.write("\n\n")
+
+            for class_name, group in groupby(data, key=lambda x: x[0]):
+                group = sorted(group, key=lambda x: x[1])
+                file.write(f"## {class_name}\n\n")
+
+                # Write table header and separator row
+                file.write(
+                    "| Parameter Name      |  Parameter Version     "
+                    "| Values      | Short Description           |\n"
+                    "|---------------------|------------------------"
+                    "|-------------|-----------------------------|\n"
+                )
+
+                # Write table rows
+                column_widths = [20, 20, 20, 70]
+                for (
+                    _,
+                    parameter_name,
+                    parameter_version,
+                    value,
+                    description,
+                    short_description,
+                ) in group:
+                    text = short_description if short_description else description
+                    wrapped_text = textwrap.fill(str(text), column_widths[3]).split("\n")
+                    wrapped_text = " ".join(wrapped_text)
+                    file.write(
+                        f"| {parameter_name:{column_widths[0]}} |"
+                        f" {parameter_version:{column_widths[1]}} |"
+                        f" {value:{column_widths[2]}} |"
+                        f" {wrapped_text} |\n"
+                    )
+                file.write("\n\n")
+
+    def produce_model_parameter_reports(self):
+        """
+        Produce a markdown report per parameter for a given array element.
+
+        Output
+        ----------
+        One markdown report per model parameter of a given array element comparing
+        values across model versions.
+        """
+        logger.info(
+            f"Comparing parameters across model versions for Telescope: {self.telescope_model.name}"
+            f" and Site: {self.telescope_model.site}."
+        )
+        io_handler_instance = io_handler.IOHandler()
+        output_path = io_handler_instance.get_output_directory(
+            label="reports", sub_dir=f"parameters/{self.telescope_model.name}"
+        )
+
+        all_params = self.telescope_model.db.get_model_parameters(
+            site=self.telescope_model.site,
+            array_element_name=self.telescope_model.name,
+            collection="telescopes",
+        )
+
+        for parameter in all_params:
+            comparison_data = []
+            if all_params[parameter]["instrument"] == self.telescope_model.name:
+                comparison_data = self._compare_parameter_across_versions(parameter)
+            if comparison_data:
+                output_filename = output_path / f"{parameter}.md"
+                with output_filename.open("w", encoding="utf-8") as file:
+                    # Write header
+                    file.write(
+                        f"# {parameter}\n\n"
+                        f"**Telescope**: {self.telescope_model.name}\n\n"
+                        f"**Description**: {comparison_data[0]['description']}\n\n"
+                        "\n"
+                    )
+
+                    # Write table header
+                    file.write(
+                        "| Model Version      | Parameter Version      "
+                        "| Value                |\n"
+                        "|--------------------|------------------------"
+                        "|----------------------|\n"
+                    )
+
+                    # Write table rows
+                    for item in comparison_data:
+                        file.write(
+                            f"| {item['model_version']} |"
+                            f" {item['parameter_version']} |"
+                            f"{item['value'].replace('](', '](../')} |\n"
+                        )
+
+                    file.write("\n")
+                    if isinstance(comparison_data[0]["value"], str) and comparison_data[0][
+                        "value"
+                    ].endswith(".md)"):
+                        file.write(
+                            f"![Parameter plot.](_images/"
+                            f"{self.telescope_model.name}_{parameter}.png)"
+                        )

simtools/runners/corsika_simtel_runner.py

@@ -6,6 +6,7 @@ from pathlib import Path
 
 from simtools.runners.corsika_runner import CorsikaRunner
 from simtools.simtel.simulator_array import SimulatorArray
+from simtools.utils.general import clear_default_sim_telarray_cfg_directories
 
 __all__ = ["CorsikaSimtelRunner"]
 
@@ -128,9 +129,7 @@ class CorsikaSimtelRunner:
         )
         with open(multipipe_script, "w", encoding="utf-8") as file:
             multipipe_command = Path(self._simtel_path).joinpath(
-                "sim_telarray/bin/multipipe_corsika "
-                f"-c {multipipe_file}"
-                " || echo 'Fan-out failed'"
+                f"sim_telarray/bin/multipipe_corsika -c {multipipe_file} || echo 'Fan-out failed'"
             )
             file.write(f"{multipipe_command}")
 
@@ -197,7 +196,8 @@ class CorsikaSimtelRunner:
         )
         command += f" | gzip > {_log_file} 2>&1 || exit"
 
-        return command
+        # Remove the default sim_telarray configuration directories
+        return clear_default_sim_telarray_cfg_directories(command)
 
     def get_file_name(self, simulation_software=None, file_type=None, run_number=None, mode=None):
         """

simtools/schemas/{integration_tests_config.metaschema.yml → application_workflow.metaschema.yml}

@@ -1,50 +1,74 @@
 ---
 $schema: http://json-schema.org/draft-06/schema#
-$ref: '#/definitions/SimtoolsIntegrationTestConfiguration'
-title: SimPipe integration test configuration metaschema
-description: YAML representation of integration test configuration metaschema
-version: 0.2.0
-name: integration_tests_config.metaschema
+$ref: '#/definitions/SimtoolsApplicationWorkflowConfiguration'
+title: SimPipe application workflow configuration metaschema
+description: YAML representation of application workflow configuration metaschema
+version: 0.3.0
+name: application_workflow.metaschema
 type: object
 additionalProperties: false
 
-
 definitions:
-  SimtoolsIntegrationTestConfiguration:
+  SimtoolsApplicationWorkflowConfiguration:
     type: object
     additionalProperties: false
     properties:
       CTA_SIMPIPE:
         "$ref": "#/definitions/CTASIMPIPE"
+      SCHEMA_VERSION:
+        type: string
+        description: "Version of the schema."
+      SCHEMA_URL:
+        type: string
+        format: uri
+        description: "URL of the schema."
+      SCHEMA_NAME:
+        type: string
+        description: "Name of the schema."
     required:
       - CTA_SIMPIPE
+      - SCHEMA_VERSION
+      - SCHEMA_NAME
     title: ""
   CTASIMPIPE:
     type: object
     additionalProperties: false
     properties:
-      APPLICATION:
+      LOG_PATH:
         type: string
-        description: "Name of simtools application"
-      TEST_NAME:
-        type: string
-        description: "Test name. Should be unique for this application."
-      MODEL_VERSION_USE_CURRENT:
-        type: boolean
-        description: |
-          "Skip test if model version is changed via command line.
-          Important e.g., for cases where DB model is incomplete for some
-          model versions."
-      CONFIGURATION:
-        "$ref": "#/definitions/Configuration"
-      INTEGRATION_TESTS:
+        description: "Path to the log file."
+      APPLICATIONS:
         type: array
         items:
-          "$ref": "#/definitions/IntegrationTest"
-    required:
-      - APPLICATION
-      - CONFIGURATION
-      - TEST_NAME
+          type: object
+          properties:
+            APPLICATION:
+              type: string
+              description: "Name of simtools application"
+            TEST_NAME:
+              type: string
+              description: "Test name. Should be unique for this application."
+            MODEL_VERSION_USE_CURRENT:
+              type: boolean
+              description: |
+                "Skip test if model version is changed via command line.
+                Important e.g., for cases where DB model is incomplete for some
+                model versions."
+            CONFIGURATION:
+              "$ref": "#/definitions/Configuration"
+            INTEGRATION_TESTS:
+              type: array
+              items:
+                "$ref": "#/definitions/IntegrationTest"
+            TEST_REQUIREMENT:
+              type: string
+              description: "Requirement ID of the test."
+            TEST_USE_CASE:
+              type: string
+              description: "Use case ID of the test."
+          required:
+            - APPLICATION
+            - CONFIGURATION
     title: CTASimpipe
   Configuration:
     description: "Command line configuration of simtools application."
@@ -82,6 +106,16 @@ definitions:
         description: |
           "Reference file used for comparison."
         type: string
+      TEST_SIMTEL_CFG_FILES:
+        description: |
+          "Reference file used for comparison of sim_telarray configuration files."
+        type: object
+        additionalProperties: false
+        patternProperties:
+          "^[0-9]+\\.[0-9]+\\.[0-9]+$":  # Semantic versioning pattern (e.g., "5.0.0", "6.0.0")
+            type: string
+            description: Path to the configuration file for the given version.
+        minProperties: 1
       TOLERANCE:
         description: "Allowed tolerance for floating point comparison."
         type: number
@@ -100,4 +134,4 @@ definitions:
             CUT_CONDITION:
               description: "Condition to apply to the cut column."
               type: string
-    title: IntegrationTest
+    title: ApplicationWorkflow

simtools/schemas/array_elements.yml

@@ -15,6 +15,7 @@ data:
     collection: "telescopes"
     observatory: "CTAO"
     site: "North"
+    design_types: ["FlashCam", "NectarCam", "test"]
   LSTS:
     collection: "telescopes"
     observatory: "CTAO"
@@ -23,6 +24,7 @@ data:
     collection: "telescopes"
     observatory: "CTAO"
     site: "South"
+    design_types: ["FlashCam", "NectarCam", "test"]
   SSTS:
     collection: "telescopes"
     observatory: "CTAO"
@@ -33,6 +35,12 @@ data:
     observatory: "CTAO"
     site: "South"
     description: "Mid-sized Schwarzschild-Couder telescope"
+  MSTx:
+    collection: "telescopes"
+    observatory: "CTAO"
+    site: ["North", "South"]
+    design_types: ["FlashCam", "NectarCam", "test"]
+    description: "Mid-sized telescope"
   ILLN:
     collection: "calibration_devices"
     observatory: "CTAO"

simtools/schemas/input/MST_mirror_2f_measurements.schema.yml

@@ -0,0 +1,39 @@
+%YAML 1.2
+---
+title: Schema for mirror panel 2F measurements
+version: 0.1.0
+meta_schema: simpipe-schema
+meta_schema_url: https://raw.githubusercontent.com/gammasim/simtools/main/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml
+meta_schema_version: 0.1.0
+name: mirror_2f_measurement
+description: |-
+  Mirror panel 2F measurements determining
+  panel radius and panel PSF.
+data:
+  - type: data_table
+    table_columns:
+      - name: mirror_panel_id
+        description: |-
+          Mirror panel ID.
+        required: true
+        unit: dimensionless
+        type: string
+      - name: mirror_curvature_radius
+        description: |-
+          Mirror panel curvature radius.
+        required: true
+        unit: cm
+        type: float64
+      - name: psf
+        description: |-
+          Spot size of mirror panel PSF at nominal distance.
+        required: true
+        unit: cm
+        type: float64
+      - name: psf_opt
+        description: |-
+          Spot size of mirror panel PSF
+          (best value, not at nominal distance).
+        required: false
+        unit: cm
+        type: float64

simtools/schemas/input/single_pe_spectrum.schema.yml

@@ -0,0 +1,38 @@
+%YAML 1.2
+---
+title: Schema for single photon electron spectrum
+version: 0.1.0
+meta_schema: simpipe-schema
+meta_schema_url: https://raw.githubusercontent.com/gammasim/simtools/main/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml
+meta_schema_version: 0.1.0
+name: single_pe_spectrum
+description: |-
+  Single photo-electron response as a function of signal amplitude.
+  May include prompt, prompt plus afterpulse, or afterpulse response.
+data:
+  - type: data_table
+    table_columns:
+      - name: amplitude
+        description: |-
+          Signal amplitude (p.e.)
+        required: true
+        unit: dimensionless
+        type: float64
+      - name: frequency (prompt)
+        description: |-
+          Single photo-electron response (prompt)
+        required: false
+        unit: dimensionless
+        type: float64
+      - name: frequency (afterpulsing)
+        description: |-
+          Single photo-electron response (afterpulsing)
+        required: false
+        unit: dimensionless
+        type: float64
+      - name: frequency (prompt+afterpulsing)
+        description: |-
+          Single photo-electron response (prompt+afterpulsing)
+        required: false
+        unit: dimensionless
+        type: float64

simtools/schemas/model_parameter.metaschema.yml

@@ -5,7 +5,7 @@ title: SimPipe DB Model Parameter Metaschema
 description: |
   YAML representation of db model parameter metaschema
   (based on simulation model DB).
-version: 0.2.0
+version: 0.3.0
 name: modelparameter.metaschema
 type: object
 additionalProperties: false
@@ -25,14 +25,115 @@ definitions:
         type: boolean
         description: "This parameter is a file."
       instrument:
-        type: string
+        type:
+          - string
+          - "null"
         description: "Associated instrument."
       site:
+        anyOf:
+          - type: string
+            enum:
+              - North
+              - South
+          - type: "null"
+          - type: array
+            items:
+              type: string
+              enum:
+                - North
+                - South
+        description: "Associated CTAO site."
+      type:
         type: string
+        description: "Data type"
+        enum:
+          - boolean
+          - dict
+          - double
+          - file
+          - float64
+          - int
+          - int64
+          - string
+          - uint
+          - uint32
+          - uint64
+      unit:
+        anyOf:
+          - type: string
+          - type: "null"
+        description: "Unit of the parameter."
+      value:
+        anyOf:
+          - type: boolean
+          - type: number
+          - type: string
+          - type: "null"
+          - type: array
+        description: "Value of the parameter."
+      parameter_version:
+        anyOf:
+          - type: string
+        description: "Parameter version."
+      schema_version:
+        anyOf:
+          - type: string
+        description: "Metaschema version."
+      unique_id:
+        anyOf:
+          - type: string
+          - type: "null"
+        description: "Unique ID of parameter definition."
+    required:
+      - file
+      - instrument
+      - parameter_version
+      - schema_version
+      - site
+      - type
+      - unit
+      - value
+...
+---
+$schema: http://json-schema.org/draft-06/schema#
+$ref: '#/definitions/SimtoolsModelParameters'
+title: SimPipe DB Model Parameter Metaschema
+description: |
+  YAML representation of db model parameter metaschema
+  (based on simulation model DB).
+version: 0.2.0
+name: modelparameter.metaschema
+type: object
+additionalProperties: false
+
+definitions:
+  SimtoolsModelParameters:
+    description: ""
+    type: object
+    properties:
+      _id:
+        type: string
+        description: "DB unique identifier."
+      entry_date:
+        type: string
+        description: "Value entry date."
+      file:
+        type: boolean
+        description: "This parameter is a file."
+      instrument:
+        type:
+          - string
+          - "null"
+        description: "Associated instrument."
+      site:
+        type:
+          - string
+          - "null"
         description: "Associated CTAO site."
         enum:
           - North
           - South
+          - null
       type:
         type: string
         description: "Data type"