gammasimtools 0.10.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

simtools/reporting/docs_read_parameters.py

@@ -0,0 +1,310 @@
+#!/usr/bin/python3
+
+r"""Class to read and manage relevant model parameters for a given telescope model."""
+
+import logging
+import textwrap
+from itertools import groupby
+from pathlib import Path
+
+from simtools.io_operations import io_handler
+from simtools.model.telescope_model import TelescopeModel
+from simtools.utils import names
+
+logger = logging.getLogger()
+
+
+class ReadParameters:
+    """Read and manage model parameter data."""
+
+    def __init__(self, db_config, telescope_model, output_path):
+        """Initialise class with a telescope model."""
+        self._logger = logging.getLogger(__name__)
+        self.db_config = db_config
+        self.telescope_model = telescope_model
+        self.output_path = output_path
+
+    def _convert_to_md(self, input_file):
+        """
+        Convert a '.dat' or '.ecsv' file to a Markdown file, preserving formatting.
+
+        Parameters
+        ----------
+        input_file: Path, str
+           Simulation data file (in '.dat' or '.ecsv' format).
+
+        Returns
+        -------
+        - Path to the created Markdown file.
+        """
+        input_file = Path(input_file)
+        output_data_path = Path(self.output_path / "_data_files")
+        output_data_path.mkdir(parents=True, exist_ok=True)
+        output_file_name = Path(input_file.stem + ".md")
+        output_file = output_data_path / output_file_name
+
+        try:
+            with (
+                input_file.open("r", encoding="utf-8") as infile,
+                output_file.open("w", encoding="utf-8") as outfile,
+            ):
+                outfile.write(f"# {input_file.stem}")
+                outfile.write("\n")
+                outfile.write("```")
+                outfile.write("\n")
+                file_contents = infile.read()
+                outfile.write(file_contents)
+                outfile.write("\n")
+                outfile.write("```")
+
+        except FileNotFoundError as exc:
+            logger.exception(f"Data file not found: {input_file}.")
+            raise FileNotFoundError(f"Data file not found: {input_file}.") from exc
+
+        return f"_data_files/{output_file_name}"
+
+    def get_all_parameter_descriptions(self):
+        """
+        Get descriptions for all model parameters.
+
+        Returns
+        -------
+            tuple: A tuple containing two dictionaries:
+                - parameter_description: Maps parameter names to their descriptions.
+                - short_description: Maps parameter names to their short descriptions.
+                - inst_class: Maps parameter names to their respective class.
+        """
+        parameter_description, short_description, inst_class = {}, {}, {}
+
+        for instrument_class in names.db_collection_to_instrument_class_key("telescopes"):
+            for parameter, details in names.model_parameters(instrument_class).items():
+                parameter_description[parameter] = details.get("description")
+                short_description[parameter] = details.get("short_description")
+                inst_class[parameter] = instrument_class
+
+        return parameter_description, short_description, inst_class
+
+    def get_array_element_parameter_data(self, telescope_model, collection="telescopes"):
+        """
+        Get model parameter data for a given array element.
+
+        Currently only configures for telescope.
+
+        Parameters
+        ----------
+        telescope_model : TelescopeModel
+            The telescope model instance.
+
+        Returns
+        -------
+        list: A list of lists containing parameter names, values with units,
+                descriptions, and short descriptions.
+        """
+        all_params = telescope_model.db.get_model_parameters(
+            site=telescope_model.site,
+            array_element_name=telescope_model.name,
+            collection=collection,
+        )
+
+        telescope_model.export_model_files()
+        parameter_descriptions = self.get_all_parameter_descriptions()
+        data = []
+
+        for parameter in all_params:
+            if all_params[parameter]["instrument"] != telescope_model.name:
+                continue
+            parameter_version = telescope_model.get_parameter_version(parameter)
+            value = telescope_model.get_parameter_value_with_unit(parameter)
+            if telescope_model.get_parameter_file_flag(parameter) and value:
+                input_file_name = telescope_model.config_file_directory / Path(value)
+                output_file_name = self._convert_to_md(input_file_name)
+                value = f"[{Path(value).name}]({output_file_name})"
+            elif isinstance(value, list):
+                value = ", ".join(str(q) for q in value)
+            else:
+                value = str(value)
+
+            description = parameter_descriptions[0].get(parameter)
+            short_description = parameter_descriptions[1].get(parameter, description)
+            inst_class = parameter_descriptions[2].get(parameter)
+            data.append(
+                [
+                    inst_class,
+                    parameter,
+                    parameter_version,
+                    value,
+                    description,
+                    short_description,
+                ]
+            )
+
+        return data
+
+    def _compare_parameter_across_versions(self, parameter_name):
+        """
+        Compare a parameter's value across different model versions.
+
+        Parameters
+        ----------
+        parameter_name : str
+            The name of the parameter to compare.
+
+        Returns
+        -------
+        list
+            A list of dictionaries containing model version, parameter value, and description.
+        """
+        all_versions = self.telescope_model.db.get_model_versions()
+        all_versions.reverse()
+        comparison_data = []
+
+        for model_version in all_versions:
+            telescope_model = TelescopeModel(
+                site=self.telescope_model.site,
+                telescope_name=self.telescope_model.name,
+                model_version=model_version,
+                label="reports",
+                mongo_db_config=self.db_config,
+            )
+
+            if not telescope_model.has_parameter(parameter_name):
+                return comparison_data
+
+            parameter_data = self.get_array_element_parameter_data(telescope_model)
+            for param in parameter_data:
+                if param[1] == parameter_name:
+                    comparison_data.append(
+                        {
+                            "model_version": model_version,
+                            "parameter_version": param[2],
+                            "value": param[3],
+                            "description": param[4],
+                        }
+                    )
+                    break
+        return comparison_data
+
+    def produce_array_element_report(self):
+        """
+        Produce a markdown report of all model parameters per array element.
+
+        Output
+        ----------
+        One markdown report of a given array element listing parameter values,
+        versions, and descriptions.
+        """
+        output_filename = Path(self.output_path / (self.telescope_model.name + ".md"))
+        output_filename.parent.mkdir(parents=True, exist_ok=True)
+        data = self.get_array_element_parameter_data(self.telescope_model)
+        # Sort data by class to prepare for grouping
+        if not isinstance(data, str):
+            data.sort(key=lambda x: (x[0], x[1]), reverse=True)
+
+        with output_filename.open("w", encoding="utf-8") as file:
+            # Group by class and write sections
+            file.write(f"# {self.telescope_model.name}\n")
+
+            if self.telescope_model.name != self.telescope_model.design_model:
+                file.write(
+                    "The design model can be found here: "
+                    f"[{self.telescope_model.design_model}]"
+                    f"({self.telescope_model.design_model}.md).\n"
+                )
+                file.write("\n\n")
+
+            for class_name, group in groupby(data, key=lambda x: x[0]):
+                group = sorted(group, key=lambda x: x[1])
+                file.write(f"## {class_name}\n\n")
+
+                # Write table header and separator row
+                file.write(
+                    "| Parameter Name      |  Parameter Version     "
+                    "| Values      | Short Description           |\n"
+                    "|---------------------|------------------------"
+                    "|-------------|-----------------------------|\n"
+                )
+
+                # Write table rows
+                column_widths = [20, 20, 20, 70]
+                for (
+                    _,
+                    parameter_name,
+                    parameter_version,
+                    value,
+                    description,
+                    short_description,
+                ) in group:
+                    text = short_description if short_description else description
+                    wrapped_text = textwrap.fill(str(text), column_widths[3]).split("\n")
+                    wrapped_text = " ".join(wrapped_text)
+                    file.write(
+                        f"| {parameter_name:{column_widths[0]}} |"
+                        f" {parameter_version:{column_widths[1]}} |"
+                        f" {value:{column_widths[2]}} |"
+                        f" {wrapped_text} |\n"
+                    )
+                file.write("\n\n")
+
+    def produce_model_parameter_reports(self):
+        """
+        Produce a markdown report per parameter for a given array element.
+
+        Output
+        ----------
+        One markdown report per model parameter of a given array element comparing
+        values across model versions.
+        """
+        logger.info(
+            f"Comparing parameters across model versions for Telescope: {self.telescope_model.name}"
+            f" and Site: {self.telescope_model.site}."
+        )
+        io_handler_instance = io_handler.IOHandler()
+        output_path = io_handler_instance.get_output_directory(
+            label="reports", sub_dir=f"parameters/{self.telescope_model.name}"
+        )
+
+        all_params = self.telescope_model.db.get_model_parameters(
+            site=self.telescope_model.site,
+            array_element_name=self.telescope_model.name,
+            collection="telescopes",
+        )
+
+        for parameter in all_params:
+            comparison_data = []
+            if all_params[parameter]["instrument"] == self.telescope_model.name:
+                comparison_data = self._compare_parameter_across_versions(parameter)
+            if comparison_data:
+                output_filename = output_path / f"{parameter}.md"
+                with output_filename.open("w", encoding="utf-8") as file:
+                    # Write header
+                    file.write(
+                        f"# {parameter}\n\n"
+                        f"**Telescope**: {self.telescope_model.name}\n\n"
+                        f"**Description**: {comparison_data[0]['description']}\n\n"
+                        "\n"
+                    )
+
+                    # Write table header
+                    file.write(
+                        "| Model Version      | Parameter Version      "
+                        "| Value                |\n"
+                        "|--------------------|------------------------"
+                        "|----------------------|\n"
+                    )
+
+                    # Write table rows
+                    for item in comparison_data:
+                        file.write(
+                            f"| {item['model_version']} |"
+                            f" {item['parameter_version']} |"
+                            f"{item['value'].replace('](', '](../')} |\n"
+                        )
+
+                    file.write("\n")
+                    if isinstance(comparison_data[0]["value"], str) and comparison_data[0][
+                        "value"
+                    ].endswith(".md)"):
+                        file.write(
+                            f"![Parameter plot.](_images/"
+                            f"{self.telescope_model.name}_{parameter}.png)"
+                        )

simtools/runners/corsika_simtel_runner.py

@@ -129,9 +129,7 @@ class CorsikaSimtelRunner:
         )
         with open(multipipe_script, "w", encoding="utf-8") as file:
             multipipe_command = Path(self._simtel_path).joinpath(
-                "sim_telarray/bin/multipipe_corsika "
-                f"-c {multipipe_file}"
-                " || echo 'Fan-out failed'"
+                f"sim_telarray/bin/multipipe_corsika -c {multipipe_file} || echo 'Fan-out failed'"
             )
             file.write(f"{multipipe_command}")
 

simtools/schemas/{integration_tests_config.metaschema.yml → application_workflow.metaschema.yml}

@@ -1,50 +1,74 @@
 ---
 $schema: http://json-schema.org/draft-06/schema#
-$ref: '#/definitions/SimtoolsIntegrationTestConfiguration'
-title: SimPipe integration test configuration metaschema
-description: YAML representation of integration test configuration metaschema
-version: 0.2.0
-name: integration_tests_config.metaschema
+$ref: '#/definitions/SimtoolsApplicationWorkflowConfiguration'
+title: SimPipe application workflow configuration metaschema
+description: YAML representation of application workflow configuration metaschema
+version: 0.3.0
+name: application_workflow.metaschema
 type: object
 additionalProperties: false
 
-
 definitions:
-  SimtoolsIntegrationTestConfiguration:
+  SimtoolsApplicationWorkflowConfiguration:
     type: object
     additionalProperties: false
     properties:
       CTA_SIMPIPE:
         "$ref": "#/definitions/CTASIMPIPE"
+      SCHEMA_VERSION:
+        type: string
+        description: "Version of the schema."
+      SCHEMA_URL:
+        type: string
+        format: uri
+        description: "URL of the schema."
+      SCHEMA_NAME:
+        type: string
+        description: "Name of the schema."
     required:
       - CTA_SIMPIPE
+      - SCHEMA_VERSION
+      - SCHEMA_NAME
     title: ""
   CTASIMPIPE:
     type: object
     additionalProperties: false
     properties:
-      APPLICATION:
+      LOG_PATH:
         type: string
-        description: "Name of simtools application"
-      TEST_NAME:
-        type: string
-        description: "Test name. Should be unique for this application."
-      MODEL_VERSION_USE_CURRENT:
-        type: boolean
-        description: |
-          "Skip test if model version is changed via command line.
-          Important e.g., for cases where DB model is incomplete for some
-          model versions."
-      CONFIGURATION:
-        "$ref": "#/definitions/Configuration"
-      INTEGRATION_TESTS:
+        description: "Path to the log file."
+      APPLICATIONS:
         type: array
         items:
-          "$ref": "#/definitions/IntegrationTest"
-    required:
-      - APPLICATION
-      - CONFIGURATION
-      - TEST_NAME
+          type: object
+          properties:
+            APPLICATION:
+              type: string
+              description: "Name of simtools application"
+            TEST_NAME:
+              type: string
+              description: "Test name. Should be unique for this application."
+            MODEL_VERSION_USE_CURRENT:
+              type: boolean
+              description: |
+                "Skip test if model version is changed via command line.
+                Important e.g., for cases where DB model is incomplete for some
+                model versions."
+            CONFIGURATION:
+              "$ref": "#/definitions/Configuration"
+            INTEGRATION_TESTS:
+              type: array
+              items:
+                "$ref": "#/definitions/IntegrationTest"
+            TEST_REQUIREMENT:
+              type: string
+              description: "Requirement ID of the test."
+            TEST_USE_CASE:
+              type: string
+              description: "Use case ID of the test."
+          required:
+            - APPLICATION
+            - CONFIGURATION
     title: CTASimpipe
   Configuration:
     description: "Command line configuration of simtools application."
@@ -110,4 +134,4 @@ definitions:
             CUT_CONDITION:
               description: "Condition to apply to the cut column."
               type: string
-    title: IntegrationTest
+    title: ApplicationWorkflow

simtools/schemas/array_elements.yml

@@ -15,6 +15,7 @@ data:
     collection: "telescopes"
     observatory: "CTAO"
     site: "North"
+    design_types: ["FlashCam", "NectarCam", "test"]
   LSTS:
     collection: "telescopes"
     observatory: "CTAO"
@@ -23,6 +24,7 @@ data:
     collection: "telescopes"
     observatory: "CTAO"
     site: "South"
+    design_types: ["FlashCam", "NectarCam", "test"]
   SSTS:
     collection: "telescopes"
     observatory: "CTAO"
@@ -33,6 +35,12 @@ data:
     observatory: "CTAO"
     site: "South"
     description: "Mid-sized Schwarzschild-Couder telescope"
+  MSTx:
+    collection: "telescopes"
+    observatory: "CTAO"
+    site: ["North", "South"]
+    design_types: ["FlashCam", "NectarCam", "test"]
+    description: "Mid-sized telescope"
   ILLN:
     collection: "calibration_devices"
     observatory: "CTAO"

simtools/schemas/model_parameter.metaschema.yml

@@ -2,6 +2,102 @@
 $schema: http://json-schema.org/draft-06/schema#
 $ref: '#/definitions/SimtoolsModelParameters'
 title: SimPipe DB Model Parameter Metaschema
+description: |
+  YAML representation of db model parameter metaschema
+  (based on simulation model DB).
+version: 0.3.0
+name: modelparameter.metaschema
+type: object
+additionalProperties: false
+
+definitions:
+  SimtoolsModelParameters:
+    description: ""
+    type: object
+    properties:
+      _id:
+        type: string
+        description: "DB unique identifier."
+      entry_date:
+        type: string
+        description: "Value entry date."
+      file:
+        type: boolean
+        description: "This parameter is a file."
+      instrument:
+        type:
+          - string
+          - "null"
+        description: "Associated instrument."
+      site:
+        anyOf:
+          - type: string
+            enum:
+              - North
+              - South
+          - type: "null"
+          - type: array
+            items:
+              type: string
+              enum:
+                - North
+                - South
+        description: "Associated CTAO site."
+      type:
+        type: string
+        description: "Data type"
+        enum:
+          - boolean
+          - dict
+          - double
+          - file
+          - float64
+          - int
+          - int64
+          - string
+          - uint
+          - uint32
+          - uint64
+      unit:
+        anyOf:
+          - type: string
+          - type: "null"
+        description: "Unit of the parameter."
+      value:
+        anyOf:
+          - type: boolean
+          - type: number
+          - type: string
+          - type: "null"
+          - type: array
+        description: "Value of the parameter."
+      parameter_version:
+        anyOf:
+          - type: string
+        description: "Parameter version."
+      schema_version:
+        anyOf:
+          - type: string
+        description: "Metaschema version."
+      unique_id:
+        anyOf:
+          - type: string
+          - type: "null"
+        description: "Unique ID of parameter definition."
+    required:
+      - file
+      - instrument
+      - parameter_version
+      - schema_version
+      - site
+      - type
+      - unit
+      - value
+...
+---
+$schema: http://json-schema.org/draft-06/schema#
+$ref: '#/definitions/SimtoolsModelParameters'
+title: SimPipe DB Model Parameter Metaschema
 description: |
   YAML representation of db model parameter metaschema
   (based on simulation model DB).

simtools/schemas/model_parameter_and_data_schema.metaschema.yml

@@ -247,6 +247,8 @@ definitions:
           - Site
           - Structure
           - Telescope
+          - configuration_sim_telarray
+          - configuration_corsika
       type:
         type: array
         items:
@@ -261,7 +263,6 @@ definitions:
             - South
     required:
       - class
-      - type
     title: Instrument
   InstrumentTypeElement:
     type: string

simtools/schemas/model_parameters/correct_nsb_spectrum_to_telescope_altitude.schema.yml

@@ -16,7 +16,7 @@ data:
     unit: dimensionless
     default: "atm_trans_2200_1_3_0_0_0.dat"
 instrument:
-  class: Telescope
+  class: configuration_sim_telarray
   type:
     - LSTN
     - LSTS

simtools/schemas/model_parameters/corsika_cherenkov_photon_bunch_size.schema.yml

@@ -10,6 +10,8 @@ developer_note: |-
 name: corsika_cherenkov_photon_bunch_size
 description: |-
   Cherenkov bunch size definition.
+instrument:
+  class: configuration_corsika
 data:
   - type: float64
     unit: dimensionless

simtools/schemas/model_parameters/corsika_cherenkov_photon_wavelength_range.schema.yml

@@ -8,6 +8,8 @@ meta_schema_version: 0.1.0
 name: corsika_cherenkov_photon_wavelength_range
 description: |-
   Cherenkov photon wavelength band.
+instrument:
+  class: configuration_corsika
 data:
   - name: wavlgl
     type: float64

simtools/schemas/model_parameters/corsika_first_interaction_height.schema.yml

@@ -10,6 +10,8 @@ description: |-
   Starting altitude of primary particles in altitude above observation level.
   This parameter should be set to values other than '0.' for specialized studies
   only. See CORSIKA manual for details.
+instrument:
+  class: configuration_corsika
 data:
   - type: float64
     unit: cm

simtools/schemas/model_parameters/corsika_iact_io_buffer.schema.yml

@@ -8,6 +8,8 @@ meta_schema_version: 0.1.0
 name: corsika_iact_io_buffer
 description: |-
   Maximum size of data blocks written by the CORSIKA IACT module.
+instrument:
+  class: configuration_corsika
 data:
   - type: int
     unit: MB

simtools/schemas/model_parameters/corsika_iact_max_bunches.schema.yml

@@ -12,6 +12,8 @@ description: |-
   Maximum number of bunches that can be kept per telescope, including in temporary files.
   Telescopes for which this limit is exceeded will have the number of bunches reduced in
   increasing powers of two (by discarding every second bunch and adding its bunch size to the bunch kept).
+instrument:
+  class: configuration_corsika
 data:
   - type: int
     unit: dimensionless

simtools/schemas/model_parameters/corsika_iact_split_auto.schema.yml

@@ -13,6 +13,8 @@ description: |-
   If the full data block for the entire array would exceed the given number of bunches,
   it gets split up into separate blocks per individual telescope, in order to avoid
   exceeding the hard I/O buffer size limit.
+instrument:
+  class: configuration_corsika
 data:
   - type: int
     unit: dimensionless

simtools/schemas/model_parameters/corsika_longitudinal_shower_development.schema.yml

@@ -9,6 +9,8 @@ name: corsika_longitudinal_shower_development
 description: |-
   Step size for recording of longitudinal shower development.
   Any value >=1. for the step size enables the recording of the shower development.
+instrument:
+  class: configuration_corsika
 data:
   - type: float64
     unit: g/cm2

simtools/schemas/model_parameters/corsika_particle_kinetic_energy_cutoff.schema.yml

@@ -8,6 +8,8 @@ meta_schema_version: 0.1.0
 name: corsika_particle_kinetic_energy_cutoff
 description: |-
   Kinetic energy cutoffs for different particle types.
+instrument:
+  class: configuration_corsika
 data:
   - name: hadrons
     type: float64

simtools/schemas/model_parameters/corsika_starting_grammage.schema.yml

@@ -10,6 +10,8 @@ description: |-
   Starting altitude of the primary particle in mass overburden for all showers.
   This parameter should be set to values other than '0.' for specialized studies
   only. See CORSIKA manual for details.
+instrument:
+  class: configuration_corsika
 data:
   - type: float64
     unit: g/cm2