gammasimtools 0.15.0__py3-none-any.whl → 0.17.0__py3-none-any.whl

simtools/reporting/docs_read_parameters.py

@@ -3,6 +3,7 @@
 r"""Class to read and manage relevant model parameters for a given telescope model."""
 
 import logging
+import re
 import textwrap
 from collections import defaultdict
 from itertools import groupby
@@ -11,8 +12,11 @@ from pathlib import Path
 import numpy as np
 
 from simtools.db import db_handler
+from simtools.io_operations import io_handler
 from simtools.model.telescope_model import TelescopeModel
+from simtools.utils import general as gen
 from simtools.utils import names
+from simtools.visualization import plot_pixels
 
 logger = logging.getLogger()
 
@@ -25,35 +29,98 @@ class ReadParameters:
         self._logger = logging.getLogger(__name__)
         self.db = db_handler.DatabaseHandler(mongo_db_config=db_config)
         self.db_config = db_config
-        self.array_element = args.get("telescope")
-        self.site = args.get("site")
+        self.array_element = args.get("telescope", None)
+        self.site = args.get("site", None)
         self.model_version = args.get("model_version", None)
         self.output_path = output_path
         self.observatory = args.get("observatory")
+        self.software = args.get("simulation_software", None)
 
-    def _convert_to_md(self, input_file):
+    @property
+    def model_version(self):
+        """Model version."""
+        return self._model_version
+
+    @model_version.setter
+    def model_version(self, model_version):
+        """
+        Set model version.
+
+        Parameters
+        ----------
+        _model_version: str or list
+            Model version (e.g., "6.0.0").
+            If a list is passed, it must contain exactly one element,
+            and only that element will be used.
+
+        Raises
+        ------
+        ValueError
+            If more than one model version is passed.
+        """
+        if isinstance(model_version, list):
+            raise ValueError(
+                f"Only one model version can be passed to {self.__class__.__name__}, not a list."
+            )
+        self._model_version = model_version
+
+    def _convert_to_md(self, parameter, parameter_version, input_file):
         """Convert a file to a Markdown file, preserving formatting."""
         input_file = Path(input_file)
+
         output_data_path = Path(self.output_path / "_data_files")
         output_data_path.mkdir(parents=True, exist_ok=True)
         output_file_name = Path(input_file.stem + ".md")
         output_file = output_data_path / output_file_name
+        image_name = f"{self.array_element}_{parameter}_{self.model_version.replace('.', '-')}"
+        outpath = Path(io_handler.IOHandler().get_output_directory().parent / "_images")
+        outpath.mkdir(parents=True, exist_ok=True)
+        image_path = Path(f"{outpath}/{image_name}")
+
+        if parameter == "camera_config_file" and parameter_version:
+            image_path = Path(f"{outpath}/{input_file.stem.replace('.', '-')}")
+            if not (image_path.with_suffix(".png")).exists():
+                logger.info("Plotting camera configuration file: %s", input_file.name)
+                plot_config = {
+                    "file_name": input_file.name,
+                    "telescope": self.array_element,
+                    "parameter_version": parameter_version,
+                    "site": self.site,
+                    "model_version": self.model_version,
+                    "parameter": parameter,
+                }
+
+                plot_pixels.plot(
+                    config=plot_config,
+                    output_file=image_path,
+                    db_config=self.db_config,
+                )
+            else:
+                logger.info(
+                    "Camera configuration file plot already exists: %s",
+                    image_path.with_suffix(".png"),
+                )
 
         try:
-            # First try with utf-8
-            try:
-                with input_file.open("r", encoding="utf-8") as infile:
-                    file_contents = infile.read()
-            except UnicodeDecodeError:
-                # If utf-8 fails, try with latin-1 (which can read any byte sequence)
-                with input_file.open("r", encoding="latin-1") as infile:
-                    file_contents = infile.read()
-
-            with output_file.open("w", encoding="utf-8") as outfile:
-                outfile.write(f"# {input_file.stem}\n")
-                outfile.write("```\n")
-                outfile.write(file_contents)
-                outfile.write("\n```")
+            # with input_file.open("r", encoding="utf-8") as infile:
+            file_contents = gen.read_file_encoded_in_utf_or_latin(input_file)
+
+            if self.model_version is not None:
+                with output_file.open("w", encoding="utf-8") as outfile:
+                    outfile.write(f"# {input_file.stem}\n")
+                    outfile.write(f"![Parameter plot.]({image_path}.png)\n\n")
+                    outfile.write(
+                        "\n\nThe full file can be found in the Simulation Model repository [here]"
+                        "(https://gitlab.cta-observatory.org/cta-science/simulations/"
+                        "simulation-model/simulation-models/-/blob/main/simulation-models/"
+                        f"model_parameters/Files/{input_file.name}).\n\n"
+                    )
+                    outfile.write("\n\n")
+                    outfile.write("The first 30 lines of the file are:\n")
+                    outfile.write("```\n")
+                    first_30_lines = "".join(file_contents[:30])
+                    outfile.write(first_30_lines)
+                    outfile.write("\n```")
 
         except FileNotFoundError as exc:
             logger.exception(f"Data file not found: {input_file}.")
@@ -61,11 +128,19 @@ class ReadParameters:
 
         return f"_data_files/{output_file_name}"
 
-    def _format_parameter_value(self, value_data, unit, file_flag):
+    def _format_parameter_value(
+        self, parameter, value_data, unit, file_flag, parameter_version=None
+    ):
         """Format parameter value based on type."""
         if file_flag:
             input_file_name = f"{self.output_path}/model/{value_data}"
-            output_file_name = self._convert_to_md(input_file_name)
+            if parameter_version is None:
+                return (
+                    f"[{Path(value_data).name}](https://gitlab.cta-observatory.org/"
+                    "cta-science/simulations/simulation-model/simulation-models/-/blob/main/"
+                    f"simulation-models/model_parameters/Files/{value_data})"
+                ).strip()
+            output_file_name = self._convert_to_md(parameter, parameter_version, input_file_name)
             return f"[{Path(value_data).name}]({output_file_name})".strip()
         if isinstance(value_data, (str | int | float)):
             return f"{value_data} {unit}".strip()
@@ -77,6 +152,26 @@ class ReadParameters:
             else ", ".join(f"{v} {unit}" for v in value_data)
         ).strip()
 
+    def _wrap_at_underscores(self, text, max_width):
+        """Wrap text at underscores to fit within a specified width."""
+        parts = text.split("_")
+        lines = []
+        current = []
+
+        for part in parts:
+            # Predict the new length if we add this part
+            next_line = "_".join([*current, part])
+            if len(next_line) <= max_width:
+                current.append(part)
+            else:
+                lines.append("_".join(current))
+                current = [part]
+
+        if current:
+            lines.append("_".join(current))
+
+        return " ".join(lines)
+
     def _group_model_versions_by_parameter_version(self, grouped_data):
         """Group model versions by parameter version and track the parameter values."""
         result = {}
@@ -152,8 +247,10 @@ class ReadParameters:
                     continue
 
                 file_flag = parameter_data.get("file", False)
-                value = self._format_parameter_value(value_data, unit, file_flag)
                 parameter_version = parameter_data.get("parameter_version")
+                value = self._format_parameter_value(
+                    parameter_name, value_data, unit, file_flag, parameter_version=None
+                )
                 model_version = version
 
                 # Group the data by parameter version and store model versions as a list
@@ -168,26 +265,29 @@ class ReadParameters:
 
         return self._group_model_versions_by_parameter_version(grouped_data)
 
-    def get_all_parameter_descriptions(self):
-        """
-        Get descriptions for all model parameters.
+    def get_all_parameter_descriptions(self, collection="telescopes"):
+        """Get descriptions for all model parameters.
 
         Returns
         -------
-            tuple: A tuple containing two dictionaries:
-                - parameter_description: Maps parameter names to their descriptions.
-                - short_description: Maps parameter names to their short descriptions.
-                - inst_class: Maps parameter names to their respective class.
+        dict
+            Nested dictionaries with first key as the parameter name and
+            the following dictionary as the value:
+            - key: description, value: description of the parameter.
+            - key: short_description, value: short description of the parameter.
+            - key: inst_class, value: class, for eg. Structure, Camera, etc.
         """
-        parameter_description, short_description, inst_class = {}, {}, {}
+        parameter_dict = {}
 
-        for instrument_class in names.db_collection_to_instrument_class_key("telescopes"):
+        for instrument_class in names.db_collection_to_instrument_class_key(collection):
             for parameter, details in names.model_parameters(instrument_class).items():
-                parameter_description[parameter] = details.get("description")
-                short_description[parameter] = details.get("short_description")
-                inst_class[parameter] = instrument_class
+                parameter_dict[parameter] = {
+                    "description": details.get("description"),
+                    "short_description": details.get("short_description"),
+                    "inst_class": instrument_class,
+                }
 
-        return parameter_description, short_description, inst_class
+        return parameter_dict
 
     def get_array_element_parameter_data(self, telescope_model, collection="telescopes"):
         """
@@ -219,8 +319,6 @@ class ReadParameters:
 
         for parameter in filter(all_parameter_data.__contains__, names.model_parameters().keys()):
             parameter_data = all_parameter_data.get(parameter)
-            if parameter_data["instrument"] != telescope_model.name:
-                continue
             parameter_version = telescope_model.get_parameter_version(parameter)
             unit = parameter_data.get("unit") or " "
             value_data = parameter_data.get("value")
@@ -229,11 +327,25 @@ class ReadParameters:
                 continue
 
             file_flag = parameter_data.get("file", False)
-            value = self._format_parameter_value(value_data, unit, file_flag)
+            value = self._format_parameter_value(
+                parameter, value_data, unit, file_flag, parameter_version
+            )
+
+            description = parameter_descriptions.get(parameter).get("description")
+            short_description = (
+                parameter_descriptions.get(parameter).get("short_description") or description
+            )
+            inst_class = parameter_descriptions.get(parameter).get("inst_class")
+
+            matching_instrument = parameter_data["instrument"] == telescope_model.name
+            if not names.is_design_type(telescope_model.name) and matching_instrument:
+                parameter = f"***{parameter}***"
+                parameter_version = f"***{parameter_version}***"
+                if not re.match(r"^\[.*\]\(.*\)$", value.strip()):
+                    value = f"***{value}***"
+                description = f"***{description}***"
+                short_description = f"***{short_description}***"
 
-            description = parameter_descriptions[0].get(parameter)
-            short_description = parameter_descriptions[1].get(parameter, description)
-            inst_class = parameter_descriptions[2].get(parameter)
             data.append(
                 [
                     inst_class,
@@ -247,6 +359,120 @@ class ReadParameters:
 
         return data
 
+    def _write_to_file(self, data, file):
+        # Write table header and separator row
+        file.write(
+            "| Parameter Name      |  Parameter Version     "
+            "| Values      | Short Description           |\n"
+            "|---------------------|------------------------"
+            "|-------------|-----------------------------|\n"
+        )
+
+        # Write table rows
+        column_widths = [10, 10, 20, 60]
+        for (
+            _,
+            parameter_name,
+            parameter_version,
+            value,
+            description,
+            short_description,
+        ) in data:
+            text = short_description if short_description else description
+            wrapped_text = textwrap.fill(str(text), column_widths[3]).split("\n")
+            wrapped_text = " ".join(wrapped_text)
+            parameter_name = self._wrap_at_underscores(parameter_name, column_widths[0])
+
+            file.write(
+                f"| {parameter_name:{column_widths[0]}} |"
+                f" {parameter_version:{column_widths[1]}} |"
+                f" {value:{column_widths[2]}} |"
+                f" {wrapped_text:{column_widths[3]}} |\n"
+            )
+        file.write("\n\n")
+
+    def get_simulation_configuration_data(self):
+        """Get data and descriptions for simulation configuration parameters."""
+
+        def get_param_data(telescope, site):
+            """Retrieve and format parameter data for one telescope-site combo."""
+            param_dict = self.db.get_simulation_configuration_parameters(
+                simulation_software=self.software,
+                site=site,
+                array_element_name=telescope,
+                model_version=self.model_version,
+            )
+
+            parameter_descriptions = self.get_all_parameter_descriptions(
+                collection=f"configuration_{self.software}"
+            )
+
+            model_output_path = Path(f"{self.output_path}/model")
+            model_output_path.mkdir(parents=True, exist_ok=True)
+            self.db.export_model_files(parameters=param_dict, dest=str(model_output_path))
+
+            data = []
+            for parameter, parameter_data in param_dict.items():
+                description = parameter_descriptions.get(parameter).get("description")
+                short_description = parameter_descriptions.get(parameter).get(
+                    "short_description", description
+                )
+                value_data = parameter_data.get("value")
+
+                if value_data is None:
+                    continue
+
+                unit = parameter_data.get("unit") or " "
+                file_flag = parameter_data.get("file", False)
+                parameter_version = parameter_data.get("parameter_version")
+                value = self._format_parameter_value(
+                    parameter, value_data, unit, file_flag, parameter_version
+                )
+
+                data.append(
+                    [
+                        telescope,
+                        parameter,
+                        parameter_version,
+                        value,
+                        description,
+                        short_description,
+                    ]
+                )
+            return data
+
+        if self.software == "corsika":
+            return get_param_data(self.array_element, self.site)
+
+        results = []
+        telescopes = self.db.get_array_elements(self.model_version)
+        for telescope in telescopes:
+            valid_site = names.get_site_from_array_element_name(telescope)
+            if not isinstance(valid_site, list):
+                results.extend(get_param_data(telescope, valid_site))
+            else:
+                for site in valid_site:
+                    results.extend(get_param_data(telescope, site))
+        return results
+
+    def produce_simulation_configuration_report(self):
+        """Write simulation configuration report."""
+        output_filename = Path(self.output_path / (f"configuration_{self.software}.md"))
+        output_filename.parent.mkdir(parents=True, exist_ok=True)
+        data = self.get_simulation_configuration_data()
+
+        with output_filename.open("w", encoding="utf-8") as file:
+            file.write(f"# configuration_{self.software}\n")
+            file.write("\n\n")
+            if self.software == "sim_telarray":
+                data.sort(key=lambda x: (x[0], x[1]))
+                for telescope, group in groupby(data, key=lambda x: x[0]):
+                    file.write(f"## [{telescope}]({telescope}.md)\n")
+                    file.write("\n\n")
+                    self._write_to_file(group, file)
+            else:
+                self._write_to_file(data, file)
+
     def produce_array_element_report(self):
         """
         Produce a markdown report of all model parameters per array element.
@@ -281,44 +507,20 @@ class ReadParameters:
                 file.write(
                     "The design model can be found here: "
                     f"[{telescope_model.design_model}]"
-                    f"({telescope_model.design_model}.md).\n"
+                    f"({telescope_model.design_model}.md).\n\n"
+                )
+                file.write(
+                    "Parameters shown in ***bold and italics*** are specific to each telescope.\n"
+                    "Parameters without emphasis are inherited from the design model.\n"
                 )
                 file.write("\n\n")
 
             for class_name, group in groupby(data, key=lambda x: x[0]):
                 group = sorted(group, key=lambda x: x[1])
                 file.write(f"## {class_name}\n\n")
+                self._write_to_file(group, file)
 
-                # Write table header and separator row
-                file.write(
-                    "| Parameter Name      |  Parameter Version     "
-                    "| Values      | Short Description           |\n"
-                    "|---------------------|------------------------"
-                    "|-------------|-----------------------------|\n"
-                )
-
-                # Write table rows
-                column_widths = [20, 20, 20, 70]
-                for (
-                    _,
-                    parameter_name,
-                    parameter_version,
-                    value,
-                    description,
-                    short_description,
-                ) in group:
-                    text = short_description if short_description else description
-                    wrapped_text = textwrap.fill(str(text), column_widths[3]).split("\n")
-                    wrapped_text = " ".join(wrapped_text)
-                    file.write(
-                        f"| {parameter_name:{column_widths[0]}} |"
-                        f" {parameter_version:{column_widths[1]}} |"
-                        f" {value:{column_widths[2]}} |"
-                        f" {wrapped_text} |\n"
-                    )
-                file.write("\n\n")
-
-    def produce_model_parameter_reports(self):
+    def produce_model_parameter_reports(self, collection="telescopes"):
         """
         Produce a markdown report per parameter for a given array element.
 
@@ -334,7 +536,7 @@ class ReadParameters:
 
         all_parameter_names = names.model_parameters(None).keys()
         all_parameter_data = self.db.get_model_parameters_for_all_model_versions(
-            site=self.site, array_element_name=self.array_element, collection="telescopes"
+            site=self.site, array_element_name=self.array_element, collection=collection
         )
 
         comparison_data = self._compare_parameter_across_versions(
@@ -347,7 +549,12 @@ class ReadParameters:
                 continue
 
             output_filename = output_path / f"{parameter}.md"
-            description = self.get_all_parameter_descriptions()[0].get(parameter)
+
+            parameter_descriptions = self.get_all_parameter_descriptions(collection=collection).get(
+                parameter
+            ) or self.get_all_parameter_descriptions(collection="telescopes").get(parameter)
+
+            description = parameter_descriptions.get("description")
             with output_filename.open("w", encoding="utf-8") as file:
                 # Write header
                 file.write(
@@ -370,16 +577,16 @@ class ReadParameters:
                     file.write(
                         f"| {item['parameter_version']} |"
                         f" {item['model_version']} |"
-                        f"{item['value'].replace('](', '](../')} |\n"
+                        f"{item['value']} |\n"
                     )
 
                 file.write("\n")
                 if comparison_data.get(parameter)[0]["file_flag"]:
-                    file.write(f"![Parameter plot.](_images/{self.array_element}_{parameter}.png)")
+                    file.write(f"![Parameter plot.](/_images/{self.array_element}_{parameter}.png)")
 
     def _write_array_layouts_section(self, file, layouts):
         """Write the array layouts section of the report."""
-        file.write("\n## Array Layouts {#array-layouts-details}\n\n")
+        file.write("\n## Array Layouts\n\n")
         for layout in layouts:
             layout_name = layout["name"]
             elements = layout["elements"]
@@ -388,10 +595,15 @@ class ReadParameters:
             for element in sorted(elements):
                 file.write(f"| [{element}]({element}.md) |\n")
             file.write("\n")
+            version = self.model_version.replace(".", "-")
+            filename = f"OBS-{self.site}_{layout_name}_{version}.png"
+            image_path = f"/_images/{filename}"
+            file.write(f"![{layout_name} Layout]({image_path})\n\n")
+            file.write("\n")
 
     def _write_array_triggers_section(self, file, trigger_configs):
         """Write the array triggers section of the report."""
-        file.write("\n## Array Trigger Configurations {#array-triggers-details}\n\n")
+        file.write("\n## Array Trigger Configurations\n\n")
         file.write(
             "| Trigger Name | Multiplicity | Width | Hard Stereo | Min Separation |\n"
             "|--------------|--------------|--------|-------------|----------------|\n"
@@ -411,24 +623,34 @@ class ReadParameters:
 
     def _write_parameters_table(self, file, all_parameter_data):
         """Write the main parameters table of the report."""
-        file.write("| Parameter | Value |\n|-----------|--------|\n")
+        file.write(
+            "| Parameter | Value | Parameter Version |\n"
+            "|-----------|--------|-------------------|\n"
+        )
         for param_name, param_data in sorted(all_parameter_data.items()):
             value = param_data.get("value")
             unit = param_data.get("unit") or " "
             file_flag = param_data.get("file", False)
+            parameter_version = param_data.get("parameter_version")
 
             if value is None:
                 continue
 
             if param_name == "array_layouts":
-                file.write("| array_layouts | [View Array Layouts](#array-layouts-details) |\n")
+                file.write(
+                    "| array_layouts | [View Array Layouts](#array-layouts)"
+                    f" | {parameter_version} |\n"
+                )
             elif param_name == "array_triggers":
                 file.write(
-                    "| array_triggers | [View Trigger Configurations](#array-triggers-details) |\n"
+                    "| array_triggers | [View Trigger Configurations]"
+                    f"(#array-trigger-configurations) | {parameter_version} |\n"
                 )
             else:
-                formatted_value = self._format_parameter_value(value, unit, file_flag)
-                file.write(f"| {param_name} | {formatted_value} |\n")
+                formatted_value = self._format_parameter_value(
+                    param_name, value, unit, file_flag, parameter_version
+                )
+                file.write(f"| {param_name} | {formatted_value} | {parameter_version} |\n")
         file.write("\n")
 
     def produce_observatory_report(self):
@@ -463,3 +685,126 @@ class ReadParameters:
                 self._write_array_triggers_section(
                     file, all_parameter_data["array_triggers"]["value"]
                 )
+
+    def get_calibration_data(self, all_parameter_data, array_element):
+        """Get calibration data and descriptions for a given array element."""
+        calibration_descriptions = self.get_all_parameter_descriptions(
+            collection="calibration_devices"
+        )
+        # get descriptions of array element positions from the telescope collection
+        telescope_descriptions = self.get_all_parameter_descriptions(collection="telescopes")
+        data = []
+        class_grouped_data = {}
+
+        for parameter in all_parameter_data.keys():
+            parameter_descriptions = calibration_descriptions.get(
+                parameter
+            ) or telescope_descriptions.get(parameter)
+
+            parameter_data = all_parameter_data.get(parameter)
+            parameter_version = parameter_data.get("parameter_version")
+            unit = parameter_data.get("unit") or " "
+            value_data = parameter_data.get("value")
+
+            if value_data is None:
+                continue
+
+            file_flag = parameter_data.get("file", False)
+            value = self._format_parameter_value(
+                parameter, value_data, unit, file_flag, parameter_version
+            )
+
+            description = parameter_descriptions.get("description")
+            short_description = parameter_descriptions.get("short_description") or description
+
+            inst_class = parameter_descriptions.get("inst_class")
+
+            matching_instrument = parameter_data["instrument"] == array_element
+            if not names.is_design_type(array_element) and matching_instrument:
+                parameter = f"***{parameter}***"
+                parameter_version = f"***{parameter_version}***"
+                if not re.match(r"^\[.*\]\(.*\)$", value.strip()):
+                    value = f"***{value}***"
+                description = f"***{description}***"
+                short_description = f"***{short_description}***"
+
+            # Group by class name
+            if inst_class not in class_grouped_data:
+                class_grouped_data[inst_class] = []
+
+            class_grouped_data[inst_class].append(
+                [
+                    inst_class,
+                    parameter,
+                    parameter_version,
+                    value,
+                    description,
+                    short_description,
+                ]
+            )
+
+        data = []
+        for class_name in sorted(class_grouped_data.keys(), reverse=True):
+            sorted_class_data = sorted(class_grouped_data[class_name], key=lambda x: x[1])
+            data.extend(sorted_class_data)
+
+        return data
+
+    def produce_calibration_reports(self):
+        """Write calibration reports."""
+        calibration_array_elements = self.db.get_array_elements(
+            self.model_version, collection="calibration_devices"
+        )
+        array_elements = calibration_array_elements.copy()
+        for element in calibration_array_elements:
+            design_model = self.db.get_design_model(
+                self.model_version, element, "calibration_devices"
+            )
+            if design_model and design_model not in array_elements:
+                array_elements.append(design_model)
+
+        for calibration_device in array_elements:
+            all_parameter_data = self.db.get_model_parameters(
+                site=names.get_site_from_array_element_name(calibration_device),
+                array_element_name=calibration_device,
+                collection="calibration_devices",
+                model_version=self.model_version,
+            )
+
+            output_filename = Path(self.output_path / (f"{calibration_device}.md"))
+            output_filename.parent.mkdir(parents=True, exist_ok=True)
+            data = self.get_calibration_data(all_parameter_data, calibration_device)
+
+            design_model = self.db.get_design_model(
+                self.model_version, calibration_device, "calibration_devices"
+            )
+
+            with output_filename.open("w", encoding="utf-8") as file:
+                file.write(f"# {calibration_device}\n")
+                file.write("\n\n")
+
+                if not names.is_design_type(calibration_device):
+                    file.write(
+                        "The design model can be found here: "
+                        f"[{design_model}]"
+                        f"({design_model}.md).\n\n"
+                    )
+                    file.write(
+                        "Parameters shown in ***bold and italics*** are specific"
+                        " to each array element.\n"
+                        "Parameters without emphasis are inherited from the design model.\n"
+                    )
+                    file.write("\n\n")
+
+                for class_name, group in groupby(data, key=lambda x: x[0]):
+                    group = sorted(group, key=lambda x: x[1])
+                    file.write(f"## {class_name}\n\n")
+                    self._write_to_file(group, file)
+
+        new_output_path = Path(self.output_path).parent.parent / "parameters"
+        new_output_path.mkdir(parents=True, exist_ok=True)
+        self.output_path = new_output_path
+        for calibration_device in array_elements:
+            self.site = names.get_site_from_array_element_name(calibration_device)
+            self.array_element = calibration_device
+            self.produce_model_parameter_reports(collection="calibration_devices")