gammasimtools 0.13.0__py3-none-any.whl → 0.15.0__py3-none-any.whl

simtools/reporting/docs_auto_report_generator.py

@@ -0,0 +1,217 @@
+#!/usr/bin/python3
+
+r"""Class to automate generation of all reports according to command line inputs."""
+
+import logging
+from collections.abc import Generator
+from pathlib import Path
+
+from simtools.db import db_handler
+from simtools.reporting.docs_read_parameters import ReadParameters
+from simtools.utils import names
+
+logger = logging.getLogger()
+
+
+class ReportGenerator:
+    """Automate report generation."""
+
+    def __init__(self, db_config, args, output_path):
+        """Initialise class."""
+        self._logger = logging.getLogger(__name__)
+        self.db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+        self.db_config = db_config
+        self.args = args
+        self.output_path = output_path
+
+    def _add_design_models_to_telescopes(self, model_version, telescopes):
+        """Add design models to the list of telescopes for a given model version."""
+        updated_telescopes = telescopes.copy()  # Copy original list to avoid modifying it directly
+        for telescope in telescopes:
+            design_model = self.db.get_design_model(model_version, telescope)  # Get design model
+            if design_model and design_model not in updated_telescopes:
+                updated_telescopes.append(design_model)  # Add design model if not already present
+        return updated_telescopes
+
+    def _filter_telescopes_by_site(self, telescopes, selected_sites):
+        """Filter telescopes by selected sites."""
+        filtered_telescopes = []
+        for telescope in telescopes:
+            sites = names.get_site_from_array_element_name(telescope)
+            sites = sites if isinstance(sites, list) else [sites]
+            if any(site in selected_sites for site in sites):
+                filtered_telescopes.append(telescope)
+        return filtered_telescopes
+
+    def _generate_array_element_report_combinations(self):
+        """Generate combinations of model versions, telescopes, and sites for array element reports.
+
+        This generator function yields tuples of (model_version, telescope, site) for array element
+        report generation based on the input arguments and flags.
+
+        The function filters telescopes by site and ensures that design models are included
+        when necessary.
+
+        Yields
+        ------
+            tuple[str, str, str]: A tuple containing (model_version, telescope, site) for each
+                valid combination based on the input arguments.
+        """
+        all_sites = names.site_names()
+
+        model_versions = (
+            self.db.get_model_versions()
+            if self.args.get("all_model_versions")
+            else [self.args["model_version"]]
+        )
+        selected_sites = all_sites if self.args.get("all_sites") else {self.args["site"]}
+
+        def get_telescopes(model_version):
+            """Get list of telescopes depending on input arguments."""
+            if not self.args.get("all_telescopes"):
+                return [self.args["telescope"]]
+            telescopes = self.db.get_array_elements(model_version)
+            all_telescopes = self._add_design_models_to_telescopes(model_version, telescopes)
+            return self._filter_telescopes_by_site(all_telescopes, selected_sites)
+
+        def get_valid_sites(telescope):
+            sites = names.get_site_from_array_element_name(telescope)
+            sites = sites if isinstance(sites, list) else [sites]
+            return [site for site in sites if site in selected_sites]
+
+        def generate_combinations():
+            for version in model_versions:
+                telescopes = get_telescopes(version)
+                for telescope in telescopes:
+                    for site in get_valid_sites(telescope):
+                        yield version, telescope, site
+
+        return generate_combinations()
+
+    def _generate_single_array_element_report(self, model_version, telescope, site):
+        """Generate a single report with given parameters."""
+        self.args.update(
+            {
+                "telescope": telescope,
+                "site": site,
+                "model_version": model_version,
+            }
+        )
+
+        output_path = Path(self.output_path) / str(model_version)
+        ReadParameters(self.db_config, self.args, output_path).produce_array_element_report()
+
+        logger.info(
+            f"Markdown report generated for {site} "
+            f"Telescope {telescope} (v{model_version}): {output_path}"
+        )
+
+    def auto_generate_array_element_reports(self):
+        """Generate all reports based on which --all_* flag is passed."""
+        if self.args.get("observatory"):
+            self.auto_generate_observatory_reports()
+            return
+        for params in self._generate_array_element_report_combinations():
+            self._generate_single_array_element_report(*params)
+
+    def _get_telescopes_from_layout(self, site: str) -> set[str]:
+        """Get unique telescopes for a given site across all versions."""
+        if not self.args.get("all_telescopes"):
+            return {self.args["telescope"]}
+
+        # Get layouts for all versions for this site
+        layouts = self.db.get_model_parameters_for_all_model_versions(
+            site=site, array_element_name=f"OBS-{site}", collection="sites"
+        )
+
+        # Collect all telescopes from hyper array layouts
+        all_telescopes = set()
+        for key, version_data in layouts.items():
+            layout = version_data.get("array_layouts", {}).get("value", [])
+            hyper_array = next(
+                (layout["elements"] for layout in layout if layout["name"] == "hyper_array"), []
+            )
+            all_telescopes.update(hyper_array)
+            all_telescopes.update(self._add_design_models_to_telescopes(key, list(all_telescopes)))
+
+        return all_telescopes
+
+    def _get_valid_sites_for_telescope(self, telescope: str) -> list[str]:
+        """Get valid sites for a given telescope."""
+        sites = names.get_site_from_array_element_name(telescope)
+        return [sites] if not isinstance(sites, list) else sites
+
+    def _generate_parameter_report_combinations(self) -> Generator[tuple[str, str], None, None]:
+        """Generate combinations of telescopes and sites for report generation."""
+        if not self.args.get("all_telescopes"):
+            # For a specific telescope, get its valid sites regardless of --all_sites
+            telescope = self.args["telescope"]
+            for site in self._get_valid_sites_for_telescope(telescope):
+                yield telescope, site
+        else:
+            # For --all_telescopes, use selected sites to get telescopes
+            all_sites = names.site_names()
+            selected_sites = all_sites if self.args.get("all_sites") else {self.args["site"]}
+
+            for site in selected_sites:
+                telescopes = self._get_telescopes_from_layout(site)
+                yield from ((telescope, site) for telescope in telescopes)
+
+    def auto_generate_parameter_reports(self):
+        """Generate parameter reports for all telescope-site combinations."""
+        for telescope, site in self._generate_parameter_report_combinations():
+            self.args.update(
+                {
+                    "telescope": telescope,
+                    "site": site,
+                }
+            )
+
+            ReadParameters(
+                self.db_config, self.args, self.output_path
+            ).produce_model_parameter_reports()
+
+            logger.info(
+                f"Markdown report generated for {site} Telescope {telescope}: {self.output_path}"
+            )
+
+    def _generate_observatory_report_combinations(self) -> Generator[tuple[str, str], None, None]:
+        """Generate combinations of sites and model versions for observatory reports.
+
+        Yields
+        ------
+            tuple[str, str]: A tuple containing (site, model_version) for each valid combination
+                based on the input arguments.
+        """
+        all_sites = names.site_names()
+        selected_sites = all_sites if self.args.get("all_sites") else {self.args["site"]}
+
+        model_versions = (
+            self.db.get_model_versions()
+            if self.args.get("all_model_versions")
+            else [self.args["model_version"]]
+        )
+
+        for site in selected_sites:
+            for version in model_versions:
+                yield site, version
+
+    def _generate_single_observatory_report(self, site: str, model_version: str):
+        """Generate a single observatory report with given parameters."""
+        self.args.update(
+            {
+                "site": site,
+                "model_version": model_version,
+                "observatory": True,
+            }
+        )
+
+        output_path = Path(self.output_path) / str(model_version)
+        ReadParameters(self.db_config, self.args, output_path).produce_observatory_report()
+
+        logger.info(f"Observatory report generated for {site} (v{model_version}): {output_path}")
+
+    def auto_generate_observatory_reports(self):
+        """Generate all observatory reports based on which --all_* flags are passed."""
+        for params in self._generate_observatory_report_combinations():
+            self._generate_single_observatory_report(*params)

simtools/reporting/docs_read_parameters.py

@@ -11,7 +11,6 @@ from pathlib import Path
 import numpy as np
 
 from simtools.db import db_handler
-from simtools.io_operations import io_handler
 from simtools.model.telescope_model import TelescopeModel
 from simtools.utils import names
 
@@ -30,20 +29,10 @@ class ReadParameters:
         self.site = args.get("site")
         self.model_version = args.get("model_version", None)
         self.output_path = output_path
+        self.observatory = args.get("observatory")
 
     def _convert_to_md(self, input_file):
-        """
-        Convert a '.dat' or '.ecsv' file to a Markdown file, preserving formatting.
-
-        Parameters
-        ----------
-        input_file: Path, str
-           Simulation data file (in '.dat' or '.ecsv' format).
-
-        Returns
-        -------
-        - Path to the created Markdown file.
-        """
+        """Convert a file to a Markdown file, preserving formatting."""
         input_file = Path(input_file)
         output_data_path = Path(self.output_path / "_data_files")
         output_data_path.mkdir(parents=True, exist_ok=True)
@@ -51,18 +40,20 @@ class ReadParameters:
         output_file = output_data_path / output_file_name
 
         try:
-            with (
-                input_file.open("r", encoding="utf-8") as infile,
-                output_file.open("w", encoding="utf-8") as outfile,
-            ):
-                outfile.write(f"# {input_file.stem}")
-                outfile.write("\n")
-                outfile.write("```")
-                outfile.write("\n")
-                file_contents = infile.read()
+            # First try with utf-8
+            try:
+                with input_file.open("r", encoding="utf-8") as infile:
+                    file_contents = infile.read()
+            except UnicodeDecodeError:
+                # If utf-8 fails, try with latin-1 (which can read any byte sequence)
+                with input_file.open("r", encoding="latin-1") as infile:
+                    file_contents = infile.read()
+
+            with output_file.open("w", encoding="utf-8") as outfile:
+                outfile.write(f"# {input_file.stem}\n")
+                outfile.write("```\n")
                 outfile.write(file_contents)
-                outfile.write("\n")
-                outfile.write("```")
+                outfile.write("\n```")
 
         except FileNotFoundError as exc:
             logger.exception(f"Data file not found: {input_file}.")
@@ -70,84 +61,6 @@ class ReadParameters:
 
         return f"_data_files/{output_file_name}"
 
-    def get_all_parameter_descriptions(self):
-        """
-        Get descriptions for all model parameters.
-
-        Returns
-        -------
-            tuple: A tuple containing two dictionaries:
-                - parameter_description: Maps parameter names to their descriptions.
-                - short_description: Maps parameter names to their short descriptions.
-                - inst_class: Maps parameter names to their respective class.
-        """
-        parameter_description, short_description, inst_class = {}, {}, {}
-
-        for instrument_class in names.db_collection_to_instrument_class_key("telescopes"):
-            for parameter, details in names.model_parameters(instrument_class).items():
-                parameter_description[parameter] = details.get("description")
-                short_description[parameter] = details.get("short_description")
-                inst_class[parameter] = instrument_class
-
-        return parameter_description, short_description, inst_class
-
-    def get_array_element_parameter_data(self, telescope_model, collection="telescopes"):
-        """
-        Get model parameter data and descriptions for a given array element.
-
-        Currently only configured for telescope.
-
-        Parameters
-        ----------
-        telescope_model : TelescopeModel
-            The telescope model instance.
-
-        Returns
-        -------
-        list: A list of lists containing parameter names, values with units,
-                descriptions, and short descriptions.
-        """
-        all_params = telescope_model.db.get_model_parameters(
-            site=telescope_model.site,
-            array_element_name=telescope_model.name,
-            collection=collection,
-            model_version=telescope_model.model_version,
-        )
-
-        telescope_model.export_model_files()
-        parameter_descriptions = self.get_all_parameter_descriptions()
-        data = []
-
-        for parameter in all_params:
-            if all_params[parameter]["instrument"] != telescope_model.name:
-                continue
-            parameter_version = telescope_model.get_parameter_version(parameter)
-            value = telescope_model.get_parameter_value_with_unit(parameter)
-            if telescope_model.get_parameter_file_flag(parameter) and value:
-                input_file_name = telescope_model.config_file_directory / Path(value)
-                output_file_name = self._convert_to_md(input_file_name)
-                value = f"[{Path(value).name}]({output_file_name})"
-            elif isinstance(value, list):
-                value = ", ".join(str(q) for q in value)
-            else:
-                value = str(value)
-
-            description = parameter_descriptions[0].get(parameter)
-            short_description = parameter_descriptions[1].get(parameter, description)
-            inst_class = parameter_descriptions[2].get(parameter)
-            data.append(
-                [
-                    inst_class,
-                    parameter,
-                    parameter_version,
-                    value,
-                    description,
-                    short_description,
-                ]
-            )
-
-        return data
-
     def _format_parameter_value(self, value_data, unit, file_flag):
         """Format parameter value based on type."""
         if file_flag:
@@ -216,14 +129,15 @@ class ReadParameters:
 
         # Iterate over each model version
         for version in all_versions:
-            Path(f"{self.output_path}/model").mkdir(parents=True, exist_ok=True)
+            parameter_dict = all_param_data.get(version, {})
+            if not parameter_dict:
+                continue
 
+            Path(f"{self.output_path}/model").mkdir(parents=True, exist_ok=True)
             self.db.export_model_files(
                 parameters=all_param_data.get(version), dest=f"{self.output_path}/model"
             )
 
-            parameter_dict = all_param_data.get(version, {})
-
             for parameter_name in filter(parameter_dict.__contains__, all_parameter_names):
                 parameter_data = parameter_dict.get(parameter_name)
 
@@ -234,7 +148,7 @@ class ReadParameters:
                 unit = parameter_data.get("unit") or " "
                 value_data = parameter_data.get("value")
 
-                if not value_data:
+                if value_data is None:
                     continue
 
                 file_flag = parameter_data.get("file", False)
@@ -254,6 +168,85 @@ class ReadParameters:
 
         return self._group_model_versions_by_parameter_version(grouped_data)
 
+    def get_all_parameter_descriptions(self):
+        """
+        Get descriptions for all model parameters.
+
+        Returns
+        -------
+            tuple: A tuple containing two dictionaries:
+                - parameter_description: Maps parameter names to their descriptions.
+                - short_description: Maps parameter names to their short descriptions.
+                - inst_class: Maps parameter names to their respective class.
+        """
+        parameter_description, short_description, inst_class = {}, {}, {}
+
+        for instrument_class in names.db_collection_to_instrument_class_key("telescopes"):
+            for parameter, details in names.model_parameters(instrument_class).items():
+                parameter_description[parameter] = details.get("description")
+                short_description[parameter] = details.get("short_description")
+                inst_class[parameter] = instrument_class
+
+        return parameter_description, short_description, inst_class
+
+    def get_array_element_parameter_data(self, telescope_model, collection="telescopes"):
+        """
+        Get model parameter data and descriptions for a given array element.
+
+        Currently only configured for telescope.
+
+        Parameters
+        ----------
+        telescope_model : TelescopeModel
+            The telescope model instance.
+
+        Returns
+        -------
+        list: A list of lists containing parameter names, values with units,
+                descriptions, and short descriptions.
+        """
+        all_parameter_data = self.db.get_model_parameters(
+            site=telescope_model.site,
+            array_element_name=telescope_model.name,
+            collection=collection,
+            model_version=telescope_model.model_version,
+        )
+
+        Path(f"{self.output_path}/model").mkdir(parents=True, exist_ok=True)
+        self.db.export_model_files(parameters=all_parameter_data, dest=f"{self.output_path}/model")
+        parameter_descriptions = self.get_all_parameter_descriptions()
+        data = []
+
+        for parameter in filter(all_parameter_data.__contains__, names.model_parameters().keys()):
+            parameter_data = all_parameter_data.get(parameter)
+            if parameter_data["instrument"] != telescope_model.name:
+                continue
+            parameter_version = telescope_model.get_parameter_version(parameter)
+            unit = parameter_data.get("unit") or " "
+            value_data = parameter_data.get("value")
+
+            if value_data is None:
+                continue
+
+            file_flag = parameter_data.get("file", False)
+            value = self._format_parameter_value(value_data, unit, file_flag)
+
+            description = parameter_descriptions[0].get(parameter)
+            short_description = parameter_descriptions[1].get(parameter, description)
+            inst_class = parameter_descriptions[2].get(parameter)
+            data.append(
+                [
+                    inst_class,
+                    parameter,
+                    parameter_version,
+                    value,
+                    description,
+                    short_description,
+                ]
+            )
+
+        return data
+
     def produce_array_element_report(self):
         """
         Produce a markdown report of all model parameters per array element.
@@ -261,6 +254,10 @@ class ReadParameters:
         Outputs one markdown report of a given array element listing parameter values,
         versions, and descriptions.
         """
+        if self.observatory:
+            self.produce_observatory_report()
+            return
+
         telescope_model = TelescopeModel(
             site=self.site,
             telescope_name=self.array_element,
@@ -332,10 +329,8 @@ class ReadParameters:
             f"Comparing parameters across model versions for Telescope: {self.array_element}"
             f" and Site: {self.site}."
         )
-        io_handler_instance = io_handler.IOHandler()
-        output_path = io_handler_instance.get_output_directory(
-            label="reports", sub_dir=f"parameters/{self.array_element}"
-        )
+        output_path = self.output_path / f"{self.array_element}"
+        Path(output_path).mkdir(parents=True, exist_ok=True)
 
         all_parameter_names = names.model_parameters(None).keys()
         all_parameter_data = self.db.get_model_parameters_for_all_model_versions(
@@ -381,3 +376,90 @@ class ReadParameters:
                 file.write("\n")
                 if comparison_data.get(parameter)[0]["file_flag"]:
                     file.write(f"![Parameter plot.](_images/{self.array_element}_{parameter}.png)")
+
+    def _write_array_layouts_section(self, file, layouts):
+        """Write the array layouts section of the report."""
+        file.write("\n## Array Layouts {#array-layouts-details}\n\n")
+        for layout in layouts:
+            layout_name = layout["name"]
+            elements = layout["elements"]
+            file.write(f"### {layout_name}\n\n")
+            file.write("| Element |\n|---------|\n")
+            for element in sorted(elements):
+                file.write(f"| [{element}]({element}.md) |\n")
+            file.write("\n")
+
+    def _write_array_triggers_section(self, file, trigger_configs):
+        """Write the array triggers section of the report."""
+        file.write("\n## Array Trigger Configurations {#array-triggers-details}\n\n")
+        file.write(
+            "| Trigger Name | Multiplicity | Width | Hard Stereo | Min Separation |\n"
+            "|--------------|--------------|--------|-------------|----------------|\n"
+        )
+        for config in trigger_configs:
+            name = config["name"]
+            mult = f"{config['multiplicity']['value']} {config['multiplicity']['unit'] or ''}"
+            width = f"{config['width']['value']} {config['width']['unit'] or ''}"
+            stereo = "Yes" if config["hard_stereo"]["value"] else "No"
+            min_sep = (
+                f"{config['min_separation']['value']} {config['min_separation']['unit'] or '-'}"
+            )
+            file.write(
+                f"| {name} | {mult.strip()} | {width.strip()} | {stereo} | {min_sep.strip()} |\n"
+            )
+        file.write("\n")
+
+    def _write_parameters_table(self, file, all_parameter_data):
+        """Write the main parameters table of the report."""
+        file.write("| Parameter | Value |\n|-----------|--------|\n")
+        for param_name, param_data in sorted(all_parameter_data.items()):
+            value = param_data.get("value")
+            unit = param_data.get("unit") or " "
+            file_flag = param_data.get("file", False)
+
+            if value is None:
+                continue
+
+            if param_name == "array_layouts":
+                file.write("| array_layouts | [View Array Layouts](#array-layouts-details) |\n")
+            elif param_name == "array_triggers":
+                file.write(
+                    "| array_triggers | [View Trigger Configurations](#array-triggers-details) |\n"
+                )
+            else:
+                formatted_value = self._format_parameter_value(value, unit, file_flag)
+                file.write(f"| {param_name} | {formatted_value} |\n")
+        file.write("\n")
+
+    def produce_observatory_report(self):
+        """Produce a markdown report of all observatory parameters for a given site."""
+        output_filename = Path(self.output_path / f"OBS-{self.site}.md")
+        output_filename.parent.mkdir(parents=True, exist_ok=True)
+
+        all_parameter_data = self.db.get_model_parameters(
+            site=self.site,
+            array_element_name="OBS-" + self.site,
+            collection="sites",
+            model_version=self.model_version,
+        )
+
+        if not all_parameter_data:
+            logger.warning(f"No observatory parameters found for site {self.site}")
+            return
+
+        Path(f"{self.output_path}/model").mkdir(parents=True, exist_ok=True)
+        self.db.export_model_files(parameters=all_parameter_data, dest=f"{self.output_path}/model")
+
+        with output_filename.open("w", encoding="utf-8") as file:
+            file.write(f"# Observatory Parameters - {self.site} Site\n\n")
+            self._write_parameters_table(file, all_parameter_data)
+
+            if "array_layouts" in all_parameter_data:
+                self._write_array_layouts_section(
+                    file, all_parameter_data["array_layouts"]["value"]
+                )
+
+            if "array_triggers" in all_parameter_data:
+                self._write_array_triggers_section(
+                    file, all_parameter_data["array_triggers"]["value"]
+                )

simtools/schemas/application_workflow.metaschema.yml

@@ -48,6 +48,9 @@ definitions:
             TEST_NAME:
               type: string
               description: "Test name. Should be unique for this application."
+            SKIP_FOR_PRODUCTION_DB:
+              type: boolean
+              description: "Skip test if production DB is used."
             MODEL_VERSION_USE_CURRENT:
               type: boolean
               description: |

simtools/schemas/model_parameter.metaschema.yml

@@ -14,6 +14,7 @@ definitions:
   SimtoolsModelParameters:
     description: ""
     type: object
+    additionalProperties: false
     properties:
       _id:
         type: string
@@ -29,6 +30,12 @@ definitions:
           - string
           - "null"
         description: "Associated instrument."
+      meta_parameter:
+        type: boolean
+        description: "Meta parameter flag for sim_telarray."
+      parameter:
+        type: string
+        description: "Parameter name."
       site:
         anyOf:
           - type: string
@@ -62,6 +69,11 @@ definitions:
         anyOf:
           - type: string
           - type: "null"
+          - type: array
+            items:
+              type:
+                - string
+                - "null"
         description: "Unit of the parameter."
       value:
         anyOf:
@@ -87,6 +99,7 @@ definitions:
     required:
       - file
       - instrument
+      - parameter
       - parameter_version
       - schema_version
       - site

simtools/schemas/model_parameter_and_data_schema.metaschema.yml

@@ -290,6 +290,10 @@ definitions:
         type: string
         description: |-
           "Naming of parameter as used this simulation software"
+      set_meta_parameter:
+        type: boolean
+        description: |-
+          "Add this parameter as sim_telarray meta parameter using option 'set'"
     required:
       - name
     title: SimulationSoftware
@@ -362,8 +366,10 @@ definitions:
       - double
       - float64
       - int
+      - int64
       - string
       - uint
+      - uint64
       - file
     title: TableColumnType
   Unit:

simtools/schemas/model_parameters/adjust_gain.schema.yml

@@ -12,7 +12,7 @@ description: |-
 short_description: |-
   Gain adjustment for amplitudes and PMT gain.
 data:
-  - type: double
+  - type: float64
     unit: dimensionless
     default: 1.0
     allowed_range:

simtools/schemas/model_parameters/altitude.schema.yml

@@ -12,7 +12,7 @@ description: |-
   (taking `telescope_axes_height` into account).
 short_description: Altitude above sea level of site centre.
 data:
-  - type: double
+  - type: float64
     unit: m
     default: 1800.
     allowed_range: