gammasimtools 0.12.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

simtools/production_configuration/extract_mc_event_data.py

@@ -0,0 +1,253 @@
+"""Generate a reduced dataset from given simulation event list and save the output to file."""
+
+import logging
+
+import numpy as np
+import tables
+from ctapipe.core import Container, Field
+from ctapipe.io import HDF5TableWriter
+from eventio import EventIOFile
+from eventio.simtel import ArrayEvent, MCEvent, MCRunHeader, MCShower, TriggerInformation
+
+DEFAULT_FILTERS = tables.Filters(complevel=5, complib="zlib", shuffle=True, bitshuffle=False)
+
+
+class ReducedDatasetContainer(Container):
+    """Container for reduced dataset information."""
+
+    simulated = Field(None, "Simulated energy")
+    core_x = Field(None, "X-coordinate of the shower core")
+    core_y = Field(None, "Y-coordinate of the shower core")
+    shower_sim_azimuth = Field(None, "Simulated azimuth angle of the shower")
+    shower_sim_altitude = Field(None, "Simulated altitude angle of the shower")
+    array_altitudes = Field(None, "Altitudes for the array")
+    array_azimuths = Field(None, "Azimuths for the array")
+
+
+class TriggeredShowerContainer(Container):
+    """Container for triggered shower information."""
+
+    shower_id_triggered = Field(None, "Triggered shower ID")
+    triggered_energies = Field(None, "Triggered energies")
+
+
+class FileNamesContainer(Container):
+    """Container for file names."""
+
+    file_names = Field(None, "Input file names")
+
+
+class MCEventExtractor:
+    """
+    Generate a reduced dataset from given simulation event list and save the output to file.
+
+    Attributes
+    ----------
+    input_files : list
+        List of input file paths to process.
+    output_file : str
+        Path to the output HDF5 file.
+    max_files : int, optional
+        Maximum number of files to process.
+    """
+
+    def __init__(self, input_files, output_file, max_files=100):
+        """
+        Initialize the MCEventExtractor with input files, output file, and max file limit.
+
+        Parameters
+        ----------
+        input_files : list
+            List of input file paths to process.
+        output_file : str
+            Path to the output HDF5 file.
+        max_files : int, optional
+            Maximum number of files to process.
+        """
+        self._logger = logging.getLogger(__name__)
+        self.input_files = input_files
+        self.output_file = output_file
+        self.max_files = max_files
+        self.shower = None
+        self.n_use = None
+        self.shower_id_offset = 0
+
+    def process_files(self):
+        """Process the input files and store them in an HDF5 file."""
+        if not self.input_files:
+            self._logger.warning("No input files provided.")
+            return
+
+        data_lists = self._initialize_data_lists()
+        self.shower_id_offset = 0
+        # Process the first file in write mode
+        self._logger.info(f"Processing file 1/{self.max_files}: {self.input_files[0]}")
+        self._process_file(self.input_files[0], data_lists, str(self.input_files[0]))
+        self._write_all_data(data_lists, mode="w")
+        self.shower_id_offset = len(data_lists["simulated"])
+        self._reset_data_lists(data_lists)
+
+        # Process remaining files in append mode
+        for i_file, file in enumerate(self.input_files[1 : self.max_files], start=2):
+            self._logger.info(f"Processing file {i_file}/{self.max_files}: {file}")
+            self._process_file(file, data_lists, str(file))
+            if len(data_lists["simulated"]) >= 1e7:
+                self._write_all_data(data_lists, mode="a")
+                self.shower_id_offset += len(data_lists["simulated"])
+                self._reset_data_lists(data_lists)
+
+        # Final write for any remaining data
+        self._write_all_data(data_lists, mode="a")
+
+    def _write_all_data(self, data_lists, mode):
+        """Write all data sections at once helper method."""
+        self._write_data(data_lists, mode=mode)
+        self._write_variable_length_data(data_lists["trigger_telescope_list_list"], mode="a")
+        self._write_file_names(data_lists["file_names"], mode="a")
+
+    def _write_file_names(self, file_names, mode="a"):
+        """Write file names to HDF5 file."""
+        print("file_names", file_names)
+        with HDF5TableWriter(
+            self.output_file, group_name="data", mode=mode, filters=DEFAULT_FILTERS
+        ) as writer:
+            file_names_container = FileNamesContainer()
+            for file_name in file_names:
+                file_names_container.file_names = file_name
+                writer.write(table_name="file_names", containers=[file_names_container])
+
+    def _write_variable_length_data(self, trigger_telescope_list_list, mode="a"):
+        """Write variable-length array data to HDF5 file."""
+        with tables.open_file(self.output_file, mode=mode) as f:
+            if "trigger_telescope_list_list" in f.root.data:
+                vlarray = f.root.data.trigger_telescope_list_list
+            else:
+                vlarray = f.create_vlarray(
+                    f.root.data,
+                    "trigger_telescope_list_list",
+                    tables.Int16Atom(),
+                    "List of triggered telescope IDs",
+                )
+
+            for item in trigger_telescope_list_list:
+                vlarray.append(item)
+
+    def _initialize_data_lists(self):
+        """Initialize data lists."""
+        return {
+            "simulated": [],
+            "shower_id_triggered": [],
+            "triggered_energies": [],
+            "core_x": [],
+            "core_y": [],
+            "trigger_telescope_list_list": [],
+            "file_names": [],
+            "shower_sim_azimuth": [],
+            "shower_sim_altitude": [],
+            "array_altitudes": [],
+            "array_azimuths": [],
+        }
+
+    def _process_file(self, file, data_lists, file_name):
+        """Process a single file and update data lists."""
+        with EventIOFile(file) as f:
+            array_altitude = None
+            array_azimuth = None
+            for eventio_object in f:
+                if isinstance(eventio_object, MCRunHeader):
+                    self._process_mc_run_header(eventio_object, data_lists)
+                elif isinstance(eventio_object, MCShower):
+                    self._process_mc_shower(
+                        eventio_object, data_lists, array_altitude, array_azimuth
+                    )
+                elif isinstance(eventio_object, MCEvent):
+                    self._process_mc_event(eventio_object, data_lists)
+                elif isinstance(eventio_object, ArrayEvent):
+                    self._process_array_event(eventio_object, data_lists)
+            data_lists["file_names"].extend([file_name])
+
+    def _process_mc_run_header(self, eventio_object, data_lists):
+        """Process MC run header and update data lists."""
+        mc_head = eventio_object.parse()
+        self.n_use = mc_head["n_use"]  # reuse factor n_use needed to extend the values below
+        array_altitude = np.mean(mc_head["alt_range"])
+        array_azimuth = np.mean(mc_head["az_range"])
+        data_lists["array_altitudes"].extend(self.n_use * [array_altitude])
+        data_lists["array_azimuths"].extend(self.n_use * [array_azimuth])
+
+    def _process_mc_shower(self, eventio_object, data_lists, array_altitude, array_azimuth):
+        """Process MC shower and update data lists."""
+        self.shower = eventio_object.parse()
+        data_lists["simulated"].extend(self.n_use * [self.shower["energy"]])
+        data_lists["shower_sim_azimuth"].extend(self.n_use * [self.shower["azimuth"]])
+        data_lists["shower_sim_altitude"].extend(self.n_use * [self.shower["altitude"]])
+        data_lists["array_altitudes"].extend(self.n_use * [array_altitude])
+        data_lists["array_azimuths"].extend(self.n_use * [array_azimuth])
+
+    def _process_mc_event(self, eventio_object, data_lists):
+        """Process MC event and update data lists."""
+        event = eventio_object.parse()
+        data_lists["core_x"].append(event["xcore"])
+        data_lists["core_y"].append(event["ycore"])
+
+    def _process_array_event(self, eventio_object, data_lists):
+        """Process array event and update data lists."""
+        for i, obj in enumerate(eventio_object):
+            if i == 0 and isinstance(obj, TriggerInformation):
+                self._process_trigger_information(obj, data_lists)
+
+    def _process_trigger_information(self, trigger_info, data_lists):
+        """Process trigger information and update data lists."""
+        trigger_info = trigger_info.parse()
+        telescopes = trigger_info["telescopes_with_data"]
+        if len(telescopes) > 0:
+            data_lists["shower_id_triggered"].append(self.shower["shower"] + self.shower_id_offset)
+            data_lists["triggered_energies"].append(self.shower["energy"])
+            data_lists["trigger_telescope_list_list"].append(np.array(telescopes, dtype=np.int16))
+
+    def _write_data(self, data_lists, mode="a"):
+        """Write data to HDF5 file using HDF5TableWriter."""
+        with HDF5TableWriter(
+            self.output_file, group_name="data", mode=mode, filters=DEFAULT_FILTERS
+        ) as writer:
+            # Write reduced dataset container
+            reduced_container = ReducedDatasetContainer()
+            for i in range(len(data_lists["simulated"])):
+                reduced_container.simulated = data_lists["simulated"][i]
+                reduced_container.core_x = data_lists["core_x"][i]
+                reduced_container.core_y = data_lists["core_y"][i]
+                reduced_container.shower_sim_azimuth = data_lists["shower_sim_azimuth"][i]
+                reduced_container.shower_sim_altitude = data_lists["shower_sim_altitude"][i]
+                reduced_container.array_altitudes = data_lists["array_altitudes"][i]
+                reduced_container.array_azimuths = data_lists["array_azimuths"][i]
+                writer.write(table_name="reduced_data", containers=[reduced_container])
+
+            # Write triggered shower container
+            triggered_container = TriggeredShowerContainer()
+            for i in range(len(data_lists["shower_id_triggered"])):
+                triggered_container.shower_id_triggered = data_lists["shower_id_triggered"][i]
+                triggered_container.triggered_energies = data_lists["triggered_energies"][i]
+                writer.write(table_name="triggered_data", containers=[triggered_container])
+
+    def _reset_data_lists(self, data_lists):
+        """Reset data lists during batch processing."""
+        for key in data_lists:
+            data_lists[key] = []
+
+    def print_dataset_information(self):
+        """Print information about the datasets in the generated HDF5 file."""
+        try:
+            with tables.open_file(self.output_file, mode="r") as reader:
+                print("Datasets in file:")
+                for key in reader.root.data._v_children.keys():  # pylint: disable=protected-access
+                    dset = reader.root.data._v_children[key]  # pylint: disable=protected-access
+                    print(f"- {key}: shape={dset.shape}, dtype={dset.dtype}")
+
+                    # Print first 5 values each
+                    print(f"  First 5 values: {dset[:5]}")
+
+                    # Print units if available
+                    # units = dset.attrs.get("units", "N/A")
+                    # print(f"  Units: {units}")
+        except Exception as exc:
+            raise ValueError("An error occurred while reading the HDF5 file") from exc

simtools/ray_tracing/mirror_panel_psf.py

@@ -295,6 +295,6 @@ class MirrorPanelPSF:
         )
         writer.ModelDataWriter.dump(
             args_dict=self.args_dict,
-            metadata=MetadataCollector(args_dict=self.args_dict).get_top_level_metadata(),
+            metadata=MetadataCollector(args_dict=self.args_dict),
             product_data=result_table,
         )

simtools/reporting/docs_read_parameters.py

@@ -4,9 +4,13 @@ r"""Class to read and manage relevant model parameters for a given telescope mod
 
 import logging
 import textwrap
+from collections import defaultdict
 from itertools import groupby
 from pathlib import Path
 
+import numpy as np
+
+from simtools.db import db_handler
 from simtools.io_operations import io_handler
 from simtools.model.telescope_model import TelescopeModel
 from simtools.utils import names
@@ -15,13 +19,16 @@ logger = logging.getLogger()
 
 
 class ReadParameters:
-    """Read and manage model parameter data."""
+    """Read and manage model parameter data for report generation."""
 
-    def __init__(self, db_config, telescope_model, output_path):
-        """Initialise class with a telescope model."""
+    def __init__(self, db_config, args, output_path):
+        """Initialise class."""
         self._logger = logging.getLogger(__name__)
+        self.db = db_handler.DatabaseHandler(mongo_db_config=db_config)
         self.db_config = db_config
-        self.telescope_model = telescope_model
+        self.array_element = args.get("telescope")
+        self.site = args.get("site")
+        self.model_version = args.get("model_version", None)
         self.output_path = output_path
 
     def _convert_to_md(self, input_file):
@@ -86,9 +93,9 @@ class ReadParameters:
 
     def get_array_element_parameter_data(self, telescope_model, collection="telescopes"):
         """
-        Get model parameter data for a given array element.
+        Get model parameter data and descriptions for a given array element.
 
-        Currently only configures for telescope.
+        Currently only configured for telescope.
 
         Parameters
         ----------
@@ -104,6 +111,7 @@ class ReadParameters:
             site=telescope_model.site,
             array_element_name=telescope_model.name,
             collection=collection,
+            model_version=telescope_model.model_version,
         )
 
         telescope_model.export_model_files()
@@ -140,75 +148,143 @@ class ReadParameters:
 
         return data
 
-    def _compare_parameter_across_versions(self, parameter_name):
+    def _format_parameter_value(self, value_data, unit, file_flag):
+        """Format parameter value based on type."""
+        if file_flag:
+            input_file_name = f"{self.output_path}/model/{value_data}"
+            output_file_name = self._convert_to_md(input_file_name)
+            return f"[{Path(value_data).name}]({output_file_name})".strip()
+        if isinstance(value_data, (str | int | float)):
+            return f"{value_data} {unit}".strip()
+        if len(value_data) > 5 and np.allclose(value_data, value_data[0]):
+            return f"all: {value_data[0]} {unit}".strip()
+        return (
+            ", ".join(f"{v} {u}" for v, u in zip(value_data, unit))
+            if isinstance(unit, list)
+            else ", ".join(f"{v} {unit}" for v in value_data)
+        ).strip()
+
+    def _group_model_versions_by_parameter_version(self, grouped_data):
+        """Group model versions by parameter version and track the parameter values."""
+        result = {}
+
+        for parameter_name, items in grouped_data.items():
+            version_grouped = defaultdict(
+                lambda: {"model_versions": [], "value": None, "file_flag": None}
+            )
+
+            for item in items:
+                param_version = item["parameter_version"]
+                version_grouped[param_version]["model_versions"].append(item["model_version"])
+
+                if version_grouped[param_version]["value"] is None:
+                    version_grouped[param_version]["value"] = item["value"]
+                    version_grouped[param_version]["file_flag"] = item["file_flag"]
+
+            result[parameter_name] = [
+                {
+                    "value": data["value"],
+                    "parameter_version": param_version,
+                    "file_flag": data["file_flag"],
+                    "model_version": ", ".join(data["model_versions"]),
+                }
+                for param_version, data in version_grouped.items()
+            ]
+
+        return result
+
+    def _compare_parameter_across_versions(self, all_param_data, all_parameter_names):
         """
         Compare a parameter's value across different model versions.
 
         Parameters
         ----------
-        parameter_name : str
-            The name of the parameter to compare.
+        all_param_data : dict
+            The dictionary containing parameter data for all versions.
+
+        all_parameter_names : list
+            The list of parameter names to compare across versions.
 
         Returns
         -------
         list
-            A list of dictionaries containing model version, parameter value, and description.
+            A list of dictionaries containing model version, parameter value, description.
         """
-        all_versions = self.telescope_model.db.get_model_versions()
-        all_versions.reverse()
-        comparison_data = []
-
-        for model_version in all_versions:
-            telescope_model = TelescopeModel(
-                site=self.telescope_model.site,
-                telescope_name=self.telescope_model.name,
-                model_version=model_version,
-                label="reports",
-                mongo_db_config=self.db_config,
+        all_versions = self.db.get_model_versions()
+        all_versions.reverse()  # latest first
+        grouped_data = defaultdict(list)
+
+        # Iterate over each model version
+        for version in all_versions:
+            Path(f"{self.output_path}/model").mkdir(parents=True, exist_ok=True)
+
+            self.db.export_model_files(
+                parameters=all_param_data.get(version), dest=f"{self.output_path}/model"
             )
 
-            if not telescope_model.has_parameter(parameter_name):
-                return comparison_data
-
-            parameter_data = self.get_array_element_parameter_data(telescope_model)
-            for param in parameter_data:
-                if param[1] == parameter_name:
-                    comparison_data.append(
-                        {
-                            "model_version": model_version,
-                            "parameter_version": param[2],
-                            "value": param[3],
-                            "description": param[4],
-                        }
-                    )
-                    break
-        return comparison_data
+            parameter_dict = all_param_data.get(version, {})
+
+            for parameter_name in filter(parameter_dict.__contains__, all_parameter_names):
+                parameter_data = parameter_dict.get(parameter_name)
+
+                # Skip if instrument doesn't match
+                if parameter_data.get("instrument") != self.array_element:
+                    continue
+
+                unit = parameter_data.get("unit") or " "
+                value_data = parameter_data.get("value")
+
+                if not value_data:
+                    continue
+
+                file_flag = parameter_data.get("file", False)
+                value = self._format_parameter_value(value_data, unit, file_flag)
+                parameter_version = parameter_data.get("parameter_version")
+                model_version = version
+
+                # Group the data by parameter version and store model versions as a list
+                grouped_data[parameter_name].append(
+                    {
+                        "value": value,
+                        "parameter_version": parameter_version,
+                        "model_version": model_version,
+                        "file_flag": file_flag,
+                    }
+                )
+
+        return self._group_model_versions_by_parameter_version(grouped_data)
 
     def produce_array_element_report(self):
         """
         Produce a markdown report of all model parameters per array element.
 
-        Output
-        ----------
-        One markdown report of a given array element listing parameter values,
+        Outputs one markdown report of a given array element listing parameter values,
         versions, and descriptions.
         """
-        output_filename = Path(self.output_path / (self.telescope_model.name + ".md"))
+        telescope_model = TelescopeModel(
+            site=self.site,
+            telescope_name=self.array_element,
+            model_version=self.model_version,
+            label="reports",
+            mongo_db_config=self.db_config,
+        )
+
+        output_filename = Path(self.output_path / (telescope_model.name + ".md"))
         output_filename.parent.mkdir(parents=True, exist_ok=True)
-        data = self.get_array_element_parameter_data(self.telescope_model)
+        data = self.get_array_element_parameter_data(telescope_model)
         # Sort data by class to prepare for grouping
         if not isinstance(data, str):
             data.sort(key=lambda x: (x[0], x[1]), reverse=True)
 
         with output_filename.open("w", encoding="utf-8") as file:
             # Group by class and write sections
-            file.write(f"# {self.telescope_model.name}\n")
+            file.write(f"# {telescope_model.name}\n")
 
-            if self.telescope_model.name != self.telescope_model.design_model:
+            if telescope_model.name != telescope_model.design_model:
                 file.write(
                     "The design model can be found here: "
-                    f"[{self.telescope_model.design_model}]"
-                    f"({self.telescope_model.design_model}.md).\n"
+                    f"[{telescope_model.design_model}]"
+                    f"({telescope_model.design_model}.md).\n"
                 )
                 file.write("\n\n")
 
@@ -249,62 +325,59 @@ class ReadParameters:
         """
         Produce a markdown report per parameter for a given array element.
 
-        Output
-        ----------
-        One markdown report per model parameter of a given array element comparing
+        Outputs one markdown report per model parameter of a given array element comparing
         values across model versions.
         """
         logger.info(
-            f"Comparing parameters across model versions for Telescope: {self.telescope_model.name}"
-            f" and Site: {self.telescope_model.site}."
+            f"Comparing parameters across model versions for Telescope: {self.array_element}"
+            f" and Site: {self.site}."
         )
         io_handler_instance = io_handler.IOHandler()
         output_path = io_handler_instance.get_output_directory(
-            label="reports", sub_dir=f"parameters/{self.telescope_model.name}"
+            label="reports", sub_dir=f"parameters/{self.array_element}"
         )
 
-        all_params = self.telescope_model.db.get_model_parameters(
-            site=self.telescope_model.site,
-            array_element_name=self.telescope_model.name,
-            collection="telescopes",
+        all_parameter_names = names.model_parameters(None).keys()
+        all_parameter_data = self.db.get_model_parameters_for_all_model_versions(
+            site=self.site, array_element_name=self.array_element, collection="telescopes"
         )
 
-        for parameter in all_params:
-            comparison_data = []
-            if all_params[parameter]["instrument"] == self.telescope_model.name:
-                comparison_data = self._compare_parameter_across_versions(parameter)
-            if comparison_data:
-                output_filename = output_path / f"{parameter}.md"
-                with output_filename.open("w", encoding="utf-8") as file:
-                    # Write header
-                    file.write(
-                        f"# {parameter}\n\n"
-                        f"**Telescope**: {self.telescope_model.name}\n\n"
-                        f"**Description**: {comparison_data[0]['description']}\n\n"
-                        "\n"
-                    )
+        comparison_data = self._compare_parameter_across_versions(
+            all_parameter_data, all_parameter_names
+        )
 
-                    # Write table header
+        for parameter in all_parameter_names:
+            parameter_data = comparison_data.get(parameter)
+            if not parameter_data:
+                continue
+
+            output_filename = output_path / f"{parameter}.md"
+            description = self.get_all_parameter_descriptions()[0].get(parameter)
+            with output_filename.open("w", encoding="utf-8") as file:
+                # Write header
+                file.write(
+                    f"# {parameter}\n\n"
+                    f"**Telescope**: {self.array_element}\n\n"
+                    f"**Description**: {description}\n\n"
+                    "\n"
+                )
+
+                # Write table header
+                file.write(
+                    "| Parameter Version      | Model Version(s)      "
+                    "| Value                |\n"
+                    "|------------------------|--------------------"
+                    "|----------------------|\n"
+                )
+
+                # Write table rows
+                for item in comparison_data.get(parameter):
                     file.write(
-                        "| Model Version      | Parameter Version      "
-                        "| Value                |\n"
-                        "|--------------------|------------------------"
-                        "|----------------------|\n"
+                        f"| {item['parameter_version']} |"
+                        f" {item['model_version']} |"
+                        f"{item['value'].replace('](', '](../')} |\n"
                     )
 
-                    # Write table rows
-                    for item in comparison_data:
-                        file.write(
-                            f"| {item['model_version']} |"
-                            f" {item['parameter_version']} |"
-                            f"{item['value'].replace('](', '](../')} |\n"
-                        )
-
-                    file.write("\n")
-                    if isinstance(comparison_data[0]["value"], str) and comparison_data[0][
-                        "value"
-                    ].endswith(".md)"):
-                        file.write(
-                            f"![Parameter plot.](_images/"
-                            f"{self.telescope_model.name}_{parameter}.png)"
-                        )
+                file.write("\n")
+                if comparison_data.get(parameter)[0]["file_flag"]:
+                    file.write(f"![Parameter plot.](_images/{self.array_element}_{parameter}.png)")

simtools/schemas/metadata.metaschema.yml

@@ -51,9 +51,7 @@ definitions:
           Describes the person or institution that is responsible for this
           data product.
         required:
-          - EMAIL
           - NAME
-          - ORGANIZATION
         type: object
         additionalProperties: false
         properties:
@@ -90,10 +88,8 @@ definitions:
           links to the data model definition.
         required:
           - DATA
-          - DESCRIPTION
           - FORMAT
           - ID
-          - VALID
         type: object
         additionalProperties: false
         properties:
@@ -125,7 +121,6 @@ definitions:
             title: Data
             required:
               - CATEGORY
-              - ASSOCIATION
               - LEVEL
               - MODEL
             type: object
@@ -346,7 +341,6 @@ definitions:
           associated ID.
         required:
           - TYPE
-          - ID
         type: object
         additionalProperties: false
         properties:
@@ -524,6 +518,13 @@ definitions:
                     - type: string
                     - type: "null"
                   default: null
+                AUTHOR:
+                  description: |-
+                    Author of document.
+                  anyOf:
+                    - type: string
+                    - type: "null"
+                  default: null
                 ID:
                   description: |-
                     Unique identifier of document.

simtools/schemas/model_parameter.metaschema.yml

@@ -205,9 +205,6 @@ definitions:
       _id:
         type: string
         description: "DB unique identifier"
-      applicable:
-        type: boolean
-        description: "Model parameter to be used for this telescope and site"
       entry_date:
         type: string
         description: "Value entry data"
@@ -257,7 +254,6 @@ definitions:
           - type: "null"
         description: "Model version."
     required:
-      - applicable
       - file
       - instrument
       - site