gammasimtools 0.16.0__py3-none-any.whl → 0.17.0__py3-none-any.whl

simtools/reporting/docs_read_parameters.py

@@ -12,11 +12,13 @@ from pathlib import Path
 import numpy as np
 
 from simtools.db import db_handler
+from simtools.io_operations import io_handler
 from simtools.model.telescope_model import TelescopeModel
+from simtools.utils import general as gen
 from simtools.utils import names
+from simtools.visualization import plot_pixels
 
 logger = logging.getLogger()
-IMAGE_PATH = "../../_images"
 
 
 class ReadParameters:
@@ -62,41 +64,61 @@ class ReadParameters:
             )
         self._model_version = model_version
 
-    def _convert_to_md(self, parameter, input_file):
+    def _convert_to_md(self, parameter, parameter_version, input_file):
         """Convert a file to a Markdown file, preserving formatting."""
         input_file = Path(input_file)
+
         output_data_path = Path(self.output_path / "_data_files")
         output_data_path.mkdir(parents=True, exist_ok=True)
         output_file_name = Path(input_file.stem + ".md")
         output_file = output_data_path / output_file_name
+        image_name = f"{self.array_element}_{parameter}_{self.model_version.replace('.', '-')}"
+        outpath = Path(io_handler.IOHandler().get_output_directory().parent / "_images")
+        outpath.mkdir(parents=True, exist_ok=True)
+        image_path = Path(f"{outpath}/{image_name}")
+
+        if parameter == "camera_config_file" and parameter_version:
+            image_path = Path(f"{outpath}/{input_file.stem.replace('.', '-')}")
+            if not (image_path.with_suffix(".png")).exists():
+                logger.info("Plotting camera configuration file: %s", input_file.name)
+                plot_config = {
+                    "file_name": input_file.name,
+                    "telescope": self.array_element,
+                    "parameter_version": parameter_version,
+                    "site": self.site,
+                    "model_version": self.model_version,
+                    "parameter": parameter,
+                }
+
+                plot_pixels.plot(
+                    config=plot_config,
+                    output_file=image_path,
+                    db_config=self.db_config,
+                )
+            else:
+                logger.info(
+                    "Camera configuration file plot already exists: %s",
+                    image_path.with_suffix(".png"),
+                )
 
         try:
-            # First try with utf-8
-            try:
-                with input_file.open("r", encoding="utf-8") as infile:
-                    file_contents = infile.read()
-            except UnicodeDecodeError:
-                # If utf-8 fails, try with latin-1 (which can read any byte sequence)
-                with input_file.open("r", encoding="latin-1") as infile:
-                    file_contents = infile.read()
+            # with input_file.open("r", encoding="utf-8") as infile:
+            file_contents = gen.read_file_encoded_in_utf_or_latin(input_file)
 
             if self.model_version is not None:
                 with output_file.open("w", encoding="utf-8") as outfile:
                     outfile.write(f"# {input_file.stem}\n")
+                    outfile.write(f"![Parameter plot.]({image_path}.png)\n\n")
                     outfile.write(
-                        "The full file can be found in the Simulation Model repository [here]"
+                        "\n\nThe full file can be found in the Simulation Model repository [here]"
                         "(https://gitlab.cta-observatory.org/cta-science/simulations/"
                         "simulation-model/simulation-models/-/blob/main/simulation-models/"
                         f"model_parameters/Files/{input_file.name}).\n\n"
                     )
-                    outfile.write(
-                        f"![Parameter plot.](../{IMAGE_PATH}/{self.array_element}_"
-                        f"{parameter}_{self.model_version.replace('.', '-')}.png)\n"
-                    )
                     outfile.write("\n\n")
                     outfile.write("The first 30 lines of the file are:\n")
                     outfile.write("```\n")
-                    first_30_lines = "\n".join(file_contents.splitlines()[:30])
+                    first_30_lines = "".join(file_contents[:30])
                     outfile.write(first_30_lines)
                     outfile.write("\n```")
 
@@ -106,11 +128,19 @@ class ReadParameters:
 
         return f"_data_files/{output_file_name}"
 
-    def _format_parameter_value(self, parameter, value_data, unit, file_flag):
+    def _format_parameter_value(
+        self, parameter, value_data, unit, file_flag, parameter_version=None
+    ):
         """Format parameter value based on type."""
         if file_flag:
             input_file_name = f"{self.output_path}/model/{value_data}"
-            output_file_name = self._convert_to_md(parameter, input_file_name)
+            if parameter_version is None:
+                return (
+                    f"[{Path(value_data).name}](https://gitlab.cta-observatory.org/"
+                    "cta-science/simulations/simulation-model/simulation-models/-/blob/main/"
+                    f"simulation-models/model_parameters/Files/{value_data})"
+                ).strip()
+            output_file_name = self._convert_to_md(parameter, parameter_version, input_file_name)
             return f"[{Path(value_data).name}]({output_file_name})".strip()
         if isinstance(value_data, (str | int | float)):
             return f"{value_data} {unit}".strip()
@@ -217,8 +247,10 @@ class ReadParameters:
                     continue
 
                 file_flag = parameter_data.get("file", False)
-                value = self._format_parameter_value(parameter_name, value_data, unit, file_flag)
                 parameter_version = parameter_data.get("parameter_version")
+                value = self._format_parameter_value(
+                    parameter_name, value_data, unit, file_flag, parameter_version=None
+                )
                 model_version = version
 
                 # Group the data by parameter version and store model versions as a list
@@ -234,25 +266,28 @@ class ReadParameters:
         return self._group_model_versions_by_parameter_version(grouped_data)
 
     def get_all_parameter_descriptions(self, collection="telescopes"):
-        """
-        Get descriptions for all model parameters.
+        """Get descriptions for all model parameters.
 
         Returns
         -------
-            tuple: A tuple containing two dictionaries:
-                - parameter_description: Maps parameter names to their descriptions.
-                - short_description: Maps parameter names to their short descriptions.
-                - inst_class: Maps parameter names to their respective class.
+        dict
+            Nested dictionaries with first key as the parameter name and
+            the following dictionary as the value:
+            - key: description, value: description of the parameter.
+            - key: short_description, value: short description of the parameter.
+            - key: inst_class, value: class, for eg. Structure, Camera, etc.
         """
-        parameter_description, short_description, inst_class = {}, {}, {}
+        parameter_dict = {}
 
         for instrument_class in names.db_collection_to_instrument_class_key(collection):
             for parameter, details in names.model_parameters(instrument_class).items():
-                parameter_description[parameter] = details.get("description")
-                short_description[parameter] = details.get("short_description")
-                inst_class[parameter] = instrument_class
+                parameter_dict[parameter] = {
+                    "description": details.get("description"),
+                    "short_description": details.get("short_description"),
+                    "inst_class": instrument_class,
+                }
 
-        return parameter_description, short_description, inst_class
+        return parameter_dict
 
     def get_array_element_parameter_data(self, telescope_model, collection="telescopes"):
         """
@@ -292,13 +327,15 @@ class ReadParameters:
                 continue
 
             file_flag = parameter_data.get("file", False)
-            value = self._format_parameter_value(parameter, value_data, unit, file_flag)
+            value = self._format_parameter_value(
+                parameter, value_data, unit, file_flag, parameter_version
+            )
 
-            description = parameter_descriptions[0].get(parameter)
-            short_description = parameter_descriptions[1].get(parameter)
-            if short_description is None:
-                short_description = description
-            inst_class = parameter_descriptions[2].get(parameter)
+            description = parameter_descriptions.get(parameter).get("description")
+            short_description = (
+                parameter_descriptions.get(parameter).get("short_description") or description
+            )
+            inst_class = parameter_descriptions.get(parameter).get("inst_class")
 
             matching_instrument = parameter_data["instrument"] == telescope_model.name
             if not names.is_design_type(telescope_model.name) and matching_instrument:
@@ -376,8 +413,10 @@ class ReadParameters:
 
             data = []
             for parameter, parameter_data in param_dict.items():
-                description = parameter_descriptions[0].get(parameter)
-                short_description = parameter_descriptions[1].get(parameter, description)
+                description = parameter_descriptions.get(parameter).get("description")
+                short_description = parameter_descriptions.get(parameter).get(
+                    "short_description", description
+                )
                 value_data = parameter_data.get("value")
 
                 if value_data is None:
@@ -386,7 +425,9 @@ class ReadParameters:
                 unit = parameter_data.get("unit") or " "
                 file_flag = parameter_data.get("file", False)
                 parameter_version = parameter_data.get("parameter_version")
-                value = self._format_parameter_value(parameter, value_data, unit, file_flag)
+                value = self._format_parameter_value(
+                    parameter, value_data, unit, file_flag, parameter_version
+                )
 
                 data.append(
                     [
@@ -508,10 +549,12 @@ class ReadParameters:
                 continue
 
             output_filename = output_path / f"{parameter}.md"
-            description = self.get_all_parameter_descriptions(collection=collection)[0].get(
-                parameter,
-                self.get_all_parameter_descriptions(collection="telescopes")[0].get(parameter),
-            )
+
+            parameter_descriptions = self.get_all_parameter_descriptions(collection=collection).get(
+                parameter
+            ) or self.get_all_parameter_descriptions(collection="telescopes").get(parameter)
+
+            description = parameter_descriptions.get("description")
             with output_filename.open("w", encoding="utf-8") as file:
                 # Write header
                 file.write(
@@ -534,14 +577,12 @@ class ReadParameters:
                     file.write(
                         f"| {item['parameter_version']} |"
                         f" {item['model_version']} |"
-                        f"{item['value'].replace('](', '](../')} |\n"
+                        f"{item['value']} |\n"
                     )
 
                 file.write("\n")
                 if comparison_data.get(parameter)[0]["file_flag"]:
-                    file.write(
-                        f"![Parameter plot.]({IMAGE_PATH}/{self.array_element}_{parameter}.png)"
-                    )
+                    file.write(f"![Parameter plot.](/_images/{self.array_element}_{parameter}.png)")
 
     def _write_array_layouts_section(self, file, layouts):
         """Write the array layouts section of the report."""
@@ -556,7 +597,7 @@ class ReadParameters:
             file.write("\n")
             version = self.model_version.replace(".", "-")
             filename = f"OBS-{self.site}_{layout_name}_{version}.png"
-            image_path = f"{IMAGE_PATH}/{filename}"
+            image_path = f"/_images/{filename}"
             file.write(f"![{layout_name} Layout]({image_path})\n\n")
             file.write("\n")
 
@@ -606,7 +647,9 @@ class ReadParameters:
                     f"(#array-trigger-configurations) | {parameter_version} |\n"
                 )
             else:
-                formatted_value = self._format_parameter_value(param_name, value, unit, file_flag)
+                formatted_value = self._format_parameter_value(
+                    param_name, value, unit, file_flag, parameter_version
+                )
                 file.write(f"| {param_name} | {formatted_value} | {parameter_version} |\n")
         file.write("\n")
 
@@ -645,7 +688,7 @@ class ReadParameters:
 
     def get_calibration_data(self, all_parameter_data, array_element):
         """Get calibration data and descriptions for a given array element."""
-        parameter_descriptions = self.get_all_parameter_descriptions(
+        calibration_descriptions = self.get_all_parameter_descriptions(
             collection="calibration_devices"
         )
         # get descriptions of array element positions from the telescope collection
@@ -654,6 +697,10 @@ class ReadParameters:
         class_grouped_data = {}
 
         for parameter in all_parameter_data.keys():
+            parameter_descriptions = calibration_descriptions.get(
+                parameter
+            ) or telescope_descriptions.get(parameter)
+
             parameter_data = all_parameter_data.get(parameter)
             parameter_version = parameter_data.get("parameter_version")
             unit = parameter_data.get("unit") or " "
@@ -663,19 +710,15 @@ class ReadParameters:
                 continue
 
             file_flag = parameter_data.get("file", False)
-            value = self._format_parameter_value(parameter, value_data, unit, file_flag)
-
-            description = parameter_descriptions[0].get(
-                parameter, telescope_descriptions[0].get(parameter)
-            )
-            short_description = (
-                parameter_descriptions[1].get(parameter, telescope_descriptions[1].get(parameter))
-                or description
-            )
-            inst_class = parameter_descriptions[2].get(
-                parameter, telescope_descriptions[2].get(parameter)
+            value = self._format_parameter_value(
+                parameter, value_data, unit, file_flag, parameter_version
             )
 
+            description = parameter_descriptions.get("description")
+            short_description = parameter_descriptions.get("short_description") or description
+
+            inst_class = parameter_descriptions.get("inst_class")
+
             matching_instrument = parameter_data["instrument"] == array_element
             if not names.is_design_type(array_element) and matching_instrument:
                 parameter = f"***{parameter}***"
@@ -764,5 +807,4 @@ class ReadParameters:
         for calibration_device in array_elements:
             self.site = names.get_site_from_array_element_name(calibration_device)
             self.array_element = calibration_device
-            print("cal: ", calibration_device)
             self.produce_model_parameter_reports(collection="calibration_devices")

simtools/runners/corsika_simtel_runner.py

@@ -42,6 +42,7 @@ class CorsikaSimtelRunner:
         keep_seeds=False,
         use_multipipe=False,
         sim_telarray_seeds=None,
+        sequential=False,
     ):
         self._logger = logging.getLogger(__name__)
         self.corsika_config = (
@@ -53,6 +54,7 @@ class CorsikaSimtelRunner:
         self._simtel_path = simtel_path
         self.sim_telarray_seeds = sim_telarray_seeds
         self.label = label
+        self.sequential = "--sequential" if sequential else ""
 
         self.base_corsika_config.set_output_file_and_directory(use_multipipe)
         self.corsika_runner = CorsikaRunner(
@@ -167,7 +169,8 @@ class CorsikaSimtelRunner:
         )
         with open(multipipe_script, "w", encoding="utf-8") as file:
             multipipe_command = Path(self._simtel_path).joinpath(
-                f"sim_telarray/bin/multipipe_corsika -c {multipipe_file} || echo 'Fan-out failed'"
+                f"sim_telarray/bin/multipipe_corsika -c {multipipe_file} {self.sequential} "
+                "|| echo 'Fan-out failed'"
             )
             file.write(f"{multipipe_command}")
 

simtools/runners/runner_services.py

@@ -138,9 +138,9 @@ class RunnerServices:
         file_label = f"_{info_for_file_name['label']}" if info_for_file_name.get("label") else ""
         zenith = self.corsika_config.get_config_parameter("THETAP")[0]
         azimuth = self.corsika_config.azimuth_angle
-        run_dir = self._get_run_number_string(info_for_file_name["run_number"])
+        run_number_string = self._get_run_number_string(info_for_file_name["run_number"])
         return (
-            f"{run_dir}_{info_for_file_name['primary']}_"
+            f"{info_for_file_name['primary']}_{run_number_string}_"
             f"za{round(zenith):02}deg_azm{azimuth:03}deg_"
             f"{info_for_file_name['site']}_{info_for_file_name['array_name']}_"
             f"{info_for_file_name['model_version']}{file_label}"
@@ -189,8 +189,9 @@ class RunnerServices:
         """
         data_suffixes = {
             "output": ".zst",
-            "corsika_output": ".zst",
+            "corsika_output": ".corsika.zst",
             "simtel_output": ".simtel.zst",
+            "event_data": ".reduced_event_data.hdf5",
         }
         run_dir = self._get_run_number_string(run_number)
         data_run_dir = self.directory["data"].joinpath(run_dir)
@@ -261,7 +262,7 @@ class RunnerServices:
         if file_type in ["log", "histogram", "corsika_log"]:
             return self._get_log_file_path(file_type, file_name)
 
-        if file_type in ["output", "corsika_output", "simtel_output"]:
+        if file_type in ["output", "corsika_output", "simtel_output", "event_data"]:
             return self._get_data_file_path(file_type, file_name, run_number)
 
         if file_type in ("sub_log", "sub_script"):

simtools/schemas/model_parameters/dsum_threshold.schema.yml

@@ -1,6 +1,47 @@
 %YAML 1.2
 ---
 title: Schema for dsum_threshold model parameter
+version: 0.2.0
+meta_schema: simpipe-schema
+meta_schema_url: https://raw.githubusercontent.com/gammasim/simtools/main/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml
+meta_schema_version: 0.1.0
+name: dsum_threshold
+description: |-
+  The amplitude level above pedestal sum above which a
+  digital sum leads to a telescope trigger.
+  Note that, like for discriminator/comparator and analog sum, the signal
+  must exceed (\'>\') the threshold here before we declare the telescope
+  triggered. The assigned threshold value would have to be one count lower
+  than in a camera-internal trigger implementation (like MSTx-FlashCam) where
+  reaching (\'>=\') the threshold is enough.
+short_description: Amplitude level above which a digital sum leads to a telescope
+  trigger.
+data:
+  - type: int64
+    unit: count
+    default: 0
+    allowed_range:
+      min: 0
+    condition: default_trigger==DigitalSum
+instrument:
+  class: Camera
+  type:
+    - MSTx-NectarCam
+    - MSTx-FlashCam
+activity:
+  setting:
+    - SetParameterFromExternal
+    - SetTriggerThresholdsFromRateScan
+  validation:
+    - ValidateParameterByExpert
+    - ValidateTriggerPerformance
+source:
+  - Observation execution
+simulation_software:
+  - name: sim_telarray
+...
+---
+title: Schema for dsum_threshold model parameter
 version: 0.1.0
 meta_schema: simpipe-schema
 meta_schema_url: https://raw.githubusercontent.com/gammasim/simtools/main/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml

simtools/schemas/production_configuration_metrics.schema.yml

@@ -29,12 +29,12 @@ definitions:
       description:
         type: string
         description: Description of the error metric.
-      target_error:
+      target_uncertainty:
         $ref: '#/definitions/TargetError'
       energy_range:
         $ref: '#/definitions/EnergyRange'
     required:
-      - target_error
+      - target_uncertainty
       - energy_range
 
   TargetError:

simtools/simtel/simtel_config_writer.py

@@ -54,12 +54,20 @@ class SimtelConfigWriter:
         Layout name.
     label: str
         Instance label. Important for output file naming.
+    simtel_path: str or Path
+        Path to the sim_telarray installation directory.
     """
 
     TAB = " " * 3
 
     def __init__(
-        self, site, model_version, layout_name=None, telescope_model_name=None, label=None
+        self,
+        site,
+        model_version,
+        layout_name=None,
+        telescope_model_name=None,
+        label=None,
+        simtel_path=None,
     ):
         """Initialize SimtelConfigWriter."""
         self._logger = logging.getLogger(__name__)
@@ -70,10 +78,9 @@ class SimtelConfigWriter:
         self._label = label
         self._layout_name = layout_name
         self._telescope_model_name = telescope_model_name
+        self._simtel_path = simtel_path
 
-    def write_telescope_config_file(
-        self, config_file_path, parameters, telescope_name=None, write_dummy_config=False
-    ):
+    def write_telescope_config_file(self, config_file_path, parameters, telescope_name=None):
         """
         Write the sim_telarray config file for a single telescope.
 
@@ -85,8 +92,6 @@ class SimtelConfigWriter:
             Model parameters
         telescope_name: str
             Name of the telescope (use self._telescope_model_name if None)
-        write_dummy_config: bool
-            Flag to write a dummy telescope configuration file.
         """
         self._logger.debug(f"Writing telescope config file {config_file_path}")
 
@@ -97,8 +102,6 @@ class SimtelConfigWriter:
             file.write("#ifdef TELESCOPE\n")
             file.write(f"   echo Configuration for {telescope_name} - TELESCOPE $(TELESCOPE)\n")
             file.write("#endif\n\n")
-            if write_dummy_config:
-                file.write("#define DUMMY_CONFIG 1\n")
 
             for par, value in parameters.items():
                 simtel_name, value = self._convert_model_parameters_to_simtel_format(
@@ -252,6 +255,8 @@ class SimtelConfigWriter:
             file.write(self.TAB + f"echo ModelVersion: {self._model_version}\n")
             file.write(self.TAB + "echo *****************************\n\n")
 
+            self._write_simtools_parameters(file)
+
             self._write_site_parameters(
                 file,
                 site_model.parameters,
@@ -392,6 +397,23 @@ class SimtelConfigWriter:
         header += f"{comment_char}\n"
         file.write(header)
 
+    def _write_simtools_parameters(self, file):
+        """Write simtools-specific parameters."""
+        meta_items = {
+            "simtools_version": simtools.version.__version__,
+            "simtools_model_production_version": self._model_version,
+        }
+        try:
+            build_opts = gen.collect_data_from_file(Path(self._simtel_path) / "build_opts.yml")
+            for key, value in build_opts.items():
+                meta_items[f"simtools_{key}"] = value
+        except (FileNotFoundError, TypeError):
+            pass  # don't expect build_opts.yml to be present on all systems
+
+        file.write(f"{self.TAB}% Simtools parameters\n")
+        for key, value in meta_items.items():
+            file.write(f"{self.TAB}metaparam global set {key} = {value}\n")
+
     def _write_site_parameters(
         self, file, site_parameters, model_path, telescope_model, sim_telarray_seeds=None
     ):
@@ -556,9 +578,9 @@ class SimtelConfigWriter:
             "disc_bins": 10,
             "fadc_sum_bins": 10,
             "fadc_sum_offset": 0,
-            "asum_threshold": 0,
-            "dsum_threshold": 0,
-            "discriminator_threshold": 1,
+            "asum_threshold": 9999,
+            "dsum_threshold": 9999,
+            "discriminator_threshold": 9999,
             "fadc_amplitude": 1.0,
             "discriminator_amplitude": 1.0,
         }
@@ -567,9 +589,7 @@ class SimtelConfigWriter:
             if key in parameters:
                 parameters[key]["value"] = val
 
-        self.write_telescope_config_file(
-            config_file_path, parameters, telescope_name, write_dummy_config=True
-        )
+        self.write_telescope_config_file(config_file_path, parameters, telescope_name)
 
         config_file_directory = Path(config_file_path).parent
         self._write_dummy_mirror_list_files(config_file_directory, telescope_name)