gammasimtools 0.5.1__py3-none-any.whl → 0.6.1__py3-none-any.whl

simtools/simtel/simtel_histograms.py

@@ -1,16 +1,20 @@
 import copy
 import logging
 
-import matplotlib.pyplot as plt
 import numpy as np
+from astropy import units as u
+from ctapipe.io import write_table
 from eventio import EventIOFile, Histograms
 from eventio.search_utils import yield_toplevel_of_type
-from matplotlib.backends.backend_pdf import PdfPages
 
-__all__ = ["BadHistogramFormat", "SimtelHistograms"]
+from simtools import version
+from simtools.io_operations.hdf5_handler import fill_hdf5_table
+from simtools.utils.names import sanitize_name
 
+__all__ = ["InconsistentHistogramFormat", "SimtelHistograms"]
 
-class BadHistogramFormat(Exception):
+
+class InconsistentHistogramFormat(Exception):
     """Exception for bad histogram format."""
 
 
@@ -32,27 +36,17 @@ class SimtelHistograms:
         Initialize SimtelHistograms
         """
         self._logger = logging.getLogger(__name__)
+        if not isinstance(histogram_files, list):
+            histogram_files = [histogram_files]
         self._histogram_files = histogram_files
         self._is_test = test
-        self.combined_hists = None
-
-    def plot_and_save_figures(self, fig_name):
-        """
-        Plot all histograms and save a single pdf file.
-
-        Parameters
-        ----------
-        fig_name: str
-            Name of the output figure file.
-        """
-        self._combine_histogram_files()
-        self._plot_combined_histograms(fig_name)
+        self._list_of_histograms = None
+        self._combined_hists = None
+        self.__meta_dict = None
 
     @property
     def number_of_histograms(self):
         """Returns number of histograms."""
-        if not hasattr(self, "combined_hists"):
-            self._combine_histogram_files()
         return len(self.combined_hists)
 
     def get_histogram_title(self, i_hist):
@@ -69,83 +63,199 @@ class SimtelHistograms:
         str
             Histogram title.
         """
-        if not hasattr(self, "combined_hists"):
-            self._combine_histogram_files()
         return self.combined_hists[i_hist]["title"]
 
-    def _combine_histogram_files(self):
-        """Combine histograms from all files into one single list of histograms."""
-        # Processing and combining histograms from multiple files
-        self.combined_hists = []
-
-        n_files = 0
-        for file in self._histogram_files:
-            count_file = True
-            with EventIOFile(file) as f:
-                for o in yield_toplevel_of_type(f, Histograms):
-                    try:
+    @property
+    def list_of_histograms(self):
+        """
+        Returns a list with the histograms for each file.
+
+        Returns
+        -------
+        list:
+            List of histograms.
+        """
+        if self._list_of_histograms is None:
+            self._list_of_histograms = []
+            for file in self._histogram_files:
+                with EventIOFile(file) as f:
+                    for o in yield_toplevel_of_type(f, Histograms):
                         hists = o.parse()
-                    except Exception:
-                        self._logger.warning(f"Problematic file {file}")
-                        count_file = False
-                        continue
-
-                    if len(self.combined_hists) == 0:
-                        # First file
-                        self.combined_hists = copy.copy(hists)
-
-                    else:
-                        # Remaining files
-                        for hist, this_combined_hist in zip(hists, self.combined_hists):
-                            # Checking consistency of histograms
-                            for key_to_test in [
-                                "lower_x",
-                                "upper_x",
-                                "n_bins_x",
-                                "title",
-                            ]:
-                                if hist[key_to_test] != this_combined_hist[key_to_test]:
-                                    msg = "Trying to add histograms with inconsistent dimensions"
-                                    self._logger.error(msg)
-                                    raise BadHistogramFormat(msg)
-
-                            this_combined_hist["data"] = np.add(
-                                this_combined_hist["data"], hist["data"]
-                            )
-
-                    n_files += int(count_file)
-
-        self._logger.debug(f"End of reading {n_files} files")
-
-    def _plot_combined_histograms(self, fig_name):
-        """
-        Plot all histograms into pdf pages and save the figure as a pdf file.
+                        self._list_of_histograms.append(hists)
+        return self._list_of_histograms
+
+    def _check_consistency(self, first_hist_file, second_hist_file):
+        """
+        Checks whether two histograms have the same format.
+        Raises an error in case they are not consistent.
 
         Parameters
         ----------
-        fig_name: str
-            Name of the output figure file.
+        first_hist_file: dict
+            One histogram from a single file.
+        second_hist_file: dict
+            One histogram from a single file.
+
+        Raises
+        ------
+        InconsistentHistogramFormat:
+            if the format of the histograms have inconsistent dimensions.
+        """
+        for key_to_test in [
+            "lower_x",
+            "upper_x",
+            "n_bins_x",
+            "title",
+        ]:
+            if first_hist_file[key_to_test] != second_hist_file[key_to_test]:
+                msg = "Trying to add histograms with inconsistent dimensions"
+                self._logger.error(msg)
+                raise InconsistentHistogramFormat(msg)
+
+    @property
+    def combined_hists(self):
+        """Add the values of the same type of histogram from the various lists into a single
+        histogram list."""
+        # Processing and combining histograms from multiple files
+        if self._combined_hists is None:
+            self._combined_hists = []
+            for i_hist, hists_one_file in enumerate(self.list_of_histograms):
+                if i_hist == 0:
+                    # First file
+                    self._combined_hists = copy.copy(hists_one_file)
+
+                else:
+                    for hist, this_combined_hist in zip(hists_one_file, self._combined_hists):
+                        self._check_consistency(hist, this_combined_hist)
+
+                        this_combined_hist["data"] = np.add(
+                            this_combined_hist["data"], hist["data"]
+                        )
+
+            self._logger.debug(f"End of reading {len(self.list_of_histograms)} files")
+        return self._combined_hists
+
+    @combined_hists.setter
+    def combined_hists(self, new_combined_hists):
         """
+        Setter for combined_hists.
 
-        pdf_pages = PdfPages(fig_name)
-        for i_hist, histo in enumerate(self.combined_hists):
-            # Test case: processing only 1/10 of the histograms
-            if self._is_test and i_hist % 10 != 0:
-                self._logger.debug(f"Skipping (test=True): {histo['title']}")
-                continue
+        Parameters
+        ----------
+        new_combined_hists:
+            Combined histograms.
+        """
+        self._combined_hists = new_combined_hists
 
-            self._logger.debug(f"Processing: {histo['title']}")
+    def _derive_trigger_rate_histograms(self, livetime):
+        """
+        Calculates the trigger rate histograms, i.e., the ratio in which the events
+        are triggered in each bin of impact distance and log energy for each histogram file for
+        the livetime defined by `livetime`.
+        The livetime gives the amount of time used in a small production to produce the histograms
+        used. It is assumed that the livetime is the same for all the histogram files used and that
+        the radius (x-axis in the histograms) is given in meters.
 
-            fig = plt.figure(figsize=(8, 6))
-            ax = plt.gca()
+        Parameters
+        ----------
+        livetime: astropy.Quantity
+            Time used in the simulation that produced the histograms. E.g., 1*u.h.
 
-            self.plot_one_histogram(i_hist, ax)
+        Returns
+        -------
+        list:
+            List with the trigger rate histograms for each file.
+        """
+        if isinstance(livetime, u.Quantity):
+            livetime = livetime.to(u.s)
+        else:
+            livetime = livetime * u.s
+        events_histogram = {}
+        trigged_events_histogram = {}
+        # Save the appropriate histograms to a dictionary
+        for i_file, hists_one_file in enumerate(self.list_of_histograms):
+            for hist in hists_one_file:
+                if hist["id"] == 1:
+                    events_histogram[i_file] = hist
+
+                elif hist["id"] == 2:
+                    trigged_events_histogram[i_file] = hist
+
+        list_of_trigger_rate_hists = []
+        # Calculate the event rate histograms
+        for i_file, hists_one_file in enumerate(self.list_of_histograms):
+            event_rate_histogram = copy.copy(events_histogram[i_file])
+            area_dict = np.pi * (
+                (events_histogram[i_file]["upper_x"]) ** 2
+                - (events_histogram[i_file]["lower_x"]) ** 2
+            )
 
-            plt.tight_layout()
-            pdf_pages.savefig(fig)
-            plt.close()
+            event_rate_histogram["data"] = (
+                np.zeros_like(trigged_events_histogram[i_file]["data"]) / livetime.unit
+            )
+            bins_with_events = events_histogram[i_file]["data"] != 0
+            event_rate_histogram["data"][bins_with_events] = (
+                trigged_events_histogram[i_file]["data"][bins_with_events]
+                / events_histogram[i_file]["data"][bins_with_events]
+                * area_dict
+                / livetime
+            )
 
-        pdf_pages.close()
+            # Keeping only the necessary information for proceeding with integration
+            keys_to_keep = [
+                "data",
+                "lower_x",
+                "lower_y",
+                "upper_x",
+                "upper_y",
+                "entries",
+                "n_bins_x",
+                "n_bins_y",
+            ]
+            event_rate_histogram = {
+                key: event_rate_histogram[key]
+                for key in keys_to_keep
+                if key in event_rate_histogram
+            }
+
+            list_of_trigger_rate_hists.append(event_rate_histogram)
+        return list_of_trigger_rate_hists
+
+    def _integrate_trigger_rate_histograms(self, hists):
+        """
+        Integrates in energy the trigger rate histogram based on the histogram bin edges.
+
+        Parameters
+        ----------
+        hists: list
+            List with the final trigger rate for each histogram.
+        """
+
+        list_of_integrated_hists = []
+        for _, hist in enumerate(hists):
+            energy_axis = np.logspace(hist["lower_y"], hist["upper_y"], hist["n_bins_y"])
+            radius_axis = np.linspace(hist["lower_x"], hist["upper_x"], hist["n_bins_x"])
+            integrated_hist = np.zeros_like(radius_axis)
+            for i_radius, _ in enumerate(radius_axis):
+                integrated_hist[i_radius] = np.sum(
+                    hist["data"][:-1, i_radius].value * np.diff(energy_axis)
+                )
+
+            list_of_integrated_hists.append(np.sum(integrated_hist) * hist["data"][0, 0].unit)
+        return list_of_integrated_hists
+
+    def trigger_rate_per_histogram(self, livetime):
+        """
+        Estimates the trigger rate for each histogram passed.
+
+        Parameters
+        ----------
+        livetime: astropy.Quantity
+            Time used in the simulation that produced the histograms.
+        """
+        hists = self._derive_trigger_rate_histograms(livetime=livetime)
+        trigger_rates = self._integrate_trigger_rate_histograms(hists)
+        return trigger_rates
 
     def plot_one_histogram(self, i_hist, ax):
         """
@@ -229,3 +339,76 @@ class SimtelHistograms:
             ax.hist(centers, bins=x_bins, weights=hist["data"])
             ax.set_xlim(xlim)
         return
+
+    @property
+    def _meta_dict(self):
+        """
+        Define the meta dictionary for exporting the histograms.
+
+        Returns
+        -------
+        dict
+            Meta dictionary for the hdf5 files with the histograms.
+        """
+
+        if self.__meta_dict is None:
+            self.__meta_dict = {
+                "simtools_version": version.__version__,
+                "note": "Only lower bin edges are given.",
+            }
+        return self.__meta_dict
+
+    def export_histograms(self, hdf5_file_name, overwrite=False):
+        """
+        Export the histograms to hdf5 files.
+
+        Parameters
+        ----------
+        hdf5_file_name: str
+            Name of the file to be saved with the hdf5 tables.
+        overwrite: bool
+            If True overwrites the histograms already saved in the hdf5 file.
+        """
+        for histogram in self.combined_hists:
+            x_bin_edges_list = np.linspace(
+                histogram["lower_x"],
+                histogram["upper_x"],
+                num=histogram["n_bins_x"] + 1,
+                endpoint=True,
+            )
+            if histogram["n_bins_y"] > 0:
+                y_bin_edges_list = np.linspace(
+                    histogram["lower_y"],
+                    histogram["upper_y"],
+                    num=histogram["n_bins_y"] + 1,
+                    endpoint=True,
+                )
+            else:
+                y_bin_edges_list = None
+
+            self._meta_dict["Title"] = sanitize_name(histogram["title"])
+
+            table = fill_hdf5_table(
+                hist=histogram["data"],
+                x_bin_edges=x_bin_edges_list,
+                y_bin_edges=y_bin_edges_list,
+                x_label=None,
+                y_label=None,
+                meta_data=self._meta_dict,
+            )
+
+            self._logger.debug(
+                f"Writing histogram with name {self._meta_dict['Title']} to " f"{hdf5_file_name}."
+            )
+            # overwrite takes precedence over append
+            if overwrite is True:
+                append = False
+            else:
+                append = True
+            write_table(
+                table,
+                hdf5_file_name,
+                f"/{self._meta_dict['Title']}",
+                append=append,
+                overwrite=overwrite,
+            )

simtools/simtel/simtel_runner.py

@@ -45,7 +45,7 @@ class SimtelRunner:
         self._script_dir = None
         self._script_file = None
 
-        self.RUNS_PER_SET = 1
+        self.runs_per_set = 1
 
     def __repr__(self):
         return f"SimtelRunner(label={self.label})\n"
@@ -140,8 +140,8 @@ class SimtelRunner:
                     file.write(f"{line}\n")
                 file.write("# End of extras\n\n")
 
-            N = 1 if test else self.RUNS_PER_SET
-            for _ in range(N):
+            n = 1 if test else self.runs_per_set
+            for _ in range(n):
                 file.write(f"{command}\n\n")
 
             # Printing out runtime
@@ -178,10 +178,10 @@ class SimtelRunner:
             self._logger.info(f"Running (test) with command: {command}")
             self._run_simtel_and_check_output(command)
         else:
-            self._logger.debug(f"Running ({self.RUNS_PER_SET}x) with command: {command}")
+            self._logger.debug(f"Running ({self.runs_per_set}x) with command: {command}")
             self._run_simtel_and_check_output(command)
 
-            for _ in range(self.RUNS_PER_SET - 1):
+            for _ in range(self.runs_per_set - 1):
                 self._run_simtel_and_check_output(command)
 
         self._check_run_result(run_number=run_number)
@@ -236,8 +236,9 @@ class SimtelRunner:
         return False
 
     @staticmethod
-    def _config_option(par, value=None):
+    def _config_option(par, value=None, weak_option=False):
         """Util function for building sim_telarray command."""
-        c = f" -C {par}"
+        option_syntax = "-W" if weak_option else "-C"
+        c = f" {option_syntax} {par}"
         c += f"={value}" if value is not None else ""
         return c

simtools/simtel/simtel_runner_array.py

@@ -2,7 +2,7 @@ import logging
 from pathlib import Path
 
 import simtools.utils.general as gen
-from simtools import io_handler
+from simtools.io_operations import io_handler
 from simtools.simtel.simtel_runner import InvalidOutputFile, SimtelRunner
 
 __all__ = ["SimtelRunnerArray"]
@@ -68,11 +68,11 @@ class SimtelRunnerArray(SimtelRunner):
         self._base_directory = self.io_handler.get_output_directory(self.label, "array-simulator")
 
         # Loading config_data
-        _config_data_in = gen.collect_data_from_yaml_or_dict(config_file, config_data)
+        _config_data_in = gen.collect_data_from_file_or_dict(config_file, config_data)
         _parameter_file = self.io_handler.get_input_data_file(
             "parameters", "simtel-runner-array_parameters.yml"
         )
-        _parameters = gen.collect_data_from_yaml_or_dict(_parameter_file, None)
+        _parameters = gen.collect_data_from_file_or_dict(_parameter_file, None)
         self.config = gen.validate_config_data(_config_data_in, _parameters)
 
         self._load_simtel_data_directories()
@@ -105,7 +105,7 @@ class SimtelRunnerArray(SimtelRunner):
 
     def get_info_for_file_name(self, run_number):
         """
-        Get a dirctionary with the info necessary for building the sim_telarray file names.
+        Get a dictionary with the info necessary for building the sim_telarray file names.
 
         Returns
         -------
@@ -226,7 +226,7 @@ class SimtelRunnerArray(SimtelRunner):
         _resources = {}
 
         _resources["runtime"] = None
-        with open(sub_log_file, "r") as file:
+        with open(sub_log_file, "r", encoding="utf-8") as file:
             for line in reversed(list(file)):
                 if "RUNTIME" in line:
                     _resources["runtime"] = int(line.split()[1])
@@ -274,7 +274,7 @@ class SimtelRunnerArray(SimtelRunner):
         command += super()._config_option("power_law", "2.5")
         command += super()._config_option("histogram_file", histogram_file)
         command += super()._config_option("output_file", output_file)
-        command += super()._config_option("random_state", "auto")
+        command += super()._config_option("random_state", "none")
         command += super()._config_option("show", "all")
         command += f" {kwargs['input_file']}"
         command += f" > {self._log_file} 2>&1 || exit"
@@ -290,5 +290,4 @@ class SimtelRunnerArray(SimtelRunner):
             msg = "sim_telarray output file does not exist."
             self._logger.error(msg)
             raise InvalidOutputFile(msg)
-
-        self._logger.debug("Everything looks fine with the sim_telarray output file.")
+        self._logger.debug(f"simtel_array output file {output_file} exists.")

simtools/simtel/simtel_runner_camera_efficiency.py

@@ -1,4 +1,5 @@
 import logging
+from pathlib import Path
 
 from simtools.simtel.simtel_runner import SimtelRunner
 
@@ -32,6 +33,7 @@ class SimtelRunnerCameraEfficiency(SimtelRunner):
         file_simtel=None,
         file_log=None,
         zenith_angle=None,
+        nsb_spectrum=None,
     ):
         """
         Initialize SimtelRunner.
@@ -47,6 +49,20 @@ class SimtelRunnerCameraEfficiency(SimtelRunner):
         self._file_simtel = file_simtel
         self._file_log = file_log
         self.zenith_angle = zenith_angle
+        self.nsb_spectrum = nsb_spectrum
+
+    @property
+    def nsb_spectrum(self):
+        """nsb_spectrum property"""
+        return self._nsb_spectrum
+
+    @nsb_spectrum.setter
+    def nsb_spectrum(self, nsb_spectrum):
+        """Setter for nsb_spectrum"""
+        if nsb_spectrum is not None:
+            self._nsb_spectrum = self._validate_or_fix_nsb_spectrum_file_format(nsb_spectrum)
+        else:
+            self._nsb_spectrum = None
 
     def _shall_run(self, **kwargs):  # pylint: disable=unused-argument; applies only to this line
         """Tells if simulations should be run again based on the existence of output files."""
@@ -83,7 +99,7 @@ class SimtelRunnerCameraEfficiency(SimtelRunner):
         # Processing camera filter
         # A special case is testeff does not support 2D distributions
         camera_filter_file = self._telescope_model.get_parameter_value("camera_filter")
-        if self._telescope_model.is_file_2D("camera_filter"):
+        if self._telescope_model.is_file_2d("camera_filter"):
             camera_filter_file = self._get_one_dim_distribution(
                 "camera_filter", "camera_filter_incidence_angle"
             )
@@ -93,7 +109,7 @@ class SimtelRunnerCameraEfficiency(SimtelRunner):
         mirror_reflectivity = self._telescope_model.get_parameter_value("mirror_reflectivity")
         if mirror_class == 2:
             mirror_reflectivity_secondary = mirror_reflectivity
-        if self._telescope_model.is_file_2D("mirror_reflectivity"):
+        if self._telescope_model.is_file_2d("mirror_reflectivity"):
             mirror_reflectivity = self._get_one_dim_distribution(
                 "mirror_reflectivity", "primary_mirror_incidence_angle"
             )
@@ -102,6 +118,8 @@ class SimtelRunnerCameraEfficiency(SimtelRunner):
             )
 
         command = str(self._simtel_source_path.joinpath("sim_telarray/bin/testeff"))
+        if self.nsb_spectrum is not None:
+            command += f" -fnsb {self.nsb_spectrum}"
         command += " -nm -nsb-extra"
         command += f" -alt {self._telescope_model.get_parameter_value('altitude')}"
         command += f" -fatm {self._telescope_model.get_parameter_value('atmospheric_transmission')}"
@@ -190,3 +208,31 @@ class SimtelRunnerCameraEfficiency(SimtelRunner):
         )
 
         return one_dim_file
+
+    def _validate_or_fix_nsb_spectrum_file_format(self, nsb_spectrum_file):
+        """
+        Validate or fix the nsb spectrum file format.
+        The nsb spectrum file format required by sim_telarray has three columns:
+            wavelength (nm), ignored, NSB flux [1e9 * ph/m2/s/sr/nm],
+        where the second column is ignored by sim_telarray and the third is used for the NSB flux.
+        This function makes sure the file has at least three columns,
+        by copying the second column to the third.
+        """
+
+        validated_nsb_spectrum_file = (
+            self._telescope_model.get_config_directory() / Path(nsb_spectrum_file).name
+        )
+        with open(nsb_spectrum_file, "r", encoding="utf-8") as file:
+            lines = file.readlines()
+        with open(validated_nsb_spectrum_file, "w", encoding="utf-8") as file:
+            for line in lines:
+                if line.startswith("#"):
+                    file.write(line)
+                    continue
+                split_line = line.split()
+                if len(split_line) == 2:
+                    split_line.append(split_line[1])
+                    file.write(f"{split_line[0]} {split_line[1]} {split_line[2]}\n")
+                else:
+                    file.write(line)
+        return validated_nsb_spectrum_file