gammasimtools 0.18.0__py3-none-any.whl → 0.20.0__py3-none-any.whl

simtools/simtel/simtel_io_event_writer.py

@@ -20,6 +20,7 @@ from simtools.corsika.primary_particle import PrimaryParticle
 from simtools.simtel.simtel_io_file_info import get_corsika_run_header
 from simtools.simtel.simtel_io_metadata import (
     get_sim_telarray_telescope_id_to_telescope_name_mapping,
+    read_sim_telarray_metadata,
 )
 from simtools.utils.geometry import calculate_circular_mean
 from simtools.utils.names import get_common_identifier_from_array_element_name
@@ -36,8 +37,8 @@ class TableSchemas:
         "simulated_energy": (np.float64, u.TeV),
         "x_core": (np.float64, u.m),
         "y_core": (np.float64, u.m),
-        "shower_azimuth": (np.float64, u.rad),
-        "shower_altitude": (np.float64, u.rad),
+        "shower_azimuth": (np.float64, u.deg),
+        "shower_altitude": (np.float64, u.deg),
         "area_weight": (np.float64, None),
     }
 
@@ -45,8 +46,8 @@ class TableSchemas:
         "shower_id": (np.uint32, None),
         "event_id": (np.uint32, None),
         "file_id": (np.uint32, None),
-        "array_altitude": (np.float64, u.rad),
-        "array_azimuth": (np.float64, u.rad),
+        "array_altitude": (np.float64, u.deg),
+        "array_azimuth": (np.float64, u.deg),
         "telescope_list": (str, None),  # Store as comma-separated string
         "telescope_list_common_id": (str, None),  # Store as comma-separated string
     }
@@ -175,7 +176,7 @@ class SimtelIOEventDataWriter:
                 "core_scatter_max": run_info["core_range"][1],
                 "zenith": 90.0 - np.degrees(run_info["direction"][1]),
                 "azimuth": np.degrees(run_info["direction"][0]),
-                "nsb_level": self._get_preliminary_nsb_level(str(file)),
+                "nsb_level": self.get_nsb_level_from_sim_telarray_metadata(file),
             }
         )
 
@@ -196,8 +197,8 @@ class SimtelIOEventDataWriter:
                 "simulated_energy": shower["energy"],
                 "x_core": None,  # filled in _process_mc_event
                 "y_core": None,  # filled in _process_mc_event
-                "shower_azimuth": shower["azimuth"],
-                "shower_altitude": shower["altitude"],
+                "shower_azimuth": np.degrees(shower["azimuth"]),
+                "shower_altitude": np.degrees(shower["altitude"]),
                 "area_weight": None,  # filled in _process_mc_event
             }
             for _ in range(self.n_use)
@@ -247,8 +248,8 @@ class SimtelIOEventDataWriter:
                 tracking_position = obj.parse()
                 tracking_positions.append(
                     {
-                        "altitude": tracking_position["altitude_raw"],
-                        "azimuth": tracking_position["azimuth_raw"],
+                        "altitude": np.degrees(tracking_position["altitude_raw"]),
+                        "azimuth": np.degrees(tracking_position["azimuth_raw"]),
                     }
                 )
 
@@ -271,7 +272,7 @@ class SimtelIOEventDataWriter:
                 "event_id": event_id,
                 "file_id": file_id,
                 "array_altitude": float(np.mean(altitudes)),
-                "array_azimuth": float(calculate_circular_mean(azimuths)),
+                "array_azimuth": float(np.degrees(calculate_circular_mean(np.deg2rad(azimuths)))),
                 "telescope_list": ",".join(map(str, telescopes)),
                 "telescope_list_common_id": ",".join(
                     [
@@ -300,14 +301,33 @@ class SimtelIOEventDataWriter:
             self.telescope_id_to_name.get(tel_id, f"Unknown_{tel_id}") for tel_id in telescope_ids
         ]
 
-    def _get_preliminary_nsb_level(self, file):
+    def get_nsb_level_from_sim_telarray_metadata(self, file):
         """
-        Return preliminary NSB level from file name.
+        Return NSB level from sim_telarray metadata.
 
-        Hardwired values are used for "dark", "half", and "full" NSB levels
-        (actual values are made up for this example). Will be replaced with
-        reading of sim_telarray metadata entry for NSB level (to be implemented,
-        see issue #1572).
+        Falls back to preliminary NSB level if not found.
+
+        Parameters
+        ----------
+        file : Path
+            Path to the sim_telarray file.
+
+        Returns
+        -------
+        float
+            NSB level.
+        """
+        metadata, _ = read_sim_telarray_metadata(file)
+        nsb_integrated_flux = metadata.get("nsb_integrated_flux")
+        return nsb_integrated_flux or self._get_nsb_level_from_file_name(str(file))
+
+    def _get_nsb_level_from_file_name(self, file):
+        """
+        Return NSB level from file name.
+
+        Hardwired values are used for "dark", "half", and "full" NSB levels.
+        Allows to read legacy sim_telarray files without 'nsb_integrated_flux'
+        metadata field.
 
         Parameters
         ----------
@@ -319,15 +339,15 @@ class SimtelIOEventDataWriter:
         float
             NSB level extracted from file name.
         """
-        nsb_levels = {"dark": 1.0, "half": 2.0, "full": 5.0}
+        nsb_levels = {"dark": 0.24, "half": 0.835, "full": 1.2}
 
         for key, value in nsb_levels.items():
             try:
                 if key in file.lower():
-                    self._logger.warning(f"NSB level set to hardwired value of {value}")
+                    self._logger.warning(f"NSB level set to hardwired value of {value} for {file}")
                     return value
             except AttributeError as exc:
                 raise AttributeError("Invalid file name.") from exc
 
-        self._logger.warning("No NSB level found in file name, defaulting to 1.0")
-        return 1.0
+        self._logger.warning(f"No NSB level found in {file}, defaulting to None")
+        return None

simtools/simtel/simtel_io_metadata.py

@@ -78,7 +78,7 @@ def _decode_dictionary(meta, encoding="utf8"):
         return {k.decode(encoding, errors="ignore"): v.decode(encoding) for k, v in meta.items()}
     except UnicodeDecodeError as e:
         _logger.warning(
-            f"Failed to decode metadata with encoding {encoding}: {e}. "
+            f"Unable to decode metadata with encoding {encoding}: {e}. "
             "Falling back to 'utf-8' with errors='ignore'."
         )
         return {safe_decode(k, encoding): safe_decode(v, encoding) for k, v in meta.items()}

simtools/simtel/simtel_table_reader.py

@@ -6,9 +6,10 @@ import re
 from pathlib import Path
 
 import astropy.units as u
+import numpy as np
 from astropy.table import Table
 
-from simtools.utils import general as gen
+from simtools.io import ascii_handler
 
 logger = logging.getLogger(__name__)
 
@@ -132,11 +133,6 @@ def _data_columns_camera_filter():
     )
 
 
-def _data_columns_lightguide_efficiency_vs_wavelength():
-    """Column description for parameter lightguide_efficiency_vs_wavelength."""
-    return _data_columns_lightguide_efficiency_vs_incidence_angle()
-
-
 def _data_columns_lightguide_efficiency_vs_incidence_angle():
     """Column description for (parameter lightguide_efficiency_vs_incidence_angle."""
     return (
@@ -201,6 +197,16 @@ def _data_columns_mirror_reflectivity(n_columns, n_dim):
     return _columns, "Mirror reflectivity"
 
 
+def _data_columns_secondary_mirror_reflectivity():
+    """Column description for secondary mirror reflectivity."""
+    columns = [
+        {"name": "wavelength", "description": "Wavelength", "unit": "nm"},
+        {"name": "reflectivity", "description": "Reflectivity", "unit": None},
+    ]
+
+    return columns, "Secondary mirror reflectivity vs wavelength"
+
+
 def _data_columns_pulse_shape(n_columns):
     """Column description for parameters discriminator_pulse_shape, fadc_pulse_shape."""
     _columns = [
@@ -223,8 +229,8 @@ def _data_columns_pulse_shape(n_columns):
     return _columns, "Pulse shape"
 
 
-def _data_columns_nsb_reference_spectrum():
-    """Column description for parameter nsb_reference_spectrum."""
+def _data_columns_nsb_spectrum():
+    """Column description for parameters describing the nsb spectrum."""
     return (
         [
             {"name": "wavelength", "description": "Wavelength", "unit": "nm"},
@@ -234,7 +240,7 @@ def _data_columns_nsb_reference_spectrum():
                 "unit": "1.e9 / (nm s m^2 sr)",
             },
         ],
-        "NSB reference spectrum",
+        "NSB spectrum",
     )
 
 
@@ -259,6 +265,8 @@ def read_simtel_table(parameter_name, file_path):
 
     if parameter_name == "atmospheric_transmission":
         return _read_simtel_data_for_atmospheric_transmission(file_path)
+    if parameter_name == "lightguide_efficiency_vs_wavelength":
+        return _read_simtel_data_for_lightguide_efficiency(file_path)
 
     rows, meta_from_simtel, n_columns, n_dim = _read_simtel_data(file_path)
     columns_info, description = _data_columns(parameter_name, n_columns, n_dim)
@@ -291,6 +299,17 @@ def _adjust_columns_length(rows, n_columns):
     return [row[:n_columns] + [0.0] * max(0, n_columns - len(row)) for row in rows]
 
 
+def _process_line_parts(parts):
+    """Convert parts to floats, skipping non-float entries."""
+    row = []
+    for p in parts:
+        try:
+            row.append(float(p))
+        except ValueError:
+            logger.debug(f"Skipping non-float part: {p}")
+    return row
+
+
 def _read_simtel_data(file_path):
     """
     Read data, comments, and (if available) axis definition from sim_telarray table.
@@ -311,7 +330,7 @@ def _read_simtel_data(file_path):
     n_dim_axis = None
     r_pol_axis = None
 
-    lines = gen.read_file_encoded_in_utf_or_latin(file_path)
+    lines = ascii_handler.read_file_encoded_in_utf_or_latin(file_path)
 
     for line in lines:
         stripped = line.strip()
@@ -326,12 +345,75 @@ def _read_simtel_data(file_path):
         elif stripped:  # Data
             data_lines.append(stripped.split("%%%")[0].split("#")[0].strip())  # Remove comments
 
-    rows = [[float(part) for part in line.split()] for line in data_lines]
+    rows = [_process_line_parts(line.split()) for line in data_lines]
     n_columns = max(len(row) for row in rows) if rows else 0
 
     return rows, "\n".join(meta_lines), n_columns, n_dim_axis
 
 
+def _read_simtel_data_for_lightguide_efficiency(file_path):
+    """
+    Read angular efficiency data and return a table with columns: angle, wavelength, efficiency.
+
+    Parameters
+    ----------
+    file_path : str or Path
+
+    Returns
+    -------
+    astropy.table.Table
+    """
+    wavelengths = []
+    data = []
+    meta_lines = []
+
+    lines = ascii_handler.read_file_encoded_in_utf_or_latin(file_path)
+
+    def extract_wavelengths_from_header(line):
+        match = re.search(r"orig\.:\s*(.*)", line)
+        return [float(wl.replace("nm", "")) for wl in match.group(1).split()]
+
+    for line in lines:
+        line = line.strip()
+
+        if not line:
+            continue
+
+        if line.startswith("#"):
+            meta_lines.append(line.lstrip("#").strip())
+            if "orig.:" in line:
+                wavelengths = extract_wavelengths_from_header(line)
+            continue
+
+        parts = line.split()
+        try:
+            theta = float(parts[0])
+            eff_values = list(map(float, parts[-len(wavelengths) :]))
+            data.extend((theta, wl, eff) for wl, eff in zip(wavelengths, eff_values))
+        except (ValueError, IndexError):
+            logger.debug(f"Skipping malformed line: {line}")
+            continue
+
+    if not data or not wavelengths:
+        raise ValueError("No valid data or wavelengths found in file")
+
+    table = Table(rows=data, names=["angle", "wavelength", "efficiency"])
+    table["angle"].unit = u.deg
+    table["wavelength"].unit = u.nm
+    table["efficiency"].unit = u.dimensionless_unscaled
+
+    table.meta.update(
+        {
+            "Name": "angular_efficiency",
+            "File": str(file_path),
+            "Description": "Angular efficiency vs wavelength",
+            "Context_from_sim_telarray": "\n".join(meta_lines),
+        }
+    )
+
+    return table
+
+
 def _read_simtel_data_for_atmospheric_transmission(file_path):
     """
     Read data and comments from sim_telarray table for atmospheric_transmission.
@@ -346,7 +428,7 @@ def _read_simtel_data_for_atmospheric_transmission(file_path):
     astropy table
         Table with atmospheric transmission.
     """
-    lines = lines = gen.read_file_encoded_in_utf_or_latin(file_path)
+    lines = ascii_handler.read_file_encoded_in_utf_or_latin(file_path)
 
     observatory_level, height_bins = _read_header_line_for_atmospheric_transmission(
         lines, file_path
@@ -366,7 +448,7 @@ def _read_simtel_data_for_atmospheric_transmission(file_path):
             wl = float(parts[0])
             for i, height in enumerate(height_bins):
                 extinction_value = float(parts[i + 1])
-                if extinction_value == 99999.0:
+                if np.isclose(extinction_value, 99999.0):
                     continue
                 wavelengths.append(wl)
                 heights.append(height)

simtools/simtel/simulator_array.py

@@ -2,7 +2,7 @@
 
 import logging
 
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 from simtools.runners.simtel_runner import InvalidOutputFileError, SimtelRunner
 from simtools.utils.general import clear_default_sim_telarray_cfg_directories
 
@@ -68,6 +68,73 @@ class SimulatorArray(SimtelRunner):
         str
             Command to run sim_telarray.
         """
+        command = self._common_run_command(run_number, weak_pointing)
+
+        command += f" {input_file}"
+        command += f" | gzip > {self._log_file} 2>&1 || exit"
+        command += super().get_config_option(
+            "power_law",
+            SimulatorArray.get_power_law_for_sim_telarray_histograms(
+                self.corsika_config.primary_particle
+            ),
+        )
+
+        return clear_default_sim_telarray_cfg_directories(command)
+
+    def make_run_command_for_calibration_simulations(
+        self, run_number=None, input_file=None, calibration_runner_args=None
+    ):
+        """
+        Build and return the command to run sim_telarray for calibration simulations.
+
+        Parameters
+        ----------
+        calibration_runner_args: dict
+            Dictionary with calibration runner arguments.
+        input_file: str
+            Full path of the input CORSIKA file
+        run_number: int (optional)
+            run number
+
+        Returns
+        -------
+        str
+            Command to run sim_telarray for pedestal simulations.
+        """
+        command = self._common_run_command(run_number)
+
+        command += super().get_config_option(
+            "Altitude",
+            self.corsika_config.array_model.site_model.get_parameter_value_with_unit(
+                "reference_point_altitude"
+            ).to_value("m"),
+        )
+        command += super().get_config_option(
+            "nsb_scaling_factor", calibration_runner_args["nsb_scaling_factor"]
+        )
+        if calibration_runner_args.get("stars"):
+            command += super().get_config_option("stars", calibration_runner_args["stars"])
+
+        if calibration_runner_args.get("run_mode") in ("pedestals", "nsb_only_pedestals"):
+            command += super().get_config_option(
+                "pedestal_events", calibration_runner_args["number_of_events"]
+            )
+        if calibration_runner_args.get("run_mode") == "nsb_only_pedestals":
+            command += self._nsb_only_pedestals_command()
+        if calibration_runner_args.get("run_mode") == "dark_pedestals":
+            command += super().get_config_option(
+                "dark_events", calibration_runner_args["number_of_events"]
+            )
+        if calibration_runner_args.get("run_mode") == "flasher":
+            command += self._flasher_command(calibration_runner_args)
+
+        command += f" {input_file}"
+        command += f" | gzip > {self._log_file} 2>&1 || exit"
+
+        return clear_default_sim_telarray_cfg_directories(command)
+
+    def _common_run_command(self, run_number, weak_pointing=None):
+        """Build generic run command for sim_telarray."""
         config_dir = self.corsika_config.array_model.get_config_directory()
         self._log_file = self.get_file_name(file_type="log", run_number=run_number)
         histogram_file = self.get_file_name(file_type="histogram", run_number=run_number)
@@ -83,12 +150,6 @@ class SimulatorArray(SimtelRunner):
         command += super().get_config_option(
             "telescope_phi", self.corsika_config.azimuth_angle, weak_pointing
         )
-        command += super().get_config_option(
-            "power_law",
-            SimulatorArray.get_power_law_for_sim_telarray_histograms(
-                self.corsika_config.primary_particle
-            ),
-        )
         command += super().get_config_option("histogram_file", histogram_file)
         command += super().get_config_option("random_state", "none")
         if self.sim_telarray_seeds and self.sim_telarray_seeds.get("random_instrument_instances"):
@@ -100,10 +161,77 @@ class SimulatorArray(SimtelRunner):
             command += super().get_config_option("random_seed", self.sim_telarray_seeds["seed"])
         command += super().get_config_option("show", "all")
         command += super().get_config_option("output_file", output_file)
-        command += f" {input_file}"
-        command += f" | gzip > {self._log_file} 2>&1 || exit"
 
-        return clear_default_sim_telarray_cfg_directories(command)
+        return command
+
+    def _flasher_command(self, calibration_runner_args):
+        """
+        Generate the command to run sim_telarray for flasher simulations.
+
+        Parameters
+        ----------
+        calibration_runner_args: dict
+            Dictionary with calibration runner arguments.
+
+        Returns
+        -------
+        str
+            Command to run sim_telarray for flasher simulations.
+        """
+        command = super().get_config_option(
+            "laser_events", calibration_runner_args["number_of_events"]
+        )
+        command += super().get_config_option(
+            "laser_photons", calibration_runner_args["flasher_photons"]
+        )
+        command += super().get_config_option(
+            "laser_var_photons", calibration_runner_args["flasher_var_photons"]
+        )
+        command += super().get_config_option(
+            "laser_pulse_exptime", calibration_runner_args["flasher_exp_time"]
+        )
+        command += super().get_config_option(
+            "laser_pulse_sigtime", calibration_runner_args["flasher_sig_time"]
+        )
+        command += super().get_config_option("laser_external_trigger", 1)
+
+        return command
+
+    def _nsb_only_pedestals_command(self):
+        """
+        Generate the command to run sim_telarray for nsb-only pedestal simulations.
+
+        Returns
+        -------
+        str
+            Command to run sim_telarray.
+        """
+        null_values = [
+            "fadc_noise",
+            "fadc_lg_noise",
+            "qe_variation",
+            "gain_variation",
+            "fadc_var_pedestal",
+            "fadc_err_pedestal",
+            "fadc_sysvar_pedestal",
+            "fadc_dev_pedestal",
+        ]
+        null_command_parts = []
+        for param in null_values:
+            null_command_parts.append(super().get_config_option(param, 0.0))
+        command = " ".join(null_command_parts)
+
+        one_values = [
+            "fadc_lg_var_pedestal",
+            "fadc_lg_err_pedestal",
+            "fadc_lg_dev_pedestal",
+            "fadc_lg_sysvar_pedestal",
+        ]
+        one_command_parts = []
+        for param in one_values:
+            one_command_parts.append(super().get_config_option(param, -1.0))
+        command += " " + " ".join(one_command_parts)
+        return command
 
     def _check_run_result(self, run_number=None):
         """

simtools/simtel/simulator_camera_efficiency.py

@@ -3,7 +3,7 @@
 import logging
 from pathlib import Path
 
-from simtools.model.model_parameter import InvalidModelParameterError
+from simtools.io import ascii_handler
 from simtools.runners.simtel_runner import SimtelRunner
 from simtools.utils import general
 
@@ -82,41 +82,27 @@ class SimulatorCameraEfficiency(SimtelRunner):
         """Prepare the command used to run testeff."""
         self._logger.debug("Preparing the command to run testeff")
 
-        # Processing camera pixel features
         pixel_shape = self._telescope_model.camera.get_pixel_shape()
         pixel_shape_cmd = "-hpix" if pixel_shape in [1, 3] else "-spix"
         pixel_diameter = self._telescope_model.camera.get_pixel_diameter()
 
-        # Processing focal length
         focal_length = self._telescope_model.get_telescope_effective_focal_length("m", True)
 
-        # Processing mirror class
-        mirror_class = 1
-        try:
-            mirror_class = self._telescope_model.get_parameter_value("mirror_class")
-        except InvalidModelParameterError:
-            pass
-
-        # Processing camera transmission
-        camera_transmission = 1
-        try:
-            camera_transmission = self._telescope_model.get_parameter_value("camera_transmission")
-        except KeyError:
-            pass
-
-        # Processing camera filter
-        # A special case is testeff does not support 2D distributions
+        mirror_class = self._telescope_model.get_parameter_value("mirror_class")
+        curvature_radius = self._get_curvature_radius(mirror_class)
+        camera_transmission = self._telescope_model.get_parameter_value("camera_transmission")
+
         camera_filter_file = self._telescope_model.get_parameter_value("camera_filter")
+        # testeff does not support 2D distributions
         if self._telescope_model.is_file_2d("camera_filter"):
             camera_filter_file = self._get_one_dim_distribution(
                 "camera_filter", "camera_filter_incidence_angle"
             )
 
-        # Processing mirror reflectivity
-        # A special case is testeff does not support 2D distributions
         mirror_reflectivity = self._telescope_model.get_parameter_value("mirror_reflectivity")
         if mirror_class == 2:
             mirror_reflectivity_secondary = mirror_reflectivity
+        # testeff does not support 2D distributions
         if self._telescope_model.is_file_2d("mirror_reflectivity"):
             mirror_reflectivity = self._get_one_dim_distribution(
                 "mirror_reflectivity", "primary_mirror_incidence_angle"
@@ -136,6 +122,7 @@ class SimulatorCameraEfficiency(SimtelRunner):
         command += f" -alt {self._site_model.get_parameter_value('corsika_observation_level')}"
         command += f" -fatm {self._site_model.get_parameter_value('atmospheric_transmission')}"
         command += f" -flen {focal_length}"
+        command += f" -fcur {curvature_radius:.3f}"
         command += f" {pixel_shape_cmd} {pixel_diameter}"
         if mirror_class == 0:
             command += f" -fmir {self._telescope_model.get_parameter_value('mirror_list')}"
@@ -227,7 +214,8 @@ class SimulatorCameraEfficiency(SimtelRunner):
             two_dim_distribution, incidence_angle_distribution
         )
         new_file_name = (
-            f"weighted_average_1D_{self._telescope_model.get_parameter_value(two_dim_parameter)}"
+            f"weighted_average_1D_{weighting_distribution_parameter}"
+            f"_{self._telescope_model.get_parameter_value(two_dim_parameter)}"
         )
         return self._telescope_model.export_table_to_model_directory(
             new_file_name, distribution_to_export
@@ -257,7 +245,7 @@ class SimulatorCameraEfficiency(SimtelRunner):
             self._telescope_model.config_file_directory / Path(nsb_spectrum_file).name
         )
 
-        lines = general.read_file_encoded_in_utf_or_latin(nsb_spectrum_file)
+        lines = ascii_handler.read_file_encoded_in_utf_or_latin(nsb_spectrum_file)
 
         with open(validated_nsb_spectrum_file, "w", encoding="utf-8") as file:
             for line in lines:
@@ -271,3 +259,24 @@ class SimulatorCameraEfficiency(SimtelRunner):
                 else:
                     file.write(line)
         return validated_nsb_spectrum_file
+
+    def _get_curvature_radius(self, mirror_class=1):
+        """Get radius of curvature of dish."""
+        if mirror_class == 2:
+            return (
+                self._telescope_model.get_parameter_value_with_unit("primary_mirror_diameter")
+                .to("m")
+                .value
+            )
+
+        if self._telescope_model.get_parameter_value("parabolic_dish"):
+            return (
+                2.0
+                * self._telescope_model.get_parameter_value_with_unit("dish_shape_length")
+                .to("m")
+                .value
+            )
+
+        return (
+            self._telescope_model.get_parameter_value_with_unit("dish_shape_length").to("m").value
+        )