gammasimtools 0.5.1__py3-none-any.whl → 0.6.1__py3-none-any.whl

simtools/model/model_utils.py

@@ -1,14 +1,11 @@
 #!/usr/bin/python3
 
-import logging
 import math
 
 from simtools.utils import names
 
 __all__ = [
     "compute_telescope_transmission",
-    "get_telescope_class",
-    "get_camera_name",
     "is_two_mirror_telescope",
     "split_simtel_parameter",
 ]
@@ -62,71 +59,6 @@ def compute_telescope_transmission(pars, off_axis):
     return pars[0] / (1.0 + pars[2] * t ** pars[4])
 
 
-def get_camera_name(telescope_model_name):
-    """
-    Get camera name from the telescope name.
-
-    Parameters
-    ----------
-    telescope_model_name: str
-        Telescope model name (e.g., LST-1).
-
-    Returns
-    -------
-    str
-        Camera name (validated by util.names).
-    """
-
-    _logger = logging.getLogger(__name__)
-    camera_name = ""
-    tel_class, tel_type = names.split_telescope_model_name(telescope_model_name)
-    if tel_class == "LST":
-        camera_name = "LST"
-    elif tel_class == "MST":
-        if "FlashCam" in tel_type:
-            camera_name = "FlashCam"
-        elif "NectarCam" in tel_type:
-            camera_name = "NectarCam"
-        else:
-            _logger.error("Camera not found for MST class telescope")
-    elif tel_class == "SCT":
-        camera_name = "SCT"
-    elif tel_class == "SST":
-        if "ASTRI" in tel_type:
-            camera_name = "ASTRI"
-        elif "GCT" in tel_type:
-            camera_name = "GCT"
-        elif "1M" in tel_type:
-            camera_name = "1M"
-        else:
-            camera_name = "SST"
-    else:
-        _logger.error("Invalid telescope name - please validate it first")
-
-    camera_name = names.validate_camera_name(camera_name)
-    _logger.debug(f"Camera name - {camera_name}")
-    return camera_name
-
-
-def get_telescope_class(telescope_model_name):
-    """
-    Get telescope class from telescope name.
-
-    Parameters
-    ----------
-    telescope_model_name: str
-        Telescope model name (ex. LST-1).
-
-    Returns
-    -------
-    str
-        Telescope class (SST, MST, ...).
-    """
-
-    tel_class, _ = names.split_telescope_model_name(telescope_model_name)
-    return tel_class
-
-
 def is_two_mirror_telescope(telescope_model_name):
     """
     Check if the telescope is a two mirror design.
@@ -141,7 +73,7 @@ def is_two_mirror_telescope(telescope_model_name):
     bool
         True if the telescope is a two mirror one.
     """
-    tel_class, tel_type = names.split_telescope_model_name(telescope_model_name)
+    tel_class, tel_type, _ = names.split_telescope_model_name(telescope_model_name)
     if tel_class == "SST":
         # Only 1M is False
         return "1M" not in tel_type

simtools/model/telescope_model.py

@@ -9,7 +9,8 @@ from astropy import units as u
 from astropy.table import Table
 
 import simtools.utils.general as gen
-from simtools import db_handler, io_handler
+from simtools import db_handler
+from simtools.io_operations import io_handler
 from simtools.model.camera import Camera
 from simtools.model.mirrors import Mirrors
 from simtools.simtel.simtel_config_writer import SimtelConfigWriter
@@ -46,8 +47,9 @@ class TelescopeModel:
         self,
         site,
         telescope_model_name,
-        mongo_db_config,
-        model_version="Current",
+        mongo_db_config=None,
+        model_version="Released",
+        db=None,
         label=None,
     ):
         """
@@ -70,7 +72,12 @@ class TelescopeModel:
         self._camera = None
 
         self.io_handler = io_handler.IOHandler()
-        self.mongo_db_config = mongo_db_config
+        self.db = None
+        if db is not None:
+            self.db = db
+        elif mongo_db_config is not None:
+            self._logger.debug("Connecting to DB")
+            self.db = db_handler.DatabaseHandler(mongo_db_config=mongo_db_config)
 
         self._parameters = {}
 
@@ -131,8 +138,9 @@ class TelescopeModel:
 
         Notes
         -----
-        Todo: Dealing with ifdef/indef etc. By now it just keeps the last version of the parameters
-        in the file.
+        This function does not deal with ifdef/indef etc., it just keeps the last version
+        of the parameters in the file. This is fine for now since we do not
+        expect to read from sim_telarray config files in the future.
 
         Parameters
         ----------
@@ -225,7 +233,9 @@ class TelescopeModel:
     def _set_config_file_directory_and_name(self):
         """Define the variable _config_file_directory and create directories, if needed."""
 
-        self._config_file_directory = self.io_handler.get_output_directory(self.label, "model")
+        self._config_file_directory = self.io_handler.get_output_directory(
+            label=self.label, sub_dir="model"
+        )
 
         # Setting file name and the location
         config_file_name = names.simtel_telescope_config_file_name(
@@ -236,19 +246,20 @@ class TelescopeModel:
     def _load_parameters_from_db(self):
         """Read parameters from DB and store them in _parameters."""
 
-        if self.mongo_db_config is None:
+        if self.db is None:
             return
 
         self._logger.debug("Reading telescope parameters from DB")
 
         self._set_config_file_directory_and_name()
-        db = db_handler.DatabaseHandler(mongo_db_config=self.mongo_db_config)
-        self._parameters = db.get_model_parameters(
+        self._parameters = self.db.get_model_parameters(
             self.site, self.name, self.model_version, only_applicable=True
         )
 
         self._logger.debug("Reading site parameters from DB")
-        _site_pars = db.get_site_parameters(self.site, self.model_version, only_applicable=True)
+        _site_pars = self.db.get_site_parameters(
+            self.site, self.model_version, only_applicable=True
+        )
         self._parameters.update(_site_pars)
 
     def has_parameter(self, par_name):
@@ -494,7 +505,6 @@ class TelescopeModel:
 
     def export_model_files(self):
         """Exports the model files into the config file directory."""
-        db = db_handler.DatabaseHandler(mongo_db_config=self.mongo_db_config)
 
         # Removing parameter files added manually (which are not in DB)
         pars_from_db = copy(self._parameters)
@@ -502,7 +512,7 @@ class TelescopeModel:
             for par in self._added_parameter_files:
                 pars_from_db.pop(par)
 
-        db.export_model_files(pars_from_db, self._config_file_directory)
+        self.db.export_model_files(pars_from_db, self._config_file_directory)
         self._is_exported_model_files_up_to_date = True
 
     def print_parameters(self):
@@ -526,12 +536,11 @@ class TelescopeModel:
     def export_derived_files(self):
         """Write to disk a file from the derived values DB."""
 
-        db = db_handler.DatabaseHandler(mongo_db_config=self.mongo_db_config)
         for par_now in self.derived.values():
             if par_now["File"]:
-                db.export_file_db(
-                    db_name=db.DB_DERIVED_VALUES,
-                    dest=self.io_handler.get_output_directory(self.label, "derived"),
+                self.db.export_file_db(
+                    db_name=self.db.DB_DERIVED_VALUES,
+                    dest=self.io_handler.get_output_directory(label=self.label, sub_dir="derived"),
                     file_name=par_now["Value"],
                 )
 
@@ -655,21 +664,19 @@ class TelescopeModel:
                 "Mirror_list_file was not found in the config directory - "
                 "Using the one found in the model_path"
             )
-        self._mirrors = Mirrors(mirror_list_file)
+        self._mirrors = Mirrors(mirror_list_file, parameters=self._parameters)
 
     def _load_reference_data(self):
         """Load the reference data for this telescope from the DB."""
         self._logger.debug("Reading reference data from DB")
-        db = db_handler.DatabaseHandler(mongo_db_config=self.mongo_db_config)
-        self._reference_data = db.get_reference_data(
+        self._reference_data = self.db.get_reference_data(
             self.site, self.model_version, only_applicable=True
         )
 
     def _load_derived_values(self):
         """Load the derived values for this telescope from the DB."""
         self._logger.debug("Reading derived data from DB")
-        db = db_handler.DatabaseHandler(mongo_db_config=self.mongo_db_config)
-        self._derived = db.get_derived_values(
+        self._derived = self.db.get_derived_values(
             self.site,
             self.name,
             self.model_version,
@@ -710,7 +717,7 @@ class TelescopeModel:
                 label=self.label,
             )
 
-    def is_file_2D(self, par):
+    def is_file_2d(self, par):
         """
         Check if the file referenced by par is a 2D table.
 
@@ -731,8 +738,8 @@ class TelescopeModel:
         file_name = self.get_parameter_value(par)
         file = self.get_config_directory().joinpath(file_name)
         with open(file, "r", encoding="utf-8") as f:
-            is2D = "@RPOL@" in f.read()
-        return is2D
+            is_2d = "@RPOL@" in f.read()
+        return is_2d
 
     def read_two_dim_wavelength_angle(self, file_name):
         """

simtools/psf_analysis.py

@@ -8,6 +8,7 @@ Author: Raul R Prado
 
 import logging
 from math import fabs, pi, sqrt
+from pathlib import Path
 
 import astropy.units as u
 import matplotlib.pyplot as plt
@@ -50,20 +51,20 @@ class PSFImage:
         self.centroid_x = None
         self.centroid_y = None
         self._total_area = total_scattered_area
-        self._stored_PSF = {}
+        self._stored_psf = {}
         if focal_length is not None:
             self._cm_to_deg = 180.0 / pi / focal_length
             self._has_focal_length = True
         else:
             self._has_focal_length = False
 
-    def read_photon_list_from_simtel_file(self, file):
+    def read_photon_list_from_simtel_file(self, photons_file):
         """
         Read photon list file generated by sim_telarray and store the photon positions (2D).
 
         Parameters
         ----------
-        file: str
+        photons_file: str
             Name of sim_telarray file with photon list.
 
         Raises
@@ -72,9 +73,15 @@ class PSFImage:
             If photon positions X and Y are not compatible or are empty.
 
         """
-        self._logger.info(f"Reading sim_telarray file {file}")
+        self._logger.info(f"Reading sim_telarray file {photons_file}")
         self._total_photons = 0
-        with open(file, "r", encoding="utf-8") as f:
+        if Path(photons_file).suffix == ".gz":
+            import gzip  # pylint: disable=import-outside-toplevel
+
+            file_open_function = gzip.open
+        else:
+            file_open_function = open
+        with file_open_function(photons_file, "rb") as f:
             for line in f:
                 self._process_simtel_line(line)
 
@@ -120,7 +127,7 @@ class PSFImage:
             A line from the photon list file generated by sim_telarray.
         """
         words = line.split()
-        if "falling on an area of" in line:
+        if b"falling on an area of" in line:
             self._total_photons += int(words[4])
             total_area_in_file = float(words[14])
             if self._total_area is None:
@@ -134,7 +141,7 @@ class PSFImage:
             else:
                 # Do nothing - Keep the original value of _total_area
                 pass
-        elif "#" in line or len(words) == 0:
+        elif b"#" in line or len(words) == 0:
             # Skipping comments
             pass
         else:
@@ -190,10 +197,10 @@ class PSFImage:
         if unit == "deg" and not self._has_focal_length:
             self._logger.error("PSF cannot be computed in deg because focal length is not set")
             return None
-        if fraction not in self._stored_PSF:
+        if fraction not in self._stored_psf:
             self._compute_psf(fraction)
         unit_factor = 1 if unit == "cm" else self._cm_to_deg
-        return self._stored_PSF[fraction] * unit_factor
+        return self._stored_psf[fraction] * unit_factor
 
     def set_psf(self, value, fraction=0.8, unit="cm"):
         """
@@ -212,7 +219,7 @@ class PSFImage:
             self._logger.error("PSF cannot be set in deg because focal length is not set")
             return
         unit_factor = 1 if unit == "cm" else 1.0 / self._cm_to_deg
-        self._stored_PSF[fraction] = value * unit_factor
+        self._stored_psf[fraction] = value * unit_factor
 
     def _compute_psf(self, fraction):
         """
@@ -223,7 +230,7 @@ class PSFImage:
         fraction: float
             Fraction of photons within the containing radius
         """
-        self._stored_PSF[fraction] = self._find_psf(fraction)
+        self._stored_psf[fraction] = self._find_psf(fraction)
 
     def _find_psf(self, fraction):
         """
@@ -252,21 +259,21 @@ class PSFImage:
         target_number = fraction * self._number_of_detected_photons
         current_radius = 1.5 * radius_sig
         start_number = self._sum_photons_in_radius(current_radius)
-        SCALE = 0.5 * sqrt(current_radius * current_radius / start_number)
+        scale = 0.5 * sqrt(current_radius * current_radius / start_number)
         delta_number = start_number - target_number
         n_iter = 0
-        MAX_ITER = 100
-        TOLERANCE = self._number_of_detected_photons / 1000.0
+        max_iter = 100
+        tolerance = self._number_of_detected_photons / 1000.0
         found_radius = False
-        while not found_radius and n_iter < MAX_ITER:
+        while not found_radius and n_iter < max_iter:
             n_iter += 1
-            dr = -delta_number * SCALE / sqrt(target_number)
+            dr = -delta_number * scale / sqrt(target_number)
             while current_radius + dr < 0:
                 dr *= 0.5
             current_radius += dr
             current_number = self._sum_photons_in_radius(current_radius)
             delta_number = current_number - target_number
-            found_radius = fabs(delta_number) < TOLERANCE
+            found_radius = fabs(delta_number) < tolerance
 
         if found_radius:
             # Diameter = 2 * radius
@@ -385,7 +392,7 @@ class PSFImage:
             psf_ls="--",
         )
         kwargs_for_image = collect_kwargs("image", kwargs)
-        kwargs_for_PSF = collect_kwargs("psf", kwargs)
+        kwargs_for_psf = collect_kwargs("psf", kwargs)
 
         ax = plt.gca()
         # Image histogram
@@ -394,7 +401,7 @@ class PSFImage:
 
         # PSF circle
         center = (0, 0) if centralized else (self.centroid_x, self.centroid_y)
-        circle = plt.Circle(center, self.get_psf(0.8) / 2, **kwargs_for_PSF)
+        circle = plt.Circle(center, self.get_psf(0.8) / 2, **kwargs_for_psf)
         ax.add_artist(circle)
 
         ax.axhline(0, color="k", linestyle="--", zorder=3, linewidth=0.5)

simtools/ray_tracing.py

@@ -1,5 +1,7 @@
+import gzip
 import logging
 import shlex
+import shutil
 import subprocess
 from copy import copy
 from math import pi, tan
@@ -12,7 +14,7 @@ import numpy as np
 from astropy.table import QTable
 
 import simtools.utils.general as gen
-from simtools import io_handler
+from simtools.io_operations import io_handler
 from simtools.model.model_utils import compute_telescope_transmission
 from simtools.model.telescope_model import TelescopeModel
 from simtools.psf_analysis import PSFImage
@@ -61,18 +63,17 @@ class RayTracing:
         """
 
         self._logger = logging.getLogger(__name__)
+        self._logger.debug("Initializing RayTracing class")
 
         self._simtel_source_path = Path(simtel_source_path)
         self._io_handler = io_handler.IOHandler()
 
         self._telescope_model = self._validate_telescope_model(telescope_model)
 
-        _config_data_in = gen.collect_data_from_yaml_or_dict(config_file, config_data)
-        _parameter_file = self._io_handler.get_input_data_file(
-            parent_dir="parameters", file_name="ray-tracing_parameters.yml"
+        self.config = gen.validate_config_data(
+            gen.collect_data_from_file_or_dict(config_file, config_data),
+            SimtelRunnerRayTracing.ray_tracing_default_configuration(False),
         )
-        _parameters = gen.collect_data_from_yaml_or_dict(_parameter_file, None)
-        self.config = gen.validate_config_data(_config_data_in, _parameters)
 
         # Due to float representation, round the off-axis angles so the values in results table
         # are the same as provided.
@@ -80,13 +81,16 @@ class RayTracing:
 
         self.label = label if label is not None else self._telescope_model.label
 
-        self._output_directory = self._io_handler.get_output_directory(self.label, "ray-tracing")
+        self._output_directory = self._io_handler.get_output_directory(
+            label=self.label, sub_dir="ray-tracing"
+        )
 
         # Loading relevant attributes in case of single mirror mode.
         if self.config.single_mirror_mode:
             # Recalculating source distance.
             self._logger.debug(
-                "Single mirror mode is activate - source distance is being recalculated to 2 * flen"
+                "Single mirror mode is activated - "
+                "source distance is being recalculated to 2 * flen"
             )
             mir_flen = self._telescope_model.get_parameter_value("mirror_focal_length")
             self._source_distance = 2 * float(mir_flen) * u.cm.to(u.km)  # km
@@ -178,18 +182,40 @@ class RayTracing:
                 simtel = SimtelRunnerRayTracing(
                     simtel_source_path=self._simtel_source_path,
                     telescope_model=self._telescope_model,
+                    test=test,
                     config_data={
                         "zenith_angle": self.config.zenith_angle * u.deg,
                         "source_distance": self._source_distance * u.km,
                         "off_axis_angle": this_off_axis * u.deg,
-                        "mirror_number": this_mirror,
+                        "mirror_numbers": this_mirror,
                         "use_random_focal_length": self.config.use_random_focal_length,
+                        "single_mirror_mode": self.config.single_mirror_mode,
                     },
-                    single_mirror_mode=self.config.single_mirror_mode,
                     force_simulate=force,
                 )
                 simtel.run(test=test, force=force)
 
+                photons_file_name = names.ray_tracing_file_name(
+                    self._telescope_model.site,
+                    self._telescope_model.name,
+                    self._source_distance,
+                    self.config.zenith_angle,
+                    this_off_axis,
+                    this_mirror if self.config.single_mirror_mode else None,
+                    self.label,
+                    "photons",
+                )
+                photons_file = self._output_directory.joinpath(photons_file_name)
+
+                self._logger.debug("Using gzip to compress the photons file.")
+
+                with open(photons_file, "rb") as f_in:
+                    with gzip.open(
+                        photons_file.with_suffix(photons_file.suffix + ".gz"), "wb"
+                    ) as f_out:
+                        shutil.copyfileobj(f_in, f_out)
+                photons_file.unlink()
+
     def analyze(
         self,
         export=True,
@@ -254,7 +280,7 @@ class RayTracing:
                     "photons",
                 )
 
-                photons_file = self._output_directory.joinpath(photons_file_name)
+                photons_file = self._output_directory.joinpath(photons_file_name + ".gz")
                 tel_transmission = compute_telescope_transmission(
                     tel_transmission_pars, this_off_axis
                 )
@@ -326,23 +352,25 @@ class RayTracing:
         """
 
         try:
-            with open(file, encoding="utf-8") as _stdin:
-                rx_output = subprocess.Popen(  # pylint: disable=consider-using-with
-                    shlex.split(
-                        f"{self._simtel_source_path}/sim_telarray/bin/rx "
-                        f"-f {containment_fraction:.2f} -v"
-                    ),
-                    stdin=_stdin,
-                    stdout=subprocess.PIPE,
-                ).communicate()[0]
+            rx_output = subprocess.Popen(  # pylint: disable=consider-using-with
+                shlex.split(
+                    f"{self._simtel_source_path}/sim_telarray/bin/rx "
+                    f"-f {containment_fraction:.2f} -v"
+                ),
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+            )
+            with gzip.open(file, "rb") as _stdin:
+                with rx_output.stdin:
+                    shutil.copyfileobj(_stdin, rx_output.stdin)
+                    try:
+                        rx_output = rx_output.communicate()[0].splitlines()[-1:][0].split()
+                    except IndexError:
+                        self._logger.error(f"Invalid output from rx: {rx_output}")
+                        raise
         except FileNotFoundError:
             self._logger.error(f"Photon list file not found: {file}")
             raise
-        try:
-            rx_output = rx_output.splitlines()[-1:][0].split()
-        except IndexError:
-            self._logger.error(f"Invalid output from rx: {rx_output}")
-            raise
         containment_diameter_cm = 2 * float(rx_output[0])
         x_mean = float(rx_output[1])
         y_mean = float(rx_output[2])
@@ -489,7 +517,7 @@ class RayTracing:
         """
         images = []
         for this_off_axis in self.config.off_axis_angle:
-            if this_off_axis in self._psf_images:
+            if self._psf_images and this_off_axis in self._psf_images:
                 images.append(self._psf_images[this_off_axis])
         if len(images) == 0:
             self._logger.error("No image found")