gammasimtools 0.26.0__py3-none-any.whl → 0.27.1__py3-none-any.whl

simtools/ray_tracing/psf_analysis.py

@@ -7,9 +7,7 @@ Main functionalities are: computing centroids, psf containers etc.
 
 import gzip
 import logging
-import shlex
-import shutil
-import subprocess
+import tempfile
 from math import fabs, pi, sqrt
 from pathlib import Path
 
@@ -18,6 +16,7 @@ import matplotlib.pyplot as plt
 import numpy as np
 
 from simtools import settings
+from simtools.job_execution import job_manager
 from simtools.utils.general import collect_kwargs, set_default_kwargs
 
 
@@ -93,36 +92,49 @@ class PSFImage:
         photons_file: str
             Name of sim_telarray file with photon list.
         """
+        rx_command = [
+            f"{settings.config.sim_telarray_path}/bin/rx",
+            "-f",
+            f"{self._containment_fraction:.2f}",
+            "-v",
+        ]
+
         try:
-            with subprocess.Popen(
-                shlex.split(
-                    f"{settings.config.sim_telarray_path}/bin/rx "
-                    f"-f {self._containment_fraction:.2f} -v"
-                ),
-                stdin=subprocess.PIPE,
-                stdout=subprocess.PIPE,
-            ) as rx_process:
-                with gzip.open(photon_file, "rb") as _stdin:
-                    with rx_process.stdin:
-                        shutil.copyfileobj(_stdin, rx_process.stdin)
-                        try:
-                            rx_output = rx_process.communicate()[0].splitlines()[-1:][0].split()
-                        except IndexError as e:
-                            raise IndexError(
-                                f"Unexpected output format from rx: {rx_process}"
-                            ) from e
+            with tempfile.NamedTemporaryFile(mode="wb", delete=False) as tmp_file:
+                with gzip.open(photon_file, "rb") as gz:
+                    tmp_file.write(gz.read())
+                tmp_file_path = tmp_file.name
+
+            try:
+                with open(tmp_file_path, "rb") as tmp_f:
+                    result = job_manager.submit(rx_command, stdin=tmp_f)
+
+                data_lines = [
+                    line.strip()
+                    for line in result.stdout.strip().split("\n")
+                    if line.strip() and not line.startswith("#")
+                ]
+
+                if not data_lines:
+                    raise IndexError("No data line found in RX output")
+
+                rx_output = data_lines[-1].split()
+                self.set_psf(
+                    2 * float(rx_output[0]), fraction=self._containment_fraction, unit="cm"
+                )
+                self.centroid_x = float(rx_output[1])
+                self.centroid_y = float(rx_output[2])
+                self._effective_area = float(rx_output[5])
+
+            finally:
+                Path(tmp_file_path).unlink(missing_ok=True)
+
         except FileNotFoundError as e:
             raise FileNotFoundError(f"Photon list file not found: {photon_file}") from e
-
-        try:
-            self.set_psf(2 * float(rx_output[0]), fraction=self._containment_fraction, unit="cm")
-            self.centroid_x = float(rx_output[1])
-            self.centroid_y = float(rx_output[2])
-            self._effective_area = float(rx_output[5])
-        except IndexError as e:
-            raise IndexError(f"Unexpected output format from rx: {rx_output}") from e
-        except ValueError as e:
-            raise ValueError(f"Invalid output format from rx: {rx_output}") from e
+        except (IndexError, ValueError) as e:
+            raise type(e)(
+                f"Invalid RX output format: {locals().get('rx_output', 'unknown')}"
+            ) from e
 
     def read_photon_list_from_simtel_file(self, photons_file):
         """
@@ -150,9 +162,9 @@ class PSFImage:
                 self._process_simtel_line(line)
 
         if not self._is_photon_positions_ok():
-            msg = "Problems reading sim_telarray file - invalid data"
-            self._logger.error(msg)
-            raise RuntimeError(msg)
+            raise RuntimeError(
+                f"Problems reading sim_telarray photons file {photons_file} - invalid data"
+            )
 
         self.centroid_x = np.mean(self.photon_pos_x)
         self.centroid_y = np.mean(self.photon_pos_y)
@@ -467,9 +479,10 @@ class PSFImage:
         ax.hist2d(data["X"], data["Y"], **kwargs_for_image)
         ax.set_aspect("equal", "datalim")
 
-        # PSF circle (80%)
+        # PSF circle (containment fraction)
+        fraction = self._containment_fraction if self._containment_fraction is not None else 0.8
         center = (0, 0) if centralized else (self.centroid_x, self.centroid_y)
-        circle = plt.Circle(center, self.get_psf(0.8) / 2, **kwargs_for_psf)
+        circle = plt.Circle(center, self.get_psf(fraction) / 2, **kwargs_for_psf)
         ax.add_artist(circle)
 
         ax.axhline(0, color="k", linestyle="--", zorder=3, linewidth=0.5)
@@ -495,7 +508,8 @@ class PSFImage:
         if radius is not None:
             radius_all = radius.to(u.cm).value if isinstance(radius, u.Quantity) else radius
         else:
-            radius_all = list(np.linspace(0, 1.6 * self.get_psf(0.8), 30))
+            fraction = self._containment_fraction if self._containment_fraction is not None else 0.8
+            radius_all = list(np.linspace(0, 1.6 * self.get_psf(fraction), 30))
         intensity = [
             self._sum_photons_in_radius(rad) / self._number_of_detected_photons
             for rad in radius_all
@@ -510,15 +524,15 @@ class PSFImage:
 
         return result
 
-    def plot_cumulative(self, file_name=None, d80=None, **kwargs):
+    def plot_cumulative(self, file_name=None, psf_diameter_cm=None, **kwargs):
         """Plot cumulative data (intensity vs radius).
 
         Parameters
         ----------
         file_name: str
             Name of the file to save the plot to.
-        d80: float
-            d80 value to be marked in the plot (in cm).
+        psf_diameter_cm: float
+            PSF diameter value to be marked in the plot (in cm).
         **kwargs:
             Customization of line plot (e.g., color, linestyle, linewidth).
         """
@@ -527,9 +541,10 @@ class PSFImage:
         ax.set_xlabel("Radius (cm)")
         ax.set_ylabel("Contained light %")
         ax.plot(data[self.__PSF_RADIUS], data[self.__PSF_CUMULATIVE], **kwargs)
-        ax.axvline(x=self.get_psf(0.8) / 2, color="b", linestyle="--", linewidth=1)
-        if d80 is not None:
-            ax.axvline(x=d80 / 2.0, color="r", linestyle="--", linewidth=1)
+        fraction = self._containment_fraction if self._containment_fraction is not None else 0.8
+        ax.axvline(x=self.get_psf(fraction) / 2, color="b", linestyle="--", linewidth=1)
+        if psf_diameter_cm is not None:
+            ax.axvline(x=psf_diameter_cm / 2.0, color="r", linestyle="--", linewidth=1)
         if file_name is not None:
             fig.savefig(file_name)
             plt.close(fig)

simtools/ray_tracing/psf_parameter_optimisation.py

@@ -857,10 +857,11 @@ def _run_ray_tracing_simulation(tel_model, site_model, args_dict, pars):
     if pars is None:
         raise ValueError("No best parameters found")
 
-    tel_model.overwrite_parameters(pars)
+    tel_model.overwrite_parameters(pars, flat_dict=True)
     ray = RayTracing(
         telescope_model=tel_model,
         site_model=site_model,
+        label=args_dict.get("label") or getattr(tel_model, "label", None),
         zenith_angle=args_dict["zenith"] * u.deg,
         source_distance=args_dict["src_distance"] * u.km,
         off_axis_angle=[0.0] * u.deg,
@@ -1381,7 +1382,7 @@ def cleanup_intermediate_files(output_dir):
     output_dir : Path
         Directory containing output files to clean up.
     """
-    patterns = ["*.log", "*.lis*"]
+    patterns = ["*.log", "*.lis*", "*.dat"]
     files_removed = 0
 
     for pattern in patterns:

simtools/ray_tracing/ray_tracing.py

@@ -51,8 +51,8 @@ class RayTracing:
     """
 
     YLABEL = {
-        "d80_cm": r"$D_{80}$",
-        "d80_deg": r"$D_{80}$",
+        "psf_cm": "PSF",
+        "psf_deg": "PSF",
         "eff_area": "Eff. mirror area",
         "eff_flen": "Eff. focal length",
     }
@@ -209,6 +209,7 @@ class RayTracing:
                 simtel = SimulatorRayTracing(
                     telescope_model=self.telescope_model,
                     site_model=self.site_model,
+                    label=self.label,
                     test=test,
                     config_data={
                         "zenith_angle": self.zenith_angle,
@@ -468,22 +469,46 @@ class RayTracing:
         else:
             self._logger.error("No results to export")
 
+    def get_psf_mm(self, row_index: int = 0) -> float:
+        """Return PSF diameter from the analysis results in mm.
+
+        Parameters
+        ----------
+        row_index : int
+            Row index into the results table (default: 0).
+
+        Returns
+        -------
+        float
+            PSF diameter in millimeters.
+        """
+        if self._results is None:
+            raise RuntimeError("No results available; run analyze() first")
+        psf = self._results["psf_cm"][row_index]
+
+        if isinstance(psf, u.Quantity):
+            psf_cm = psf.to_value(u.cm)
+        else:
+            psf_cm = float(psf)
+
+        return psf_cm * 10.0
+
     def _read_results(self):
         """Read existing results file and store it in _results."""
         self._results = astropy.io.ascii.read(self._file_results, format="ecsv")
 
-    def plot(self, key, save=False, d80=None, **kwargs):
+    def plot(self, key, save=False, psf_diameter_cm=None, **kwargs):
         """
         Plot key vs off-axis angle and save the figure in pdf.
 
         Parameters
         ----------
         key: str
-            d80_cm, d80_deg, eff_area or eff_flen
+            psf_cm, psf_deg, eff_area or eff_flen
         save: bool
             If True, figure will be saved.
-        d80: float
-            d80 for cumulative PSF plot.
+        psf_diameter_cm: float
+            PSF diameter value to be marked in the cumulative PSF plot (in cm).
         **kwargs:
             kwargs for plt.plot
 
@@ -533,7 +558,9 @@ class RayTracing:
                     image_cumulative_file_name
                 )
                 self._logger.info(f"Saving cumulative PSF to {image_cumulative_file}")
-                image.plot_cumulative(file_name=image_cumulative_file, d80=d80)
+                image.plot_cumulative(
+                    file_name=image_cumulative_file, psf_diameter_cm=psf_diameter_cm
+                )
 
     def plot_histogram(self, key, **kwargs):
         """
@@ -542,7 +569,7 @@ class RayTracing:
         Parameters
         ----------
         key: str
-            d80_cm, d80_deg, eff_area or eff_flen
+            psf_cm, psf_deg, eff_area or eff_flen
         **kwargs:
             kwargs for plt.hist
 
@@ -564,7 +591,7 @@ class RayTracing:
         Parameters
         ----------
         key: str
-            d80_cm, d80_deg, eff_area or eff_flen
+            psf_cm, psf_deg, eff_area or eff_flen
 
         Returns
         -------
@@ -588,7 +615,7 @@ class RayTracing:
         Parameters
         ----------
         key: str
-            d80_cm, d80_deg, eff_area or eff_flen
+            psf_cm, psf_deg, eff_area or eff_flen
 
         Returns
         -------

simtools/runners/corsika_runner.py

@@ -1,26 +1,17 @@
 """Generate run scripts and directories for CORSIKA simulations."""
 
 import logging
-import stat
+from pathlib import Path
 
 from simtools import settings
-from simtools.io import io_handler
 from simtools.runners.runner_services import RunnerServices
 
 
-class MissingRequiredEntryInCorsikaConfigError(Exception):
-    """Exception for missing required entry in corsika config."""
-
-
 class CorsikaRunner:
     """
-    Generate run scripts and directories for CORSIKA simulations. Run simulations if requested.
-
-    CorsikaRunner is responsible for configuring and running CORSIKA, using corsika_autoinputs
-    provided by the sim_telarray package. CorsikaRunner generates shell scripts to be run
-    externally or by the simulator module simulator.
+    Prepare and run CORSIKA simulations.
 
-    CorsikaRunner is configured through a CorsikaConfig instance.
+    Generate run scripts and directories for CORSIKA simulations. Run simulations if requested.
 
     Parameters
     ----------
@@ -28,8 +19,6 @@ class CorsikaRunner:
         CORSIKA configuration.
     label: str
         Instance label.
-    keep_seeds: bool
-        Use seeds based on run number and primary particle. If False, use sim_telarray seeds.
     use_multipipe: bool
         Use multipipe to run CORSIKA and sim_telarray.
     curved_atmosphere_min_zenith_angle: Quantity
@@ -40,7 +29,6 @@ class CorsikaRunner:
         self,
         corsika_config,
         label=None,
-        keep_seeds=False,
         use_multipipe=False,
         curved_atmosphere_min_zenith_angle=None,
     ):
@@ -50,73 +38,63 @@ class CorsikaRunner:
         self.label = label
 
         self.corsika_config = corsika_config
-        self._keep_seeds = keep_seeds
         self._use_multipipe = use_multipipe
         self.curved_atmosphere_min_zenith_angle = curved_atmosphere_min_zenith_angle
 
-        self.io_handler = io_handler.IOHandler()
-
-        self.runner_service = RunnerServices(corsika_config, label)
-        self._directory = self.runner_service.load_data_directories("corsika")
+        self.runner_service = RunnerServices(corsika_config, run_type="corsika", label=label)
+        self.file_list = None
 
-    def prepare_run_script(
-        self, run_number=None, extra_commands=None, input_file=None, use_pfp=True
-    ):
+    def prepare_run(self, run_number, sub_script, extra_commands=None, corsika_file=None):
         """
-        Prepare and write CORSIKA run script.
+        Prepare CORSIKA run script and run directory.
+
+        The CORSIKA run directory includes all input files needed for the simulation.
 
         Parameters
         ----------
-        use_pfp: bool
-            Whether to use the preprocessor in preparing the CORSIKA input file
         run_number: int
             Run number.
+        sub_script: str or Path
+            Path to the CORSIKA run script to be created.
+        corsika_file: str or Path
+            Path to the multipipe script (used only if use_multipipe is True).
         extra_commands: str
             Additional commands for running simulations.
-
-        Returns
-        -------
-        Path:
-            Full path of the run script file.
         """
-        if input_file is not None:
-            self._logger.warning(
-                "input_file parameter is not used in CorsikaRunner.prepare_run_script"
-            )
-        self.corsika_config.run_number = run_number
-
-        script_file_path = self.get_file_name(
-            file_type="sub_script", run_number=self.corsika_config.run_number
-        )
-        corsika_input_file = self.corsika_config.generate_corsika_input_file(
-            use_multipipe=self._use_multipipe, use_test_seeds=self._keep_seeds
+        self.file_list = self.runner_service.load_files(run_number=run_number)
+
+        self.corsika_config.generate_corsika_input_file(
+            self._use_multipipe,
+            self.runner_service.get_file_name("corsika_input", run_number=run_number),
+            self.runner_service.get_file_name("corsika_output", run_number=run_number)
+            if not self._use_multipipe
+            else corsika_file,
         )
 
-        # CORSIKA input file for a specific run, created by the preprocessor pfp
-        corsika_input_tmp_name = self.corsika_config.get_corsika_config_file_name(
-            file_type="config_tmp", run_number=self.corsika_config.run_number
-        )
-        corsika_input_tmp_file = self._directory["inputs"].joinpath(corsika_input_tmp_name)
-        # CORSIKA log file naming (temporary and final)
-        corsika_log_tmp_file = (
-            self._directory["data"]
-            .joinpath(f"run{self.corsika_config.run_number:06}")
-            .joinpath(f"run{self.corsika_config.run_number}.log")
-        )
-        corsika_log_file = self.get_file_name(
-            file_type="corsika_log", run_number=self.corsika_config.run_number
-        )
+        self._logger.debug(f"Extra commands to be added to the run script: {extra_commands}")
 
-        if use_pfp:
-            pfp_command = self._get_pfp_command(corsika_input_tmp_file, corsika_input_file)
-        autoinputs_command = self._get_autoinputs_command(
-            self.corsika_config.run_number, corsika_input_tmp_file
-        )
+        corsika_run_dir = self.runner_service.get_file_name(
+            "corsika_output", run_number=run_number
+        ).parent
 
-        self._logger.debug(f"Extra commands to be added to the run script: {extra_commands}")
-        self._logger.debug(f"CORSIKA data will be set to {self._directory['data']}")
+        self._export_run_script(run_number, sub_script, corsika_run_dir, extra_commands)
 
-        with open(script_file_path, "w", encoding="utf-8") as file:
+    def _corsika_executable(self):
+        """Get the CORSIKA executable path."""
+        if self.corsika_config.use_curved_atmosphere:
+            self._logger.debug("Using curved-atmosphere CORSIKA binary.")
+            return Path(settings.config.corsika_exe_curved)
+        self._logger.debug("Using flat-atmosphere CORSIKA binary.")
+        return Path(settings.config.corsika_exe)
+
+    def _export_run_script(self, run_number, sub_script, corsika_run_dir, extra_commands):
+        """Export CORSIKA run script."""
+        corsika_log_file = self.runner_service.get_file_name(
+            "corsika_log", run_number=run_number
+        ).with_suffix("")  # remove .gz from log file
+        corsika_input = self.runner_service.get_file_name("corsika_input", run_number=run_number)
+        sub_script = Path(sub_script)
+        with open(sub_script, "w", encoding="utf-8") as file:
             file.write("#!/usr/bin/env bash\n")
             file.write("set -e\n")
             file.write("set -o pipefail\n")
@@ -129,136 +107,19 @@ class CorsikaRunner:
                 file.write(f"{extra_commands}\n")
                 file.write("# End of extras\n\n")
 
-            file.write(f"export CORSIKA_DATA={self._directory['data']}\n")
+            file.write(f"export CORSIKA_DATA={corsika_run_dir}\n")
             file.write('mkdir -p "$CORSIKA_DATA"\n')
             file.write('cd "$CORSIKA_DATA" || exit 2\n')
-            if use_pfp:
-                file.write("\n# Running pfp\n")
-                file.write(pfp_command)
-                file.write("\n# Replacing the XXXXXX placeholder with the run number\n")
-                file.write(
-                    f"sed -i 's/XXXXXX/{self.corsika_config.run_number:06}/g' "
-                    f"{corsika_input_tmp_file}\n"
-                )
-            else:
-                file.write("\n# Copying CORSIKA input file to run location\n")
-                file.write(f"cp {corsika_input_file} {corsika_input_tmp_file}")
-            file.write("\n# Running corsika_autoinputs\n")
-            file.write(autoinputs_command)
-            file.write("\n# Moving log files to the corsika log directory\n")
-            file.write(f"gzip {corsika_log_tmp_file}\n")
-            file.write(f"mv -v {corsika_log_tmp_file}.gz {corsika_log_file}\n")
-
-            file.write('\necho "RUNTIME: $SECONDS"\n')
-
-        script_file_path.chmod(script_file_path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP)
-        return script_file_path
 
-    def get_resources(self, run_number=None):
-        """Return computing resources used."""
-        return self.runner_service.get_resources(run_number)
-
-    def _get_pfp_command(self, input_tmp_file, corsika_input_file):
-        """
-        Get pfp pre-processor command.
-
-        pfp is a pre-processor tool and part of sim_telarray.
-
-        Parameters
-        ----------
-        input_tmp_file: Path
-            Temporary input file.
-
-        Returns
-        -------
-        str
-            pfp command.
-        """
-        cmd = settings.config.sim_telarray_path / "bin/pfp"
-        cmd = str(cmd) + f" -V -DWITHOUT_MULTIPIPE - < {corsika_input_file}"
-        cmd += f" > {input_tmp_file} || exit\n"
-        return cmd
-
-    def _get_autoinputs_command(self, run_number, input_tmp_file):
-        """
-        Get autoinputs command.
-
-        corsika_autoinputs is a tool to generate random seeds and user/host dependent
-        parameters for CORSIKA configuration.
-
-        Parameters
-        ----------
-        run_number: int
-            Run number.
-        input_tmp_file: Path
-            Temporary input file.
-
-        Returns
-        -------
-        str
-            autoinputs command.
-        """
-        if self.corsika_config.use_curved_atmosphere:
-            corsika_bin_path = settings.config.corsika_exe_curved
-            self._logger.debug("Using curved-atmosphere CORSIKA binary.")
-        else:
-            corsika_bin_path = settings.config.corsika_exe
-            self._logger.debug("Using flat-atmosphere CORSIKA binary.")
-
-        log_file = self.get_file_name(file_type="log", run_number=run_number)
-        if self._use_multipipe:
-            log_file = log_file.with_name(f"multipipe_{log_file.name}")
-
-        cmd = settings.config.sim_telarray_path.joinpath("bin/corsika_autoinputs")
-        cmd = str(cmd) + f" --run {corsika_bin_path}"
-        cmd += f" -R {run_number}"
-        cmd += ' -p "$CORSIKA_DATA"'
-        if self._keep_seeds:
-            logging.warning(
-                "Using --keep-seeds option in corsika_autoinputs is not recommended. "
-                "It should only be used for testing purposes."
+            file.write("\n# Running corsika\n")
+            file.write(
+                f"{self._corsika_executable()} < {corsika_input} > {corsika_log_file} 2>&1\n"
             )
-            cmd += " --keep-seeds"
-        cmd += f" {input_tmp_file} | gzip > {log_file} 2>&1"
-        cmd += " || exit 1\n"
-        return cmd
+            file.write("\n# Cleanup\n")
+            file.write(f"gzip {corsika_log_file}\n")
 
-    def get_file_name(
-        self,
-        simulation_software="corsika",
-        file_type=None,
-        run_number=None,
-        mode="",
-        model_version_index=0,
-    ):
-        """
-        Get the full path of a file for a given run number.
-
-        Parameters
-        ----------
-        simulation_software: str
-            Simulation software.
-        file_type: str
-            File type.
-        run_number: int
-            Run number.
-        model_version_index: int
-            Index of the model version.
-            This is used to select the correct simulator_array instance in case
-            multiple array models are simulated.
+            file.write('\necho "RUNTIME: $SECONDS"\n')
 
-        Returns
-        -------
-        str
-            File name with full path.
-        """
-        if simulation_software.lower() != "corsika":
-            raise ValueError(
-                f"simulation_software ({simulation_software}) is not supported in CorsikaRunner"
-            )
-        return self.runner_service.get_file_name(
-            file_type=file_type,
-            run_number=run_number,
-            mode=mode,
-            _model_version_index=model_version_index,
-        )
+    def get_resources(self, sub_out_file):
+        """Return computing resources used."""
+        return self.runner_service.get_resources(sub_out_file)