gammasimtools 0.8.1__py3-none-any.whl → 0.9.0__py3-none-any.whl

simtools/simtel/simtel_table_reader.py

@@ -0,0 +1,410 @@
+#!/usr/bin/python3
+"""Read tabular data in sim_telarray format and return as astropy table."""
+
+import logging
+import re
+
+import astropy.units as u
+from astropy.table import Table
+
+from simtools.utils import general as gen
+
+logger = logging.getLogger(__name__)
+
+
+def _data_columns(parameter_name, n_columns, n_dim):
+    """
+    Get column information for a given parameter.
+
+    Individual functions are adapted to the specific format of the sim_telarray tables.
+
+    Parameters
+    ----------
+    parameter_name: str
+        Model parameter name.
+    n_columns: int
+        Number of columns in the table.
+    n_dim: list
+        List of columns for n-dimensional tables defined by RPOL lines.
+
+    Returns
+    -------
+    list, str
+        List of columns for n-dimensional tables, description.
+    """
+    if parameter_name == "mirror_reflectivity":
+        return _data_columns_mirror_reflectivity(n_columns, n_dim)
+    if parameter_name in ("discriminator_pulse_shape", "fadc_pulse_shape"):
+        return _data_columns_pulse_shape(n_columns)
+    try:
+        return globals()[f"_data_columns_{parameter_name}"]()
+    except KeyError as exc:
+        raise ValueError(
+            f"Unsupported parameter for sim_telarray table reading: {parameter_name}"
+        ) from exc
+
+
+def _data_columns_atmospheric_profile():
+    """Column representation for parameter atmospheric_profile."""
+    return (
+        [
+            {"name": "altitude", "description": "Altitude", "unit": "km"},
+            {"name": "density", "description": "Density", "unit": "g/cm^3"},
+            {"name": "thickness", "description": "Thickness", "unit": "g/cm^2"},
+            {
+                "name": "refractive_index",
+                "description": "Refractive index (n-1)",
+                "unit": None,
+            },
+            {
+                "name": "temperature",
+                "description": "Temperature",
+                "unit": "K",
+            },
+            {
+                "name": "pressure",
+                "description": "Pressure",
+                "unit": "mbar",
+            },
+            {
+                "name": "pw/w",
+                "description": "Partial pressure of water vapor",
+                "unit": None,
+            },
+        ],
+        "Atmospheric profile",
+    )
+
+
+def _data_columns_pm_photoelectron_spectrum():
+    """Column description for parameter pm_photoelectron_spectrum."""
+    return (
+        [
+            {"name": "amplitude", "description": "Signal amplitude", "unit": None},
+            {
+                "name": "response",
+                "description": "response without afterpulsing component",
+                "unit": None,
+            },
+            {
+                "name": "response_with_ap",
+                "description": "response including afterpulsing component",
+                "unit": None,
+            },
+        ],
+        "Photoelectron spectrum",
+    )
+
+
+def _data_columns_quantum_efficiency():
+    """Column description for parameter quantum_efficiency."""
+    return (
+        [
+            {"name": "wavelength", "description": "Wavelength", "unit": "nm"},
+            {
+                "name": "efficiency",
+                "description": "Quantum efficiency",
+                "unit": None,
+            },
+            {
+                "name": "efficiency_rms",
+                "description": "Quantum efficiency (standard deviation)",
+                "unit": None,
+            },
+        ],
+        "Quantum efficiency",
+    )
+
+
+def _data_columns_camera_filter():
+    """Column description for parameter camera_filter."""
+    return (
+        [
+            {"name": "wavelength", "description": "Wavelength", "unit": "nm"},
+            {
+                "name": "transmission",
+                "description": "Average transmission",
+                "unit": None,
+            },
+        ],
+        "Camera window transmission",
+    )
+
+
+def _data_columns_lightguide_efficiency_vs_wavelength():
+    """Column description for parameter lightguide_efficiency_vs_wavelength."""
+    return _data_columns_lightguide_efficiency_vs_incidence_angle()
+
+
+def _data_columns_lightguide_efficiency_vs_incidence_angle():
+    """Column description for (parameter lightguide_efficiency_vs_incidence_angle."""
+    return (
+        [
+            {"name": "angle", "description": "Incidence angle", "unit": "deg"},
+            {
+                "name": "efficiency",
+                "description": "Light guide efficiency",
+                "unit": None,
+            },
+        ],
+        "Light guide efficiency vs incidence angle",
+    )
+
+
+def _data_columns_mirror_reflectivity(n_columns, n_dim):
+    """Column description for parameter mirror_reflectivity."""
+    _columns = [
+        {"name": "wavelength", "description": "Wavelength", "unit": "nm"},
+    ]
+    if n_dim:
+        for angle in n_dim:
+            _columns.append(
+                {
+                    "name": f"reflectivity_{angle}deg",
+                    "description": f"Mirror reflectivity at {angle} deg",
+                    "unit": None,
+                },
+            )
+    else:
+        _columns.append(
+            {
+                "name": "reflectivity",
+                "description": "Mirror reflectivity",
+                "unit": None,
+            },
+        )
+        if n_columns == 3:
+            _columns.append(
+                {
+                    "name": "reflectivity_rms",
+                    "description": "Mirror reflectivity (standard deviation)",
+                    "unit": None,
+                },
+            )
+        if n_columns == 4:
+            _columns.append(
+                {
+                    "name": "reflectivity_min",
+                    "description": "Mirror reflectivity (min)",
+                    "unit": None,
+                },
+            )
+            _columns.append(
+                {
+                    "name": "reflectivity_max",
+                    "description": "Mirror reflectivity (max)",
+                    "unit": None,
+                },
+            )
+
+    return _columns, "Mirror reflectivity"
+
+
+def _data_columns_pulse_shape(n_columns):
+    """Column description for parameters discriminator_pulse_shape, fadc_pulse_shape."""
+    _columns = [
+        {"name": "time", "description": "Time", "unit": "ns"},
+        {
+            "name": "amplitude",
+            "description": "Amplitude",
+            "unit": None,
+        },
+    ]
+    if n_columns == 3:
+        _columns.append(
+            {
+                "name": "amplitude (low gain)",
+                "description": "Amplitude (low gain)",
+                "unit": None,
+            },
+        )
+
+    return _columns, "Pulse shape"
+
+
+def _data_columns_nsb_reference_spectrum():
+    """Column description for parameter nsb_reference_spectrum."""
+    return (
+        [
+            {"name": "wavelength", "description": "Wavelength", "unit": "nm"},
+            {
+                "name": "differential photon rate",
+                "description": "Differential photon rate",
+                "unit": "1.e9 / (nm s m^2 sr)",
+            },
+        ],
+        "NSB reference spectrum",
+    )
+
+
+def read_simtel_table(parameter_name, file_path):
+    """
+    Read sim_telarray table file for a given parameter.
+
+    Parameters
+    ----------
+    parameter_name: str
+        Model parameter name.
+    file_path: Path
+        Name (full path) of the sim_telarray table file.
+
+    Returns
+    -------
+    Table
+        Astropy table.
+    """
+    if parameter_name == "atmospheric_transmission":
+        return _read_simtel_data_for_atmospheric_transmission(file_path)
+
+    rows, meta_from_simtel, n_columns, n_dim = _read_simtel_data(file_path)
+    columns_info, description = _data_columns(parameter_name, n_columns, n_dim)
+
+    rows = _adjust_columns_length(rows, len(columns_info))
+
+    metadata = {
+        "Name": parameter_name,
+        "File": str(file_path),
+        "Description:": description,
+        "Context_from_sim_telarray": meta_from_simtel,
+    }
+
+    table = Table(rows=rows, names=[col["name"] for col in columns_info])
+    for col, info in zip(table.colnames, columns_info):
+        table[col].unit = info.get("unit")
+        table[col].description = info.get("description")
+    table.meta.update(metadata)
+
+    return table
+
+
+def _adjust_columns_length(rows, n_columns):
+    """
+    Adjust row lengths to match the specified column count.
+
+    - Truncate rows with extra columns beyond the specified count 'n_columns'.
+    - Pad shorter rows with zeros.
+    """
+    return [row[:n_columns] + [0.0] * max(0, n_columns - len(row)) for row in rows]
+
+
+def _read_simtel_data(file_path):
+    """
+    Read data, comments, and (if available) axis definition from sim_telarray table.
+
+    Parameters
+    ----------
+    file_path: Path
+        Path to the sim_telarray table file.
+
+    Returns
+    -------
+    str, str, int, str
+        data, metadata (comments), number of columns (max value), n-dimensional axis description.
+    """
+    logger.debug(f"Reading sim_telarray table from {file_path}")
+    meta_lines = []
+    data_lines = []
+    n_dim_axis = None
+    r_pol_axis = None
+
+    lines = gen.read_file_encoded_in_utf_or_latin(file_path)
+
+    for line in lines:
+        stripped = line.strip()
+        if "@RPOL@" in stripped:  # RPOL description for N-dimensional tables
+            match = re.search(r"#@RPOL@\[(\w+)=\]", stripped)
+            if match:
+                r_pol_axis = match.group(1)
+        elif r_pol_axis and r_pol_axis in stripped:  # N-dimensional axis description
+            n_dim_axis = stripped.split("=")[1].split()
+        elif stripped.startswith("#"):  # Metadata
+            meta_lines.append(stripped.lstrip("#").strip())
+        elif stripped:  # Data
+            data_lines.append(stripped.split("%%%")[0].split("#")[0].strip())  # Remove comments
+
+    rows = [[float(part) for part in line.split()] for line in data_lines]
+    n_columns = max(len(row) for row in rows) if rows else 0
+
+    return rows, "\n".join(meta_lines), n_columns, n_dim_axis
+
+
+def _read_simtel_data_for_atmospheric_transmission(file_path):
+    """
+    Read data and comments from sim_telarray table for atmospheric_transmission.
+
+    Parameters
+    ----------
+    file_path: Path
+        Path to the sim_telarray table file.
+
+    Returns
+    -------
+    astropy table
+        Table with atmospheric transmission.
+    """
+    lines = lines = gen.read_file_encoded_in_utf_or_latin(file_path)
+
+    observatory_level, height_bins = _read_header_line_for_atmospheric_transmission(
+        lines, file_path
+    )
+
+    wavelengths = []
+    heights = []
+    extinctions = []
+    meta_lines = []
+
+    for line in lines:
+        if line.startswith("#") or not line.strip():
+            meta_lines.append(line.lstrip("#").strip())
+            continue
+        parts = line.split()
+        try:
+            wl = float(parts[0])
+            for i, height in enumerate(height_bins):
+                extinction_value = float(parts[i + 1])
+                if extinction_value == 99999.0:
+                    continue
+                wavelengths.append(wl)
+                heights.append(height)
+                extinctions.append(extinction_value)
+        except (ValueError, IndexError):
+            logger.debug(f"Skipping malformed line: {line.strip()}")
+
+    table = Table()
+    table["wavelength"] = wavelengths
+    table["altitude"] = heights
+    table["extinction"] = extinctions
+
+    table.meta.update(
+        {
+            "Name": "atmospheric_transmission",
+            "File": str(file_path),
+            "Description": "Atmospheric transmission",
+            "Context_from_sim_telarray": "\n".join(meta_lines),
+            "observatory_level": observatory_level,
+        }
+    )
+
+    return table
+
+
+def _read_header_line_for_atmospheric_transmission(lines, file_path):
+    """Reader observatory level and height bins from header line for atmospheric transmission."""
+    header_line = None
+    observatory_level = None
+    for line in lines:
+        if "H2=" in line and "H1=" in line:
+            match_h2 = re.search(r"H2=\s*([\d.]+)", line)
+            if match_h2:
+                observatory_level = float(match_h2.group(1)) * u.km
+
+            if "H1=" in line:
+                header_line = line.split("H1=")[-1].strip()
+            break
+
+    if header_line is None:
+        raise ValueError(f"Header with 'H1=' not found file {file_path}")
+
+    height_bins = [float(x.replace(",", "")) for x in header_line.split()]
+
+    return observatory_level, height_bins

simtools/simtel/simulator_camera_efficiency.py

@@ -154,10 +154,12 @@ class SimulatorCameraEfficiency(SimtelRunner):
         command += " 300"  # Xmax
         command += f" {self._telescope_model.get_parameter_value('atmospheric_profile')}"
         command += f" {self.zenith_angle}"
-        command += f" 2>{self._file_log}"
-        command += f" >{self._file_simtel}"
 
-        return f"cd {self._simtel_path.joinpath('sim_telarray')} && {command}"
+        return (
+            f"cd {self._simtel_path.joinpath('sim_telarray')} && {command}",
+            self._file_simtel,
+            self._file_log,
+        )
 
     def _check_run_result(self, run_number=None):  # pylint: disable=unused-argument
         """Check run results.
@@ -169,7 +171,7 @@ class SimulatorCameraEfficiency(SimtelRunner):
         """
         # Checking run
         if not self._file_simtel.exists():
-            msg = "Camera efficiency simulation results file does not exist"
+            msg = f"Camera efficiency simulation results file does not exist ({self._file_simtel})."
             self._logger.error(msg)
             raise RuntimeError(msg)
 

simtools/simtel/simulator_light_emission.py

@@ -1,7 +1,7 @@
 """Simulation using the light emission package for calibration."""
 
 import logging
-import os
+import stat
 from pathlib import Path
 
 import astropy.units as u
@@ -469,5 +469,5 @@ class SimulatorLightEmission(SimtelRunner):
                 file.write(f"{command_plot}\n\n")
                 file.write("# End\n\n")
 
-        os.system(f"chmod ug+x {_script_file}")
+        _script_file.chmod(_script_file.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP)
         return _script_file

simtools/simtel/simulator_ray_tracing.py

@@ -191,9 +191,8 @@ class SimulatorRayTracing(SimtelRunner):
             command += super().get_config_option("mirror_align_random_distance", "0.")
             command += super().get_config_option("mirror_align_random_vertical", "0.,28.,0.,0.")
         command += " " + str(self._corsika_file)
-        command += f" 2>&1 > {self._log_file} 2>&1"
 
-        return command
+        return command, self._log_file, self._log_file
 
     def _check_run_result(self, run_number=None):  # pylint: disable=unused-argument
         """

simtools/simulator.py

@@ -2,6 +2,8 @@
 
 import logging
 import re
+import shutil
+import tarfile
 from collections import defaultdict
 from pathlib import Path
 
@@ -68,7 +70,7 @@ class Simulator:
         self.runs = self._initialize_run_list()
         self._results = defaultdict(list)
         self._test = self.args_dict.get("test", False)
-        self._submit_engine = self.args_dict.get("submit_engine", "local")
+        self.submit_engine = self.args_dict.get("submit_engine", "local")
         self._submit_options = self.args_dict.get("submit_options", None)
         self._extra_commands = extra_commands
 
@@ -265,7 +267,7 @@ class Simulator:
         input_file_list: str or list of str
             Single file or list of files of shower simulations.
         """
-        self._logger.info(f"Submission command: {self._submit_engine}")
+        self._logger.info(f"Submission command: {self.submit_engine}")
 
         runs_and_files_to_submit = self._get_runs_and_files_to_submit(
             input_file_list=input_file_list
@@ -281,7 +283,7 @@ class Simulator:
             )
 
             job_manager = JobManager(
-                submit_engine=self._submit_engine,
+                submit_engine=self.submit_engine,
                 submit_options=self._submit_options,
                 test=self._test,
             )
@@ -389,50 +391,46 @@ class Simulator:
             run number
 
         """
-        self._results["output"].append(
-            str(self._simulation_runner.get_file_name(file_type="output", run_number=run_number))
+        keys = ["output", "sub_out", "log", "input", "hist", "corsika_log"]
+        defaults = {key: None for key in keys}
+        results = {key: defaults[key] for key in keys}
+        results["output"] = str(
+            self._simulation_runner.get_file_name(file_type="output", run_number=run_number)
         )
-        self._results["sub_out"].append(
-            str(
-                self._simulation_runner.get_file_name(
-                    file_type="sub_log", run_number=run_number, mode="out"
-                )
+        results["sub_out"] = str(
+            self._simulation_runner.get_file_name(
+                file_type="sub_log", mode="out", run_number=run_number
             )
         )
 
-        if self.simulation_software in ["simtel", "corsika_simtel"]:
-            self._results["log"].append(
-                str(
-                    self._simulation_runner.get_file_name(
-                        simulation_software="simtel", file_type="log", run_number=run_number
-                    )
+        if "simtel" in self.simulation_software:
+            results["log"] = str(
+                self._simulation_runner.get_file_name(
+                    file_type="log", simulation_software="simtel", run_number=run_number
                 )
             )
-            self._results["input"].append(str(file))
-            self._results["hist"].append(
-                str(
-                    self._simulation_runner.get_file_name(
-                        simulation_software="simtel", file_type="histogram", run_number=run_number
-                    )
+            results["input"] = str(file)
+            results["hist"] = str(
+                self._simulation_runner.get_file_name(
+                    file_type="histogram", simulation_software="simtel", run_number=run_number
                 )
             )
-        else:
-            self._results["corsika_autoinputs_log"].append(
-                str(
-                    self._simulation_runner.get_file_name(
-                        simulation_software="corsika", file_type="log", run_number=run_number
-                    )
+
+        if "corsika" in self.simulation_software:
+            results["corsika_log"] = str(
+                self._simulation_runner.get_file_name(
+                    file_type="corsika_log", simulation_software="corsika", run_number=run_number
                 )
             )
-            self._results["input"].append(None)
-            self._results["hist"].append(None)
-            self._results["log"].append(None)
+
+        for key in keys:
+            self._results[key].append(results[key])
 
     def get_file_list(self, file_type="output"):
         """
         Get list of files generated by simulations.
 
-        Options are "input", "output", "hist", "log".
+        Options are "input", "output", "hist", "log", "corsika_log".
         Not all file types are available for all simulation types.
         Returns an empty list for an unknown file type.
 
@@ -548,7 +546,7 @@ class Simulator:
 
     def save_file_lists(self):
         """Save files lists for output and log files."""
-        for file_type in ["output", "log", "hist"]:
+        for file_type in ["output", "log", "corsika_log", "hist"]:
             file_name = self.io_handler.get_output_directory(label=self.label).joinpath(
                 f"{file_type}_files.txt"
             )
@@ -560,3 +558,52 @@ class Simulator:
                         f.write(f"{line}\n")
             else:
                 self._logger.debug(f"No files to save for {file_type} files.")
+
+    def pack_for_register(self, directory_for_grid_upload=None):
+        """
+        Pack simulation output files for registering on the grid.
+
+        Parameters
+        ----------
+        directory_for_grid_upload: str
+            Directory for the tarball with output files.
+
+        """
+        self._logger.info(
+            f"Packing the output files for registering on the grid ({directory_for_grid_upload})"
+        )
+        output_files = self.get_file_list(file_type="output")
+        log_files = self.get_file_list(file_type="log")
+        corsika_log_files = self.get_file_list(file_type="corsika_log")
+        histogram_files = self.get_file_list(file_type="hist")
+        tar_file_name = Path(log_files[0]).name.replace("log.gz", "log_hist.tar.gz")
+        directory_for_grid_upload = (
+            Path(directory_for_grid_upload)
+            if directory_for_grid_upload
+            else self.io_handler.get_output_directory(label=self.label).joinpath(
+                "directory_for_grid_upload"
+            )
+        )
+        directory_for_grid_upload.mkdir(parents=True, exist_ok=True)
+
+        tar_file_name = directory_for_grid_upload.joinpath(tar_file_name)
+
+        with tarfile.open(tar_file_name, "w:gz") as tar:
+            files_to_tar = (
+                (log_files[:1] if log_files else [])
+                + (histogram_files[:1] if histogram_files else [])
+                + (corsika_log_files[:1] if corsika_log_files else [])
+            )
+            for file_to_tar in files_to_tar:
+                tar.add(file_to_tar, arcname=Path(file_to_tar).name)
+
+        for file_to_move in [*output_files]:
+            source_file = Path(file_to_move)
+            destination_file = directory_for_grid_upload / source_file.name
+            if destination_file.exists():
+                self._logger.warning(f"Overwriting existing file: {destination_file}")
+            # Note that this will overwrite previous files which exist in the directory
+            # It should be fine for normal production since each run is on a separate node
+            # so no files are expected there.
+            shutil.move(source_file, destination_file)
+        self._logger.info(f"Output files for the grid placed in {directory_for_grid_upload!s}")

simtools/testing/configuration.py

@@ -3,7 +3,6 @@
 import logging
 from pathlib import Path
 
-import pytest
 import yaml
 
 import simtools.utils.general as gen
@@ -11,6 +10,10 @@ import simtools.utils.general as gen
 _logger = logging.getLogger(__name__)
 
 
+class VersionError(Exception):
+    """Raise if model version requested is not supported."""
+
+
 def get_list_of_test_configurations(config_files):
     """
     Return list of test configuration dictionaries or test names.
@@ -121,7 +124,7 @@ def _skip_test_for_model_version(config, model_version_requested):
         return
     model_version_config = config["CONFIGURATION"]["MODEL_VERSION"]
     if model_version_requested != model_version_config:
-        pytest.skip(
+        raise VersionError(
             f"Model version requested {model_version_requested} not supported for this test"
         )
 
@@ -158,13 +161,14 @@ def _prepare_test_options(config, output_path, model_version=None):
 
     tmp_config_file = output_path / "tmp_config.yml"
     config_file_model_version = config.get("MODEL_VERSION")
-    if model_version is not None and "MODEL_VERSION" in config:
+    if model_version and "MODEL_VERSION" in config:
         config.update({"MODEL_VERSION": model_version})
-    if "OUTPUT_PATH" in config:
-        config.update({"OUTPUT_PATH": str(Path(output_path).joinpath(config["OUTPUT_PATH"]))})
-        config.update({"USE_PLAIN_OUTPUT_PATH": True})
-    if "DATA_DIRECTORY" in config:
-        config.update({"DATA_DIRECTORY": str(Path(output_path).joinpath(config["DATA_DIRECTORY"]))})
+
+    for key in ["OUTPUT_PATH", "DATA_DIRECTORY", "PACK_FOR_GRID_REGISTER"]:
+        if key in config:
+            config[key] = str(Path(output_path).joinpath(config[key]))
+            if key == "OUTPUT_PATH":
+                config["USE_PLAIN_OUTPUT_PATH"] = True
 
     _logger.info(f"Writing config file: {tmp_config_file}")
     with open(tmp_config_file, "w", encoding="utf-8") as file: