gammasimtools 0.8.2__py3-none-any.whl → 0.9.0__py3-none-any.whl

simtools/runners/simtel_runner.py

@@ -1,7 +1,8 @@
 """Base class for running sim_telarray simulations."""
 
 import logging
-import os
+import stat
+import subprocess
 from pathlib import Path
 
 import simtools.utils.general as gen
@@ -96,7 +97,7 @@ class SimtelRunner:
 
             file.write('\necho "RUNTIME: $SECONDS"\n')
 
-        os.system(f"chmod ug+x {script_file_path}")
+        script_file_path.chmod(script_file_path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP)
         return script_file_path
 
     def run(self, test=False, input_file=None, run_number=None):
@@ -114,15 +115,17 @@ class SimtelRunner:
         """
         self._logger.debug("Running sim_telarray")
 
-        command = self._make_run_command(run_number=run_number, input_file=input_file)
+        command, stdout_file, stderr_file = self._make_run_command(
+            run_number=run_number, input_file=input_file
+        )
 
         if test:
             self._logger.info(f"Running (test) with command: {command}")
-            self._run_simtel_and_check_output(command)
+            self._run_simtel_and_check_output(command, stdout_file, stderr_file)
         else:
             self._logger.debug(f"Running ({self.runs_per_set}x) with command: {command}")
             for _ in range(self.runs_per_set):
-                self._run_simtel_and_check_output(command)
+                self._run_simtel_and_check_output(command, stdout_file, stderr_file)
 
         self._check_run_result(run_number=run_number)
 
@@ -148,7 +151,7 @@ class SimtelRunner:
         self._logger.error(msg)
         raise SimtelExecutionError(msg)
 
-    def _run_simtel_and_check_output(self, command):
+    def _run_simtel_and_check_output(self, command, stdout_file, stderr_file):
         """
         Run the sim_telarray command and check the exit code.
 
@@ -157,8 +160,24 @@ class SimtelRunner:
         SimtelExecutionError
             if run was not successful.
         """
-        if os.system(command) != 0:
+        stdout_file = stdout_file if stdout_file else "/dev/null"
+        stderr_file = stderr_file if stderr_file else "/dev/null"
+        with (
+            open(f"{stdout_file}", "w", encoding="utf-8") as stdout,
+            open(f"{stderr_file}", "w", encoding="utf-8") as stderr,
+        ):
+            result = subprocess.run(
+                command,
+                shell=True,
+                text=True,
+                stdout=stdout,
+                stderr=stderr,
+            )
+
+        if result.returncode != 0:
+            self._logger.error(result.stderr)
             self._raise_simtel_error()
+        return result.returncode
 
     def _make_run_command(self, run_number=None, input_file=None):
         self._logger.debug(
@@ -167,7 +186,7 @@ class SimtelRunner:
         )
         input_file = input_file if input_file else "nofile"
         run_number = run_number if run_number else 1
-        return f"{input_file}-{run_number}"
+        return f"{input_file}-{run_number}", None, None
 
     @staticmethod
     def get_config_option(par, value=None, weak_option=False):

simtools/schemas/integration_tests_config.metaschema.yml

@@ -3,7 +3,7 @@ $schema: http://json-schema.org/draft-06/schema#
 $ref: '#/definitions/SimtoolsIntegrationTestConfiguration'
 title: SimPipe integration test configuration metaschema
 description: YAML representation of integration test configuration metaschema
-version: 0.1.0
+version: 0.2.0
 name: integration_tests_config.metaschema
 type: object
 additionalProperties: false
@@ -60,6 +60,20 @@ definitions:
         description: |
           "Path of output file tested to be generated by integration test."
         type: string
+      TEST_OUTPUT_FILES:
+        type: array
+        items:
+          type: object
+          properties:
+            PATH_DESCRIPTOR:
+              type: string
+            FILE:
+              type: string
+            EXPECTED_OUTPUT:
+              description: |
+                "Expected output of integration test."
+              type: object
+          required: ["PATH_DESCRIPTOR", "FILE"]
       FILE_TYPE:
         description: |
           "Expected file type of output file generated by integration test."
@@ -68,10 +82,6 @@ definitions:
         description: |
           "Reference file used for comparison."
         type: string
-      EXPECTED_OUTPUT:
-        description: |
-          "Expected output of integration test."
-        type: object
       TOLERANCE:
         description: "Allowed tolerance for floating point comparison."
         type: number

simtools/schemas/model_parameter.metaschema.yml

@@ -2,12 +2,99 @@
 $schema: http://json-schema.org/draft-06/schema#
 $ref: '#/definitions/SimtoolsModelParameters'
 title: SimPipe DB Model Parameter Metaschema
-description: YAML representation of db model parameter metaschema
-version: 0.1.0
+description: |
+  YAML representation of db model parameter metaschema
+  (based on simulation model DB).
+version: 0.2.0
 name: modelparameter.metaschema
 type: object
 additionalProperties: false
 
+definitions:
+  SimtoolsModelParameters:
+    description: ""
+    type: object
+    properties:
+      _id:
+        type: string
+        description: "DB unique identifier."
+      entry_date:
+        type: string
+        description: "Value entry date."
+      file:
+        type: boolean
+        description: "This parameter is a file."
+      instrument:
+        type: string
+        description: "Associated instrument."
+      site:
+        type: string
+        description: "Associated CTAO site."
+        enum:
+          - North
+          - South
+      type:
+        type: string
+        description: "Data type"
+        enum:
+          - boolean
+          - dict
+          - double
+          - file
+          - float64
+          - int
+          - int64
+          - string
+          - uint
+          - uint32
+          - uint64
+      unit:
+        anyOf:
+          - type: string
+          - type: "null"
+        description: "Unit of the parameter."
+      value:
+        anyOf:
+          - type: boolean
+          - type: number
+          - type: string
+          - type: "null"
+          - type: array
+        description: "Value of the parameter."
+      parameter_version:
+        anyOf:
+          - type: string
+        description: "Parameter version."
+      schema_version:
+        anyOf:
+          - type: string
+        description: "Metaschema version."
+      unique_id:
+        anyOf:
+          - type: string
+          - type: "null"
+        description: "Unique ID of parameter definition."
+    required:
+      - file
+      - instrument
+      - parameter_version
+      - schema_version
+      - site
+      - type
+      - unit
+      - value
+...
+---
+$schema: http://json-schema.org/draft-06/schema#
+$ref: '#/definitions/SimtoolsModelParameters'
+title: SimPipe DB Model Parameter Metaschema
+description: |
+  YAML representation of db model parameter metaschema
+  (based on model_parameters DB).
+version: 0.1.0
+name: modelparameter.metaschema
+type: object
+additionalProperties: false
 
 definitions:
   SimtoolsModelParameters:
@@ -48,6 +135,7 @@ definitions:
           - boolean
           - double
           - int
+          - int64
           - string
           - uint
           - file

simtools/schemas/model_parameters/effective_focal_length.schema.yml

@@ -33,12 +33,42 @@ short_description: |-
   Effective focal length.
   Only to be used for image analysis, has no effect on the simulation.
 data:
-  - type: double
+  - description: |-
+      Mean effective length for all directions of incidence.
+    type: double
     unit: cm
     default: 0.0
     allowed_range:
       min: 0.0
       max: 10000.0
+  - description: |-
+      Effective length for incidence directions in mirror/camera x-z plane
+      (if non-zero).
+    type: double
+    unit: cm
+    default: 0.0
+    allowed_range:
+      min: 0.0
+      max: 10000.0
+  - description: |-
+      Effective length for incidence directions in mirror/camera y-z plane
+      (if non-zero).
+    type: double
+    unit: cm
+    default: 0.0
+    allowed_range:
+      min: 0.0
+      max: 10000.0
+  - description: |-
+      Any displacement along x in the focal plane from asymmetric PSF behavior.
+    type: double
+    unit: cm
+    default: 0.0
+  - description: |-
+      Any displacement along y in the focal plane from asymmetric PSF behavior.
+    type: double
+    unit: cm
+    default: 0.0
 instrument:
   class: Structure
   type:

simtools/simtel/simtel_table_reader.py

@@ -0,0 +1,410 @@
+#!/usr/bin/python3
+"""Read tabular data in sim_telarray format and return as astropy table."""
+
+import logging
+import re
+
+import astropy.units as u
+from astropy.table import Table
+
+from simtools.utils import general as gen
+
+logger = logging.getLogger(__name__)
+
+
+def _data_columns(parameter_name, n_columns, n_dim):
+    """
+    Get column information for a given parameter.
+
+    Individual functions are adapted to the specific format of the sim_telarray tables.
+
+    Parameters
+    ----------
+    parameter_name: str
+        Model parameter name.
+    n_columns: int
+        Number of columns in the table.
+    n_dim: list
+        List of columns for n-dimensional tables defined by RPOL lines.
+
+    Returns
+    -------
+    list, str
+        List of columns for n-dimensional tables, description.
+    """
+    if parameter_name == "mirror_reflectivity":
+        return _data_columns_mirror_reflectivity(n_columns, n_dim)
+    if parameter_name in ("discriminator_pulse_shape", "fadc_pulse_shape"):
+        return _data_columns_pulse_shape(n_columns)
+    try:
+        return globals()[f"_data_columns_{parameter_name}"]()
+    except KeyError as exc:
+        raise ValueError(
+            f"Unsupported parameter for sim_telarray table reading: {parameter_name}"
+        ) from exc
+
+
+def _data_columns_atmospheric_profile():
+    """Column representation for parameter atmospheric_profile."""
+    return (
+        [
+            {"name": "altitude", "description": "Altitude", "unit": "km"},
+            {"name": "density", "description": "Density", "unit": "g/cm^3"},
+            {"name": "thickness", "description": "Thickness", "unit": "g/cm^2"},
+            {
+                "name": "refractive_index",
+                "description": "Refractive index (n-1)",
+                "unit": None,
+            },
+            {
+                "name": "temperature",
+                "description": "Temperature",
+                "unit": "K",
+            },
+            {
+                "name": "pressure",
+                "description": "Pressure",
+                "unit": "mbar",
+            },
+            {
+                "name": "pw/w",
+                "description": "Partial pressure of water vapor",
+                "unit": None,
+            },
+        ],
+        "Atmospheric profile",
+    )
+
+
+def _data_columns_pm_photoelectron_spectrum():
+    """Column description for parameter pm_photoelectron_spectrum."""
+    return (
+        [
+            {"name": "amplitude", "description": "Signal amplitude", "unit": None},
+            {
+                "name": "response",
+                "description": "response without afterpulsing component",
+                "unit": None,
+            },
+            {
+                "name": "response_with_ap",
+                "description": "response including afterpulsing component",
+                "unit": None,
+            },
+        ],
+        "Photoelectron spectrum",
+    )
+
+
+def _data_columns_quantum_efficiency():
+    """Column description for parameter quantum_efficiency."""
+    return (
+        [
+            {"name": "wavelength", "description": "Wavelength", "unit": "nm"},
+            {
+                "name": "efficiency",
+                "description": "Quantum efficiency",
+                "unit": None,
+            },
+            {
+                "name": "efficiency_rms",
+                "description": "Quantum efficiency (standard deviation)",
+                "unit": None,
+            },
+        ],
+        "Quantum efficiency",
+    )
+
+
+def _data_columns_camera_filter():
+    """Column description for parameter camera_filter."""
+    return (
+        [
+            {"name": "wavelength", "description": "Wavelength", "unit": "nm"},
+            {
+                "name": "transmission",
+                "description": "Average transmission",
+                "unit": None,
+            },
+        ],
+        "Camera window transmission",
+    )
+
+
+def _data_columns_lightguide_efficiency_vs_wavelength():
+    """Column description for parameter lightguide_efficiency_vs_wavelength."""
+    return _data_columns_lightguide_efficiency_vs_incidence_angle()
+
+
+def _data_columns_lightguide_efficiency_vs_incidence_angle():
+    """Column description for (parameter lightguide_efficiency_vs_incidence_angle."""
+    return (
+        [
+            {"name": "angle", "description": "Incidence angle", "unit": "deg"},
+            {
+                "name": "efficiency",
+                "description": "Light guide efficiency",
+                "unit": None,
+            },
+        ],
+        "Light guide efficiency vs incidence angle",
+    )
+
+
+def _data_columns_mirror_reflectivity(n_columns, n_dim):
+    """Column description for parameter mirror_reflectivity."""
+    _columns = [
+        {"name": "wavelength", "description": "Wavelength", "unit": "nm"},
+    ]
+    if n_dim:
+        for angle in n_dim:
+            _columns.append(
+                {
+                    "name": f"reflectivity_{angle}deg",
+                    "description": f"Mirror reflectivity at {angle} deg",
+                    "unit": None,
+                },
+            )
+    else:
+        _columns.append(
+            {
+                "name": "reflectivity",
+                "description": "Mirror reflectivity",
+                "unit": None,
+            },
+        )
+        if n_columns == 3:
+            _columns.append(
+                {
+                    "name": "reflectivity_rms",
+                    "description": "Mirror reflectivity (standard deviation)",
+                    "unit": None,
+                },
+            )
+        if n_columns == 4:
+            _columns.append(
+                {
+                    "name": "reflectivity_min",
+                    "description": "Mirror reflectivity (min)",
+                    "unit": None,
+                },
+            )
+            _columns.append(
+                {
+                    "name": "reflectivity_max",
+                    "description": "Mirror reflectivity (max)",
+                    "unit": None,
+                },
+            )
+
+    return _columns, "Mirror reflectivity"
+
+
+def _data_columns_pulse_shape(n_columns):
+    """Column description for parameters discriminator_pulse_shape, fadc_pulse_shape."""
+    _columns = [
+        {"name": "time", "description": "Time", "unit": "ns"},
+        {
+            "name": "amplitude",
+            "description": "Amplitude",
+            "unit": None,
+        },
+    ]
+    if n_columns == 3:
+        _columns.append(
+            {
+                "name": "amplitude (low gain)",
+                "description": "Amplitude (low gain)",
+                "unit": None,
+            },
+        )
+
+    return _columns, "Pulse shape"
+
+
+def _data_columns_nsb_reference_spectrum():
+    """Column description for parameter nsb_reference_spectrum."""
+    return (
+        [
+            {"name": "wavelength", "description": "Wavelength", "unit": "nm"},
+            {
+                "name": "differential photon rate",
+                "description": "Differential photon rate",
+                "unit": "1.e9 / (nm s m^2 sr)",
+            },
+        ],
+        "NSB reference spectrum",
+    )
+
+
+def read_simtel_table(parameter_name, file_path):
+    """
+    Read sim_telarray table file for a given parameter.
+
+    Parameters
+    ----------
+    parameter_name: str
+        Model parameter name.
+    file_path: Path
+        Name (full path) of the sim_telarray table file.
+
+    Returns
+    -------
+    Table
+        Astropy table.
+    """
+    if parameter_name == "atmospheric_transmission":
+        return _read_simtel_data_for_atmospheric_transmission(file_path)
+
+    rows, meta_from_simtel, n_columns, n_dim = _read_simtel_data(file_path)
+    columns_info, description = _data_columns(parameter_name, n_columns, n_dim)
+
+    rows = _adjust_columns_length(rows, len(columns_info))
+
+    metadata = {
+        "Name": parameter_name,
+        "File": str(file_path),
+        "Description:": description,
+        "Context_from_sim_telarray": meta_from_simtel,
+    }
+
+    table = Table(rows=rows, names=[col["name"] for col in columns_info])
+    for col, info in zip(table.colnames, columns_info):
+        table[col].unit = info.get("unit")
+        table[col].description = info.get("description")
+    table.meta.update(metadata)
+
+    return table
+
+
+def _adjust_columns_length(rows, n_columns):
+    """
+    Adjust row lengths to match the specified column count.
+
+    - Truncate rows with extra columns beyond the specified count 'n_columns'.
+    - Pad shorter rows with zeros.
+    """
+    return [row[:n_columns] + [0.0] * max(0, n_columns - len(row)) for row in rows]
+
+
+def _read_simtel_data(file_path):
+    """
+    Read data, comments, and (if available) axis definition from sim_telarray table.
+
+    Parameters
+    ----------
+    file_path: Path
+        Path to the sim_telarray table file.
+
+    Returns
+    -------
+    str, str, int, str
+        data, metadata (comments), number of columns (max value), n-dimensional axis description.
+    """
+    logger.debug(f"Reading sim_telarray table from {file_path}")
+    meta_lines = []
+    data_lines = []
+    n_dim_axis = None
+    r_pol_axis = None
+
+    lines = gen.read_file_encoded_in_utf_or_latin(file_path)
+
+    for line in lines:
+        stripped = line.strip()
+        if "@RPOL@" in stripped:  # RPOL description for N-dimensional tables
+            match = re.search(r"#@RPOL@\[(\w+)=\]", stripped)
+            if match:
+                r_pol_axis = match.group(1)
+        elif r_pol_axis and r_pol_axis in stripped:  # N-dimensional axis description
+            n_dim_axis = stripped.split("=")[1].split()
+        elif stripped.startswith("#"):  # Metadata
+            meta_lines.append(stripped.lstrip("#").strip())
+        elif stripped:  # Data
+            data_lines.append(stripped.split("%%%")[0].split("#")[0].strip())  # Remove comments
+
+    rows = [[float(part) for part in line.split()] for line in data_lines]
+    n_columns = max(len(row) for row in rows) if rows else 0
+
+    return rows, "\n".join(meta_lines), n_columns, n_dim_axis
+
+
+def _read_simtel_data_for_atmospheric_transmission(file_path):
+    """
+    Read data and comments from sim_telarray table for atmospheric_transmission.
+
+    Parameters
+    ----------
+    file_path: Path
+        Path to the sim_telarray table file.
+
+    Returns
+    -------
+    astropy table
+        Table with atmospheric transmission.
+    """
+    lines = lines = gen.read_file_encoded_in_utf_or_latin(file_path)
+
+    observatory_level, height_bins = _read_header_line_for_atmospheric_transmission(
+        lines, file_path
+    )
+
+    wavelengths = []
+    heights = []
+    extinctions = []
+    meta_lines = []
+
+    for line in lines:
+        if line.startswith("#") or not line.strip():
+            meta_lines.append(line.lstrip("#").strip())
+            continue
+        parts = line.split()
+        try:
+            wl = float(parts[0])
+            for i, height in enumerate(height_bins):
+                extinction_value = float(parts[i + 1])
+                if extinction_value == 99999.0:
+                    continue
+                wavelengths.append(wl)
+                heights.append(height)
+                extinctions.append(extinction_value)
+        except (ValueError, IndexError):
+            logger.debug(f"Skipping malformed line: {line.strip()}")
+
+    table = Table()
+    table["wavelength"] = wavelengths
+    table["altitude"] = heights
+    table["extinction"] = extinctions
+
+    table.meta.update(
+        {
+            "Name": "atmospheric_transmission",
+            "File": str(file_path),
+            "Description": "Atmospheric transmission",
+            "Context_from_sim_telarray": "\n".join(meta_lines),
+            "observatory_level": observatory_level,
+        }
+    )
+
+    return table
+
+
+def _read_header_line_for_atmospheric_transmission(lines, file_path):
+    """Reader observatory level and height bins from header line for atmospheric transmission."""
+    header_line = None
+    observatory_level = None
+    for line in lines:
+        if "H2=" in line and "H1=" in line:
+            match_h2 = re.search(r"H2=\s*([\d.]+)", line)
+            if match_h2:
+                observatory_level = float(match_h2.group(1)) * u.km
+
+            if "H1=" in line:
+                header_line = line.split("H1=")[-1].strip()
+            break
+
+    if header_line is None:
+        raise ValueError(f"Header with 'H1=' not found file {file_path}")
+
+    height_bins = [float(x.replace(",", "")) for x in header_line.split()]
+
+    return observatory_level, height_bins