gammasimtools 0.12.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

simtools/schemas/model_parameter_and_data_schema.metaschema.yml

@@ -115,6 +115,7 @@ definitions:
           - number
           - string
           - boolean
+          - "null"
       table_columns:
         type: array
         items:
@@ -265,12 +266,15 @@ definitions:
       - class
     title: Instrument
   InstrumentTypeElement:
-    type: string
+    type:
+      - string
+      - "null"
     description: "specific type of instrument"
     enum:
       - Atmosphere
       - Observatory
       - None
+      - null
     title: InstrumentTypeElement
   SimulationSoftware:
     type: object
@@ -297,7 +301,9 @@ definitions:
       - $ref: '#/definitions/DataClass'
     title: DataUnion
   Setting:
-    type: string
+    type:
+      - string
+      - "null"
     description: "list of setting workflows"
     enum:
       - SetArrayElementCoordinates
@@ -315,10 +321,12 @@ definitions:
       - SetTelescopeFiducialSphere
       - SetTelescopeShadowingParameters
       - SetTriggerThresholdsFromRateScan
-      - None
+      - null
     title: Setting
   Validation:
-    type: string
+    type:
+      - string
+      - "null"
     description: "list of validation workflows"
     enum:
       - ValidateArrayElementCoordinates
@@ -344,7 +352,7 @@ definitions:
       - ValidateTelescopeStructure
       - ValidateTriggerPerformance
       - ValidationWorkflowMissing
-      - None
+      - null
   TableColumnType:
     type: string
     enum:

simtools/schemas/model_parameters/array_coordinates.schema.yml

@@ -12,7 +12,7 @@ description: |-
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Site
   type:

simtools/schemas/model_parameters/array_layouts.schema.yml

@@ -24,6 +24,9 @@ data:
               type: string
               format: array_element
               description: Name of the array element.
+          comment:
+            type: string
+            description: Additional comment.
         required:
           - name
           - elements

simtools/schemas/model_parameters/asum_shaping.schema.yml

@@ -13,7 +13,7 @@ description: |-
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Camera
   type:

simtools/schemas/model_parameters/atmospheric_profile.schema.yml

@@ -11,7 +11,7 @@ description: Density, thickness, and index of refraction as function of altitude
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Site
   type:

simtools/schemas/model_parameters/camera_config_file.schema.yml

@@ -15,7 +15,7 @@ description: |-
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Camera
   type:

simtools/schemas/model_parameters/camera_degraded_map.schema.yml

@@ -19,7 +19,7 @@ short_description: |-
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Camera
   type:

simtools/schemas/model_parameters/camera_filter.schema.yml

@@ -22,7 +22,7 @@ short_description: |-
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Camera
   type:

simtools/schemas/model_parameters/dsum_shaping.schema.yml

@@ -24,7 +24,7 @@ short_description: |-
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Camera
   type:

simtools/schemas/model_parameters/fadc_dev_pedestal.schema.yml

@@ -25,7 +25,7 @@ data:
 instrument:
   class: Camera
   type:
-    - None
+    - null
 activity:
   setting:
     - SetParameterFromExternal

simtools/schemas/model_parameters/fadc_lg_dev_pedestal.schema.yml

@@ -24,7 +24,7 @@ data:
 instrument:
   class: Camera
   type:
-    - None
+    - null
 activity:
   setting:
     - SetParameterFromExternal

simtools/schemas/model_parameters/fadc_lg_max_sum.schema.yml

@@ -27,12 +27,12 @@ data:
 instrument:
   class: Camera
   type:
-    - None
+    - null
 activity:
   setting:
-    - None
+    - null
   validation:
-    - None
+    - null
 source:
   - Initial instrument setup
 simulation_software:

simtools/schemas/model_parameters/fadc_max_sum.schema.yml

@@ -27,12 +27,12 @@ data:
 instrument:
   class: Camera
   type:
-    - None
+    - null
 activity:
   setting:
-    - None
+    - null
   validation:
-    - None
+    - null
 source:
   - Initial instrument setup
 simulation_software:

simtools/schemas/model_parameters/fake_mirror_list.schema.yml

@@ -15,7 +15,7 @@ description: |-
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Structure
   type:

simtools/schemas/model_parameters/lightguide_efficiency_vs_incidence_angle.schema.yml

@@ -17,7 +17,7 @@ short_description: Lightguide efficiency as a function of incidence angle.
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Camera
   type:

simtools/schemas/model_parameters/lightguide_efficiency_vs_wavelength.schema.yml

@@ -19,7 +19,7 @@ description: Lightguide efficiency for on-axis photons vs wavelength.
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Camera
   type:

simtools/schemas/model_parameters/mirror_list.schema.yml

@@ -11,7 +11,7 @@ description: List of mirror positions, diameters, focal lengths, and shape codes
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Structure
   type:

simtools/schemas/model_parameters/nsb_reference_spectrum.schema.yml

@@ -15,7 +15,7 @@ description: |-
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Site
   type:

simtools/schemas/model_parameters/nsb_skymap.schema.yml

@@ -20,7 +20,7 @@ short_description: Sky map (Az/Alt) of NSB enhancement factors.
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Site
   type:

simtools/schemas/model_parameters/primary_mirror_degraded_map.schema.yml

@@ -22,7 +22,7 @@ short_description: |-
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Structure
   type:

simtools/schemas/model_parameters/primary_mirror_segmentation.schema.yml

@@ -14,7 +14,7 @@ short_description: |-
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Structure
   type:

simtools/schemas/model_parameters/secondary_mirror_degraded_map.schema.yml

@@ -22,7 +22,7 @@ short_description: |-
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Structure
   type:

simtools/schemas/model_parameters/secondary_mirror_segmentation.schema.yml

@@ -17,7 +17,7 @@ short_description: |-
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Structure
   type:

simtools/schemas/plot_configuration.metaschema.yml

@@ -0,0 +1,162 @@
+---
+$schema: http://json-schema.org/draft-06/schema#
+$ref: '#/definitions/SimtoolsPlotConfiguration'
+title: SimPipe plot configuration metaschema
+description: YAML representation of plot configuration metaschema
+version: 0.1.0
+name: plot_configuration.metaschema
+type: object
+additionalProperties: false
+
+definitions:
+  SimtoolsPlotConfiguration:
+    type: object
+    additionalProperties: false
+    properties:
+      CTA_SIMPIPE:
+        "$ref": "#/definitions/CTASIMPIPE"
+      SCHEMA_VERSION:
+        type: string
+        description: "Version of the schema."
+      SCHEMA_URL:
+        type: string
+        format: uri
+        description: "URL of the schema."
+      SCHEMA_NAME:
+        type: string
+        description: "Name of the schema."
+    required:
+      - CTA_SIMPIPE
+    title: "SimtoolsPlotConfiguration"
+  CTASIMPIPE:
+    type: object
+    additionalProperties: false
+    properties:
+      PLOT:
+        "$ref": "#/definitions/Plot"
+    required:
+      - PLOT
+    title: "CTASimpipe"
+  Plot:
+    description: "Plot configuration of simtools data visualization."
+    type: object
+    additionalProperties: false
+    properties:
+      TYPE:
+        type: string
+        description: "Type of the plot (e.g., mirror reflectivity, quantum_efficiency)."
+      TITLE:
+        type: string
+        description: "Title of the plot."
+      XTITLE:
+        type: string
+        description: "Title of x-axis."
+      YTITLE:
+        type: string
+        description: "Title of y-axis."
+      XSCALE:
+        type: string
+        enum: ["linear", "log"]
+        description: "Scale of x-axis (linear or log)."
+      YSCALE:
+        type: string
+        enum: ["linear", "log"]
+        description: "Scale of y-axis (linear or log)."
+      XLIM:
+        type: array
+        items:
+          type: [number, "null"]
+          description: "Limits for x-axis [min, max]. Use null for auto-limit."
+        minItems: 2
+        maxItems: 2
+      YLIM:
+        type: array
+        items:
+          type: [number, "null"]
+          description: "Limits for y-axis [min, max]. Use null for auto-limit."
+        minItems: 2
+        maxItems: 2
+      ERROR_TYPE:
+        type: string
+        enum: ["fill_between", "errorbar", "none"]
+        description: "Type of errors."
+      NO_MARKERS:
+        type: boolean
+        description: "Whether to display markers on data points."
+      PLOT_RATIO:
+        type: boolean
+        description: "Whether to display ratio plot."
+      TABLES:
+        type: array
+        description: "List of tables to plot."
+        items:
+          "$ref": "#/definitions/TableConfig"
+    required:
+      - TYPE
+      - TITLE
+      - XTITLE
+      - YTITLE
+      - TABLES
+    title: "Plot"
+  TableConfig:
+    type: object
+    description: "Configuration for a data table to plot."
+    additionalProperties: false
+    properties:
+      PARAMETER:
+        type: string
+        description: "Parameter name to retrieve data for."
+      FILE_NAME:
+        type: string
+        description: "Path to the data file."
+      TYPE:
+        type: string
+        description: "Type of data file."
+      TELESCOPE:
+        type: string
+        description: "Telescope descriptor to retrieve data for."
+      SITE:
+        type: string
+        description: "Site name (North/South)."
+      MODEL_VERSION:
+        type: string
+        description: "Model version to use."
+      PARAMETER_VERSION:
+        type: string
+        description: "Parameter version to use."
+      LABEL:
+        type: string
+        description: "Label for the plot legend."
+      COLUMN_X:
+        type: string
+        description: "Column name to use for x-axis."
+      COLUMN_Y:
+        type: string
+        description: "Column name to use for y-axis."
+      COLUMN_X_ERR:
+        type: string
+        description: "Column name to use for x-axis error."
+      COLUMN_Y_ERR:
+        type: string
+        description: "Column name to use for y-axis error."
+      NORMALIZE_Y:
+        type: boolean
+        description: "Whether to normalize y values."
+      SELECT_VALUES:
+        type: object
+        description: "Selection criteria for data filtering."
+        properties:
+          COLUMN_NAME:
+            type: string
+            description: "Column name to use for filtering."
+          VALUE:
+            type: [number, string]
+            description: "Value to filter by."
+        required:
+          - COLUMN_NAME
+          - VALUE
+    required:
+      - COLUMN_X
+      - COLUMN_Y
+      - LABEL
+    title: "TableConfig"

simtools/schemas/production_tables.schema.yml

@@ -29,7 +29,7 @@ definitions:
             pattern: '^\d+\.\d+\.\d+$'
         propertyNames:
           description: Allowed parameter name patterns.
-          pattern: '^([A-Za-z](ST|LL)[N,S,x]-\d{2,3}|[A-Za-z](ST|LL)[N,S,x]-(design|FlashCam|NectarCam)|OBS-(North|South)|Dummy-Telescope)$'
+          pattern: '^([A-Za-z](ST|LL|CT)[N,S,x]-\d{2,3}|[A-Za-z](ST|LL|CT)[N,S,x]-(design|FlashCam|NectarCam)|OBS-(North|South)|Dummy-Telescope)$'
       design_model:
         type: object
         description: Design models.

simtools/simtel/simtel_config_reader.py

@@ -4,13 +4,38 @@
 import logging
 import re
 
+import astropy.units as u
 import numpy as np
 
 import simtools.utils.general as gen
+from simtools.utils import names
 
 __all__ = ["SimtelConfigReader"]
 
 
+def get_list_of_simtel_parameters(simtel_config_file):
+    """
+    Return list of simtel parameters found in simtel configuration file.
+
+    Parameters
+    ----------
+    simtel_config_file: str
+        File name for sim_telarray configuration
+
+    Returns
+    -------
+    list
+        List of parameters found in simtel configuration file.
+
+    """
+    simtel_parameter_set = set()
+    with open(simtel_config_file, encoding="utf-8") as file:
+        for line in file:
+            parts_of_lines = re.split(r",\s*|\s+", line.strip())
+            simtel_parameter_set.add(parts_of_lines[1].lower())
+    return sorted(simtel_parameter_set)
+
+
 class SimtelConfigReader:
     """
     Reads model parameters from configuration files and converts to the simtools representation.
@@ -26,7 +51,7 @@ class SimtelConfigReader:
             -C limits=no-internal -C initlist=no-internal -C list=no-internal\
             -C typelist=no-internal -C maximum_telescopes=30\
             -DNSB_AUTOSCALE -DNECTARCAM -DHYPER_LAYOUT\
-            -DNUM_TELESCOPES=30 /dev/null 2>|/dev/null | grep '(@cfg)'
+            -DNUM_TELESCOPES=30 /dev/null 2>|/dev/null | grep '(@cfg)' | sed 's/^(@cfg)
 
     Parameters
     ----------
@@ -61,7 +86,12 @@ class SimtelConfigReader:
             else None
         )
         self.parameter_name = self.schema_dict.get("name") if self.schema_dict else parameter_name
-        self.simtel_parameter_name = self._get_simtel_parameter_name(self.parameter_name)
+        try:
+            self.simtel_parameter_name = names.get_simulation_software_name_from_parameter_name(
+                self.parameter_name
+            ).upper()
+        except (KeyError, AttributeError):
+            self.simtel_parameter_name = self.parameter_name.upper()
         self.simtel_telescope_name = simtel_telescope_name
         self.camera_pixels = camera_pixels
         self.parameter_dict = self.read_simtel_config_file(
@@ -73,7 +103,7 @@ class SimtelConfigReader:
         return data_type == "limits" and self.parameter_dict.get("type") == "bool"
 
     def _get_schema_values(self, data_type):
-        """Check schema values for limits and defaults."""
+        """Check schema values for limits, unit, and default."""
         try:
             if data_type == "limits":
                 _from_schema = [
@@ -82,14 +112,25 @@ class SimtelConfigReader:
                 ]
                 return _from_schema[0] if _from_schema[1] is None else _from_schema
             if len(self.schema_dict["data"]) == 1:
-                return self.schema_dict["data"][0]["default"]
-            return [data.get("default") for data in self.schema_dict["data"]]
+                return self.schema_dict["data"][0].get(data_type)
+            return [data.get(data_type) for data in self.schema_dict["data"]]
         except (KeyError, IndexError):
             return None
 
     @staticmethod
     def _values_match(_from_simtel, _from_schema):
-        """Check if values match (are close for floats)."""
+        """
+        Check if values match (are close for floats).
+
+        Convert where necessary astropy.Quantity to float.
+
+        """
+        if isinstance(_from_simtel, u.Quantity):
+            _from_simtel = _from_simtel.value
+        if isinstance(_from_simtel, np.ndarray) and len(_from_simtel) > 0:
+            _from_simtel = np.array(
+                [v.value if isinstance(v, u.Quantity) else v for v in _from_simtel]
+            )
         try:
             if not isinstance(_from_schema, list | np.ndarray) and _from_simtel == _from_schema:
                 return True
@@ -184,6 +225,7 @@ class SimtelConfigReader:
                     dtype=_para_dict.get("type"),
                     n_dim=_para_dict.get("dimension"),
                     default=_para_dict.get("default"),
+                    is_limit=(key == "limits"),
                 )
             except KeyError:
                 pass
@@ -232,7 +274,7 @@ class SimtelConfigReader:
 
         return column, except_from_all
 
-    def _add_value_from_simtel_cfg(self, column, dtype=None, n_dim=1, default=None):
+    def _add_value_from_simtel_cfg(self, column, dtype=None, n_dim=1, default=None, is_limit=False):
         """
         Extract value(s) from simtel configuration file columns.
 
@@ -276,7 +318,42 @@ class SimtelConfigReader:
         if dtype == "bool":
             column = np.array([bool(int(item)) for item in column])
 
-        return self._process_column(column, dtype)
+        column, ndim = self._process_column(column, dtype)
+        if not is_limit:
+            column = self._add_units(column)
+        return column, ndim
+
+    def _add_units(self, column):
+        """
+        Add units as given in schema file to column.
+
+        Take into account array types and dimensionless units.
+        Ensure that integer values are returned as integers (astropy converts
+        values to floats when multiplying them with units).
+
+        """
+        try:
+            unit = self._get_schema_values("unit")
+        except TypeError:  # no schema defined
+            return column
+        if unit is None or unit == "dimensionless":
+            return column
+
+        if isinstance(column, np.ndarray) and len(column) == len(unit):
+            return np.array(
+                [
+                    col * u.Unit(un) if un != "dimensionless" else col
+                    for col, un in zip(column, unit)
+                ],
+                dtype=object,
+            )
+        if isinstance(unit, str):
+            column_with_unit = column * u.Unit(unit)
+            if isinstance(column, int | np.integer):
+                return u.Quantity(int(column_with_unit.value), unit, dtype=type(column))
+            return column_with_unit
+
+        return None
 
     def _process_column(self, column, dtype):
         """
@@ -324,29 +401,3 @@ class SimtelConfigReader:
         if self.camera_pixels is not None and self.simtel_parameter_name in ["NIGHTSKY_BACKGROUND"]:
             return str(np.dtype(column[0].lower())), self.camera_pixels
         return str(np.dtype(column[0].lower())), int(column[1])
-
-    def _get_simtel_parameter_name(self, parameter_name):
-        """
-        Return parameter name as used in sim_telarray.
-
-        This is documented in the schema file.
-
-        Parameters
-        ----------
-        parameter_name: str
-            Model parameter name (as used in simtools)
-
-        Returns
-        -------
-        str
-            Parameter name as used in sim_telarray.
-
-        """
-        try:
-            for sim_soft in self.schema_dict["simulation_software"]:
-                if sim_soft["name"] == "sim_telarray":
-                    return sim_soft["internal_parameter_name"].upper()
-        except (KeyError, TypeError):
-            pass
-
-        return parameter_name.upper()

simtools/simtel/simtel_table_reader.py

@@ -3,6 +3,7 @@
 
 import logging
 import re
+from pathlib import Path
 
 import astropy.units as u
 from astropy.table import Table
@@ -253,6 +254,9 @@ def read_simtel_table(parameter_name, file_path):
     Table
         Astropy table.
     """
+    if Path(file_path).suffix == ".ecsv":  # table is already in correct format
+        return Table.read(file_path, format="ascii.ecsv")
+
     if parameter_name == "atmospheric_transmission":
         return _read_simtel_data_for_atmospheric_transmission(file_path)