gammasimtools 0.11.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

simtools/schemas/model_parameters/primary_mirror_degraded_map.schema.yml

@@ -22,7 +22,7 @@ short_description: |-
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Structure
   type:

simtools/schemas/model_parameters/primary_mirror_segmentation.schema.yml

@@ -14,7 +14,7 @@ short_description: |-
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Structure
   type:

simtools/schemas/model_parameters/secondary_mirror_degraded_map.schema.yml

@@ -22,7 +22,7 @@ short_description: |-
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Structure
   type:

simtools/schemas/model_parameters/secondary_mirror_segmentation.schema.yml

@@ -17,7 +17,7 @@ short_description: |-
 data:
   - type: file
     unit: dimensionless
-    default: None
+    default: null
 instrument:
   class: Structure
   type:

simtools/schemas/plot_configuration.metaschema.yml

@@ -0,0 +1,162 @@
+---
+$schema: http://json-schema.org/draft-06/schema#
+$ref: '#/definitions/SimtoolsPlotConfiguration'
+title: SimPipe plot configuration metaschema
+description: YAML representation of plot configuration metaschema
+version: 0.1.0
+name: plot_configuration.metaschema
+type: object
+additionalProperties: false
+
+definitions:
+  SimtoolsPlotConfiguration:
+    type: object
+    additionalProperties: false
+    properties:
+      CTA_SIMPIPE:
+        "$ref": "#/definitions/CTASIMPIPE"
+      SCHEMA_VERSION:
+        type: string
+        description: "Version of the schema."
+      SCHEMA_URL:
+        type: string
+        format: uri
+        description: "URL of the schema."
+      SCHEMA_NAME:
+        type: string
+        description: "Name of the schema."
+    required:
+      - CTA_SIMPIPE
+    title: "SimtoolsPlotConfiguration"
+  CTASIMPIPE:
+    type: object
+    additionalProperties: false
+    properties:
+      PLOT:
+        "$ref": "#/definitions/Plot"
+    required:
+      - PLOT
+    title: "CTASimpipe"
+  Plot:
+    description: "Plot configuration of simtools data visualization."
+    type: object
+    additionalProperties: false
+    properties:
+      TYPE:
+        type: string
+        description: "Type of the plot (e.g., mirror reflectivity, quantum_efficiency)."
+      TITLE:
+        type: string
+        description: "Title of the plot."
+      XTITLE:
+        type: string
+        description: "Title of x-axis."
+      YTITLE:
+        type: string
+        description: "Title of y-axis."
+      XSCALE:
+        type: string
+        enum: ["linear", "log"]
+        description: "Scale of x-axis (linear or log)."
+      YSCALE:
+        type: string
+        enum: ["linear", "log"]
+        description: "Scale of y-axis (linear or log)."
+      XLIM:
+        type: array
+        items:
+          type: [number, "null"]
+          description: "Limits for x-axis [min, max]. Use null for auto-limit."
+        minItems: 2
+        maxItems: 2
+      YLIM:
+        type: array
+        items:
+          type: [number, "null"]
+          description: "Limits for y-axis [min, max]. Use null for auto-limit."
+        minItems: 2
+        maxItems: 2
+      ERROR_TYPE:
+        type: string
+        enum: ["fill_between", "errorbar", "none"]
+        description: "Type of errors."
+      NO_MARKERS:
+        type: boolean
+        description: "Whether to display markers on data points."
+      PLOT_RATIO:
+        type: boolean
+        description: "Whether to display ratio plot."
+      TABLES:
+        type: array
+        description: "List of tables to plot."
+        items:
+          "$ref": "#/definitions/TableConfig"
+    required:
+      - TYPE
+      - TITLE
+      - XTITLE
+      - YTITLE
+      - TABLES
+    title: "Plot"
+  TableConfig:
+    type: object
+    description: "Configuration for a data table to plot."
+    additionalProperties: false
+    properties:
+      PARAMETER:
+        type: string
+        description: "Parameter name to retrieve data for."
+      FILE_NAME:
+        type: string
+        description: "Path to the data file."
+      TYPE:
+        type: string
+        description: "Type of data file."
+      TELESCOPE:
+        type: string
+        description: "Telescope descriptor to retrieve data for."
+      SITE:
+        type: string
+        description: "Site name (North/South)."
+      MODEL_VERSION:
+        type: string
+        description: "Model version to use."
+      PARAMETER_VERSION:
+        type: string
+        description: "Parameter version to use."
+      LABEL:
+        type: string
+        description: "Label for the plot legend."
+      COLUMN_X:
+        type: string
+        description: "Column name to use for x-axis."
+      COLUMN_Y:
+        type: string
+        description: "Column name to use for y-axis."
+      COLUMN_X_ERR:
+        type: string
+        description: "Column name to use for x-axis error."
+      COLUMN_Y_ERR:
+        type: string
+        description: "Column name to use for y-axis error."
+      NORMALIZE_Y:
+        type: boolean
+        description: "Whether to normalize y values."
+      SELECT_VALUES:
+        type: object
+        description: "Selection criteria for data filtering."
+        properties:
+          COLUMN_NAME:
+            type: string
+            description: "Column name to use for filtering."
+          VALUE:
+            type: [number, string]
+            description: "Value to filter by."
+        required:
+          - COLUMN_NAME
+          - VALUE
+    required:
+      - COLUMN_X
+      - COLUMN_Y
+      - LABEL
+    title: "TableConfig"

simtools/schemas/production_tables.schema.yml

@@ -29,7 +29,7 @@ definitions:
             pattern: '^\d+\.\d+\.\d+$'
         propertyNames:
           description: Allowed parameter name patterns.
-          pattern: '^([A-Za-z](ST|LL)[N,S,x]-\d{2,3}|[A-Za-z](ST|LL)[N,S,x]-(design|FlashCam|NectarCam)|OBS-(North|South)|Dummy-Telescope)$'
+          pattern: '^([A-Za-z](ST|LL|CT)[N,S,x]-\d{2,3}|[A-Za-z](ST|LL|CT)[N,S,x]-(design|FlashCam|NectarCam)|OBS-(North|South)|Dummy-Telescope)$'
       design_model:
         type: object
         description: Design models.

simtools/simtel/simtel_config_reader.py

@@ -4,13 +4,38 @@
 import logging
 import re
 
+import astropy.units as u
 import numpy as np
 
 import simtools.utils.general as gen
+from simtools.utils import names
 
 __all__ = ["SimtelConfigReader"]
 
 
+def get_list_of_simtel_parameters(simtel_config_file):
+    """
+    Return list of simtel parameters found in simtel configuration file.
+
+    Parameters
+    ----------
+    simtel_config_file: str
+        File name for sim_telarray configuration
+
+    Returns
+    -------
+    list
+        List of parameters found in simtel configuration file.
+
+    """
+    simtel_parameter_set = set()
+    with open(simtel_config_file, encoding="utf-8") as file:
+        for line in file:
+            parts_of_lines = re.split(r",\s*|\s+", line.strip())
+            simtel_parameter_set.add(parts_of_lines[1].lower())
+    return sorted(simtel_parameter_set)
+
+
 class SimtelConfigReader:
     """
     Reads model parameters from configuration files and converts to the simtools representation.
@@ -26,7 +51,7 @@ class SimtelConfigReader:
             -C limits=no-internal -C initlist=no-internal -C list=no-internal\
             -C typelist=no-internal -C maximum_telescopes=30\
             -DNSB_AUTOSCALE -DNECTARCAM -DHYPER_LAYOUT\
-            -DNUM_TELESCOPES=30 /dev/null 2>|/dev/null | grep '(@cfg)'
+            -DNUM_TELESCOPES=30 /dev/null 2>|/dev/null | grep '(@cfg)' | sed 's/^(@cfg)
 
     Parameters
     ----------
@@ -61,7 +86,12 @@ class SimtelConfigReader:
             else None
         )
         self.parameter_name = self.schema_dict.get("name") if self.schema_dict else parameter_name
-        self.simtel_parameter_name = self._get_simtel_parameter_name(self.parameter_name)
+        try:
+            self.simtel_parameter_name = names.get_simulation_software_name_from_parameter_name(
+                self.parameter_name
+            ).upper()
+        except (KeyError, AttributeError):
+            self.simtel_parameter_name = self.parameter_name.upper()
         self.simtel_telescope_name = simtel_telescope_name
         self.camera_pixels = camera_pixels
         self.parameter_dict = self.read_simtel_config_file(
@@ -73,7 +103,7 @@ class SimtelConfigReader:
         return data_type == "limits" and self.parameter_dict.get("type") == "bool"
 
     def _get_schema_values(self, data_type):
-        """Check schema values for limits and defaults."""
+        """Check schema values for limits, unit, and default."""
         try:
             if data_type == "limits":
                 _from_schema = [
@@ -82,14 +112,25 @@ class SimtelConfigReader:
                 ]
                 return _from_schema[0] if _from_schema[1] is None else _from_schema
             if len(self.schema_dict["data"]) == 1:
-                return self.schema_dict["data"][0]["default"]
-            return [data.get("default") for data in self.schema_dict["data"]]
+                return self.schema_dict["data"][0].get(data_type)
+            return [data.get(data_type) for data in self.schema_dict["data"]]
         except (KeyError, IndexError):
             return None
 
     @staticmethod
     def _values_match(_from_simtel, _from_schema):
-        """Check if values match (are close for floats)."""
+        """
+        Check if values match (are close for floats).
+
+        Convert where necessary astropy.Quantity to float.
+
+        """
+        if isinstance(_from_simtel, u.Quantity):
+            _from_simtel = _from_simtel.value
+        if isinstance(_from_simtel, np.ndarray) and len(_from_simtel) > 0:
+            _from_simtel = np.array(
+                [v.value if isinstance(v, u.Quantity) else v for v in _from_simtel]
+            )
         try:
             if not isinstance(_from_schema, list | np.ndarray) and _from_simtel == _from_schema:
                 return True
@@ -184,6 +225,7 @@ class SimtelConfigReader:
                     dtype=_para_dict.get("type"),
                     n_dim=_para_dict.get("dimension"),
                     default=_para_dict.get("default"),
+                    is_limit=(key == "limits"),
                 )
             except KeyError:
                 pass
@@ -232,7 +274,7 @@ class SimtelConfigReader:
 
         return column, except_from_all
 
-    def _add_value_from_simtel_cfg(self, column, dtype=None, n_dim=1, default=None):
+    def _add_value_from_simtel_cfg(self, column, dtype=None, n_dim=1, default=None, is_limit=False):
         """
         Extract value(s) from simtel configuration file columns.
 
@@ -276,7 +318,42 @@ class SimtelConfigReader:
         if dtype == "bool":
             column = np.array([bool(int(item)) for item in column])
 
-        return self._process_column(column, dtype)
+        column, ndim = self._process_column(column, dtype)
+        if not is_limit:
+            column = self._add_units(column)
+        return column, ndim
+
+    def _add_units(self, column):
+        """
+        Add units as given in schema file to column.
+
+        Take into account array types and dimensionless units.
+        Ensure that integer values are returned as integers (astropy converts
+        values to floats when multiplying them with units).
+
+        """
+        try:
+            unit = self._get_schema_values("unit")
+        except TypeError:  # no schema defined
+            return column
+        if unit is None or unit == "dimensionless":
+            return column
+
+        if isinstance(column, np.ndarray) and len(column) == len(unit):
+            return np.array(
+                [
+                    col * u.Unit(un) if un != "dimensionless" else col
+                    for col, un in zip(column, unit)
+                ],
+                dtype=object,
+            )
+        if isinstance(unit, str):
+            column_with_unit = column * u.Unit(unit)
+            if isinstance(column, int | np.integer):
+                return u.Quantity(int(column_with_unit.value), unit, dtype=type(column))
+            return column_with_unit
+
+        return None
 
     def _process_column(self, column, dtype):
         """
@@ -324,29 +401,3 @@ class SimtelConfigReader:
         if self.camera_pixels is not None and self.simtel_parameter_name in ["NIGHTSKY_BACKGROUND"]:
             return str(np.dtype(column[0].lower())), self.camera_pixels
         return str(np.dtype(column[0].lower())), int(column[1])
-
-    def _get_simtel_parameter_name(self, parameter_name):
-        """
-        Return parameter name as used in sim_telarray.
-
-        This is documented in the schema file.
-
-        Parameters
-        ----------
-        parameter_name: str
-            Model parameter name (as used in simtools)
-
-        Returns
-        -------
-        str
-            Parameter name as used in sim_telarray.
-
-        """
-        try:
-            for sim_soft in self.schema_dict["simulation_software"]:
-                if sim_soft["name"] == "sim_telarray":
-                    return sim_soft["internal_parameter_name"].upper()
-        except (KeyError, TypeError):
-            pass
-
-        return parameter_name.upper()

simtools/simtel/simtel_table_reader.py

@@ -3,6 +3,7 @@
 
 import logging
 import re
+from pathlib import Path
 
 import astropy.units as u
 from astropy.table import Table
@@ -253,6 +254,9 @@ def read_simtel_table(parameter_name, file_path):
     Table
         Astropy table.
     """
+    if Path(file_path).suffix == ".ecsv":  # table is already in correct format
+        return Table.read(file_path, format="ascii.ecsv")
+
     if parameter_name == "atmospheric_transmission":
         return _read_simtel_data_for_atmospheric_transmission(file_path)
 

simtools/utils/general.py

@@ -2,6 +2,7 @@
 
 import copy
 import datetime
+import glob
 import json
 import logging
 import os
@@ -455,6 +456,34 @@ def find_file(name, loc):
     raise FileNotFoundError(msg)
 
 
+def resolve_file_patterns(file_names):
+    """
+    Return a list of files names from string, list, or wildcard pattern.
+
+    Parameters
+    ----------
+    file_names: str, list
+        File names to be searched for (wildcards allowed).
+
+    Returns
+    -------
+    list
+        List of file names found.
+    """
+    if file_names is None:
+        raise ValueError("No file list provided.")
+    if not isinstance(file_names, list):
+        file_names = [file_names]
+
+    _files = []
+    for file_name in file_names:
+        # use glob (and not Path.glob) for easier wildcard handling
+        _files.extend(Path(f) for f in glob.glob(str(file_name), recursive=True))  # noqa: PTH207
+    if not _files:
+        raise FileNotFoundError(f"No files found: {file_names}")
+    return _files
+
+
 def get_log_excerpt(log_file, n_last_lines=30):
     """
     Get an excerpt from a log file, namely the n_last_lines of the file.
@@ -696,16 +725,28 @@ def validate_data_type(reference_dtype, value=None, dtype=None, allow_subtypes=T
     ):
         return True
 
-    if np.issubdtype(dtype, np.bool_) and reference_dtype in ("boolean", "bool"):
-        return True
+    if reference_dtype in ("boolean", "bool"):
+        return _is_valid_boolean_type(dtype, value)
 
-    if np.issubdtype(dtype, np.integer) and (
-        np.issubdtype(reference_dtype, np.integer) or np.issubdtype(reference_dtype, np.floating)
-    ):
-        return True
+    return _is_valid_numeric_type(dtype, reference_dtype)
 
-    if np.issubdtype(dtype, np.floating) and np.issubdtype(reference_dtype, np.floating):
+
+def _is_valid_boolean_type(dtype, value):
+    """Check if dtype or value is a valid boolean type."""
+    if value in {0, 1}:
         return True
+    return np.issubdtype(dtype, np.bool_)
+
+
+def _is_valid_numeric_type(dtype, reference_dtype):
+    """Check if dtype is a valid numeric type compared to reference_dtype."""
+    if np.issubdtype(dtype, np.integer):
+        return np.issubdtype(reference_dtype, np.integer) or np.issubdtype(
+            reference_dtype, np.floating
+        )
+
+    if np.issubdtype(dtype, np.floating):
+        return np.issubdtype(reference_dtype, np.floating)
 
     return False
 
@@ -851,8 +892,8 @@ def get_structure_array_from_table(table, column_names):
         Structured array containing the table data.
     """
     return np.array(
-        list(zip(*[np.array(table[col]) for col in column_names])),
-        dtype=[(col, np.array(table[col]).dtype) for col in column_names],
+        list(zip(*[np.array(table[col]) for col in column_names if col in table.colnames])),
+        dtype=[(col, np.array(table[col]).dtype) for col in column_names if col in table.colnames],
     )
 
 

simtools/utils/names.py

@@ -353,6 +353,8 @@ def get_array_element_type_from_name(array_element_name):
     """
     Get array element type from array element name (e.g "MSTN" from "MSTN-01").
 
+    For sites, return site name.
+
     Parameters
     ----------
     array_element_name: str
@@ -363,7 +365,10 @@ def get_array_element_type_from_name(array_element_name):
     str
         Array element type.
     """
-    return _validate_name(array_element_name.split("-")[0], array_elements())
+    try:  # e.g. instrument is 'North' as given for the site parameters
+        return validate_site_name(array_element_name)
+    except ValueError:  # any other telescope or calibration device
+        return _validate_name(array_element_name.split("-")[0], array_elements())
 
 
 def get_array_element_id_from_name(array_element_name):
@@ -460,7 +465,7 @@ def get_collection_name_from_array_element_name(array_element_name, array_elemen
     """
     try:
         return array_elements()[get_array_element_type_from_name(array_element_name)]["collection"]
-    except ValueError as exc:
+    except (ValueError, KeyError) as exc:
         if array_elements_only:
             raise ValueError(f"Invalid array element name {array_element_name}") from exc
     try:

simtools/utils/value_conversion.py

@@ -123,13 +123,15 @@ def _split_value_is_quantity(value, is_integer=False):
 
 
 def _split_value_is_string(value, is_integer=False):
-    """Split vale and unit for a string."""
+    """Split value and unit for a string."""
     if value.isdigit():  # single integer value
         return int(value), None
     try:  # single value with/without unit
-        return int(u.Quantity(value).value) if is_integer else u.Quantity(value).value, str(
-            u.Quantity(value).unit
-        )
+        quantity = u.Quantity(value)
+        unit = str(quantity.unit)
+        if unit.isdigit():  # cases where numbers are wrongly identified as units
+            raise ValueError
+        return (int(quantity.value), unit) if is_integer else (quantity.value, unit)
     except ValueError:
         return _split_value_is_list(gen.convert_string_to_list(value), is_integer)
     except TypeError:  # string value (not numerical)

simtools/visualization/plot_tables.py

@@ -2,11 +2,11 @@
 """Plot tabular data."""
 
 import numpy as np
+from astropy.table import Table
 
 import simtools.utils.general as gen
+from simtools.db import db_handler
 from simtools.io_operations import legacy_data_handler
-from simtools.model.site_model import SiteModel
-from simtools.model.telescope_model import TelescopeModel
 from simtools.visualization import visualize
 
 
@@ -50,11 +50,15 @@ def read_table_data(config, db_config):
         if "parameter" in _config:
             table = _read_table_from_model_database(_config, db_config)
         elif "file_name" in _config:
-            table = legacy_data_handler.read_legacy_data_as_table(
-                _config["file_name"], _config["type"]
-            )
+            if "legacy" in _config.get("type", ""):
+                table = legacy_data_handler.read_legacy_data_as_table(
+                    _config["file_name"], _config["type"]
+                )
+            else:
+                table = Table.read(_config["file_name"], format="ascii.ecsv")
         else:
             raise ValueError("No table data defined in configuration.")
+
         if _config.get("normalize_y"):
             table[_config["column_y"]] = (
                 table[_config["column_y"]] / table[_config["column_y"]].max()
@@ -66,7 +70,13 @@ def read_table_data(config, db_config):
                 _config["select_values"]["value"],
             )
         data[_config["label"]] = gen.get_structure_array_from_table(
-            table, [_config["column_x"], _config["column_y"]]
+            table,
+            [
+                _config["column_x"],
+                _config["column_y"],
+                _config.get("column_x_err"),
+                _config.get("column_y_err"),
+            ],
         )
     return data
 
@@ -85,20 +95,15 @@ def _read_table_from_model_database(table_config, db_config):
     Table
         Astropy table.
     """
-    if "telescope" in table_config:
-        model = TelescopeModel(
-            site=table_config["site"],
-            telescope_name=table_config["telescope"],
-            model_version=table_config["model_version"],
-            mongo_db_config=db_config,
-        )
-    else:
-        model = SiteModel(
-            site=table_config["site"],
-            model_version=table_config["model_version"],
-            mongo_db_config=db_config,
-        )
-    return model.get_model_file_as_table(table_config["parameter"])
+    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+    return db.export_model_file(
+        parameter=table_config["parameter"],
+        site=table_config["site"],
+        array_element_name=table_config.get("telescope"),
+        parameter_version=table_config.get("parameter_version"),
+        model_version=table_config.get("model_version"),
+        export_file_as_table=True,
+    )
 
 
 def _select_values_from_table(table, column_name, value):