gammasimtools 0.12.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

simtools/utils/general.py

@@ -2,6 +2,7 @@
 
 import copy
 import datetime
+import glob
 import json
 import logging
 import os
@@ -455,6 +456,34 @@ def find_file(name, loc):
     raise FileNotFoundError(msg)
 
 
+def resolve_file_patterns(file_names):
+    """
+    Return a list of files names from string, list, or wildcard pattern.
+
+    Parameters
+    ----------
+    file_names: str, list
+        File names to be searched for (wildcards allowed).
+
+    Returns
+    -------
+    list
+        List of file names found.
+    """
+    if file_names is None:
+        raise ValueError("No file list provided.")
+    if not isinstance(file_names, list):
+        file_names = [file_names]
+
+    _files = []
+    for file_name in file_names:
+        # use glob (and not Path.glob) for easier wildcard handling
+        _files.extend(Path(f) for f in glob.glob(str(file_name), recursive=True))  # noqa: PTH207
+    if not _files:
+        raise FileNotFoundError(f"No files found: {file_names}")
+    return _files
+
+
 def get_log_excerpt(log_file, n_last_lines=30):
     """
     Get an excerpt from a log file, namely the n_last_lines of the file.
@@ -696,16 +725,28 @@ def validate_data_type(reference_dtype, value=None, dtype=None, allow_subtypes=T
     ):
         return True
 
-    if np.issubdtype(dtype, np.bool_) and reference_dtype in ("boolean", "bool"):
-        return True
+    if reference_dtype in ("boolean", "bool"):
+        return _is_valid_boolean_type(dtype, value)
 
-    if np.issubdtype(dtype, np.integer) and (
-        np.issubdtype(reference_dtype, np.integer) or np.issubdtype(reference_dtype, np.floating)
-    ):
-        return True
+    return _is_valid_numeric_type(dtype, reference_dtype)
 
-    if np.issubdtype(dtype, np.floating) and np.issubdtype(reference_dtype, np.floating):
+
+def _is_valid_boolean_type(dtype, value):
+    """Check if dtype or value is a valid boolean type."""
+    if value in {0, 1}:
         return True
+    return np.issubdtype(dtype, np.bool_)
+
+
+def _is_valid_numeric_type(dtype, reference_dtype):
+    """Check if dtype is a valid numeric type compared to reference_dtype."""
+    if np.issubdtype(dtype, np.integer):
+        return np.issubdtype(reference_dtype, np.integer) or np.issubdtype(
+            reference_dtype, np.floating
+        )
+
+    if np.issubdtype(dtype, np.floating):
+        return np.issubdtype(reference_dtype, np.floating)
 
     return False
 
@@ -851,8 +892,8 @@ def get_structure_array_from_table(table, column_names):
         Structured array containing the table data.
     """
     return np.array(
-        list(zip(*[np.array(table[col]) for col in column_names])),
-        dtype=[(col, np.array(table[col]).dtype) for col in column_names],
+        list(zip(*[np.array(table[col]) for col in column_names if col in table.colnames])),
+        dtype=[(col, np.array(table[col]).dtype) for col in column_names if col in table.colnames],
     )
 
 

simtools/utils/names.py

@@ -353,6 +353,8 @@ def get_array_element_type_from_name(array_element_name):
     """
     Get array element type from array element name (e.g "MSTN" from "MSTN-01").
 
+    For sites, return site name.
+
     Parameters
     ----------
     array_element_name: str
@@ -363,7 +365,10 @@ def get_array_element_type_from_name(array_element_name):
     str
         Array element type.
     """
-    return _validate_name(array_element_name.split("-")[0], array_elements())
+    try:  # e.g. instrument is 'North' as given for the site parameters
+        return validate_site_name(array_element_name)
+    except ValueError:  # any other telescope or calibration device
+        return _validate_name(array_element_name.split("-")[0], array_elements())
 
 
 def get_array_element_id_from_name(array_element_name):
@@ -460,7 +465,7 @@ def get_collection_name_from_array_element_name(array_element_name, array_elemen
     """
     try:
         return array_elements()[get_array_element_type_from_name(array_element_name)]["collection"]
-    except ValueError as exc:
+    except (ValueError, KeyError) as exc:
         if array_elements_only:
             raise ValueError(f"Invalid array element name {array_element_name}") from exc
     try:

simtools/visualization/plot_tables.py

@@ -2,11 +2,11 @@
 """Plot tabular data."""
 
 import numpy as np
+from astropy.table import Table
 
 import simtools.utils.general as gen
+from simtools.db import db_handler
 from simtools.io_operations import legacy_data_handler
-from simtools.model.site_model import SiteModel
-from simtools.model.telescope_model import TelescopeModel
 from simtools.visualization import visualize
 
 
@@ -50,11 +50,15 @@ def read_table_data(config, db_config):
         if "parameter" in _config:
             table = _read_table_from_model_database(_config, db_config)
         elif "file_name" in _config:
-            table = legacy_data_handler.read_legacy_data_as_table(
-                _config["file_name"], _config["type"]
-            )
+            if "legacy" in _config.get("type", ""):
+                table = legacy_data_handler.read_legacy_data_as_table(
+                    _config["file_name"], _config["type"]
+                )
+            else:
+                table = Table.read(_config["file_name"], format="ascii.ecsv")
         else:
             raise ValueError("No table data defined in configuration.")
+
         if _config.get("normalize_y"):
             table[_config["column_y"]] = (
                 table[_config["column_y"]] / table[_config["column_y"]].max()
@@ -66,7 +70,13 @@ def read_table_data(config, db_config):
                 _config["select_values"]["value"],
             )
         data[_config["label"]] = gen.get_structure_array_from_table(
-            table, [_config["column_x"], _config["column_y"]]
+            table,
+            [
+                _config["column_x"],
+                _config["column_y"],
+                _config.get("column_x_err"),
+                _config.get("column_y_err"),
+            ],
         )
     return data
 
@@ -85,20 +95,15 @@ def _read_table_from_model_database(table_config, db_config):
     Table
         Astropy table.
     """
-    if "telescope" in table_config:
-        model = TelescopeModel(
-            site=table_config["site"],
-            telescope_name=table_config["telescope"],
-            model_version=table_config["model_version"],
-            mongo_db_config=db_config,
-        )
-    else:
-        model = SiteModel(
-            site=table_config["site"],
-            model_version=table_config["model_version"],
-            mongo_db_config=db_config,
-        )
-    return model.get_model_file_as_table(table_config["parameter"])
+    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+    return db.export_model_file(
+        parameter=table_config["parameter"],
+        site=table_config["site"],
+        array_element_name=table_config.get("telescope"),
+        parameter_version=table_config.get("parameter_version"),
+        model_version=table_config.get("model_version"),
+        export_file_as_table=True,
+    )
 
 
 def _select_values_from_table(table, column_name, value):

simtools/visualization/visualize.py

@@ -144,22 +144,23 @@ def _add_unit(title, array):
     str
         Title with units.
     """
-    unit = ""
-    if isinstance(array, u.Quantity):
-        unit = str(array[0].unit)
-        if len(unit) > 0:
-            unit = f" [{unit}]"
-        if re.search(r"\d", unit):
-            unit = re.sub(r"(\d)", r"^\1", unit)
-            unit = unit.replace("[", r"$[").replace("]", r"]$")
-        if "[" in title and "]" in title:
-            _logger.warning(
-                "Tried to add a unit from astropy.unit, "
-                "but axis already has an explicit unit. Left axis title as is."
-            )
-            unit = ""
+    if title and "[" in title and "]" in title:
+        _logger.warning(
+            "Tried to add a unit from astropy.unit, "
+            "but axis already has an explicit unit. Left axis title as is."
+        )
+        return title
+
+    if not isinstance(array, u.Quantity) or not str(array[0].unit):
+        return title
 
-    return f"{title}{unit}"
+    unit = str(array[0].unit)
+    unit_str = f" [{unit}]"
+    if re.search(r"\d", unit_str):
+        unit_str = re.sub(r"(\d)", r"^\1", unit_str)
+        unit_str = unit_str.replace("[", r"[$").replace("]", r"$]")
+
+    return f"{title}{unit_str}"
 
 
 def set_style(palette="default", big_plot=False):
@@ -418,27 +419,74 @@ def setup_plot(kwargs, plot_ratio, plot_difference):
     return fig, ax1, gs
 
 
+def _plot_error_plots(kwargs, data_now, x_col, y_col, x_err_col, y_err_col, color):
+    """Plot error plots."""
+    if kwargs.get("error_type") == "fill_between" and y_err_col:
+        plt.fill_between(
+            data_now[x_col],
+            data_now[y_col] - data_now[y_err_col],
+            data_now[y_col] + data_now[y_err_col],
+            alpha=0.2,
+            color=color,
+        )
+
+    if kwargs.get("error_type") == "errorbar" and (x_err_col or y_err_col):
+        plt.errorbar(
+            data_now[x_col],
+            data_now[y_col],
+            xerr=data_now[x_err_col] if x_err_col else None,
+            yerr=data_now[y_err_col] if y_err_col else None,
+            fmt=".",
+            color=color,
+        )
+
+
+def _get_data_columns(data_now):
+    """Return data columns depending on availability."""
+    columns = data_now.dtype.names
+    assert len(columns) >= 2, "Input array must have at least two columns with titles."
+    x_col, y_col = columns[:2]
+    if len(columns) == 3:
+        x_err_col = None
+        y_err_col = columns[2]
+    elif len(columns) == 4:
+        x_err_col = columns[2]
+        y_err_col = columns[3]
+    else:
+        x_err_col = None
+        y_err_col = None
+    return x_col, y_col, x_err_col, y_err_col
+
+
 def plot_main_data(data_dict, kwargs, plot_args):
     """Plot the main data."""
     for label, data_now in data_dict.items():
-        assert len(data_now.dtype.names) == 2, "Input array must have two columns with titles."
-        x_column_name, y_column_name = data_now.dtype.names[0], data_now.dtype.names[1]
-        x_title = kwargs["xtitle"] if kwargs.get("xtitle") else x_column_name
-        y_title = kwargs["ytitle"] if kwargs.get("ytitle") else y_column_name
-        x_title_unit = _add_unit(x_title, data_now[x_column_name])
-        y_title_unit = _add_unit(y_title, data_now[y_column_name])
-        plt.plot(data_now[x_column_name], data_now[y_column_name], label=label, **plot_args)
+        x_col, y_col, x_err_col, y_err_col = _get_data_columns(data_now)
+
+        x_title = kwargs.get("xtitle", x_col)
+        y_title = kwargs.get("ytitle", y_col)
+
+        x_title_unit = _add_unit(x_title, data_now[x_col])
+        y_title_unit = _add_unit(y_title, data_now[y_col])
+
+        (line,) = plt.plot(data_now[x_col], data_now[y_col], label=label, **plot_args)
+        color = line.get_color()
+
+        _plot_error_plots(kwargs, data_now, x_col, y_col, x_err_col, y_err_col, color)
 
     plt.xscale(kwargs["xscale"])
     plt.yscale(kwargs["yscale"])
     plt.xlim(kwargs["xlim"])
     plt.ylim(kwargs["ylim"])
     plt.ylabel(y_title_unit)
+
     if not (kwargs["plot_ratio"] or kwargs["plot_difference"]):
         plt.xlabel(x_title_unit)
+
     if kwargs["title"]:
         plt.title(kwargs["title"], y=1.02)
-    if "_default" not in list(data_dict.keys()) and not kwargs["no_legend"]:
+
+    if "_default" not in data_dict and not kwargs["no_legend"]:
         plt.legend()
 
 

simtools/_dev_version/__init__.py

@@ -1,9 +0,0 @@
-# Try to use setuptools_scm to get the current version; this is only used
-# in development installations from the git repository.
-# see src/simtools/version.py for details
-try:
-    from setuptools_scm import get_version
-
-    version = get_version(root="../../..", relative_to=__file__)
-except Exception as e:
-    raise ImportError(f"setuptools_scm broken or not installed: {e}")

simtools/production_configuration/limits_calculation.py

@@ -1,202 +0,0 @@
-"""Calculate the thresholds for energy, radial distance, and viewcone."""
-
-import astropy.units as u
-import numpy as np
-
-
-class LimitCalculator:
-    """
-    Compute thresholds/limits for energy, radial distance, and viewcone.
-
-    Histograms are generated with simtools-generate-simtel-array-histograms with --hdf5 flag.
-
-    Event data is read from the generated HDF5 file from the following tables:
-    - angle_to_observing_position__triggered_showers_ for the viewcone limit.
-    - event_weight__ra3d__log10_e__ for the energy and radial distance limit.
-
-
-    Parameters
-    ----------
-    event_data_file : list of astropy.table.Table
-        The list of tables containing the event data.
-    """
-
-    def __init__(self, event_data_file_tables):
-        """
-        Initialize the LimitCalculator with the given event data file.
-
-        Parameters
-        ----------
-        event_data_file : list of astropy.table.Table
-            The list of tables containing the event data.
-        """
-        self.angle_to_observing_position__triggered_showers_ = None
-        self.event_weight__ra3d__log10_e__ = None
-
-        for table in event_data_file_tables:
-            if (
-                "Title" in table.meta
-                and table.meta["Title"] == "angle_to_observing_position__triggered_showers_"
-            ):
-                self.angle_to_observing_position__triggered_showers_ = table
-            elif "Title" in table.meta and table.meta["Title"] == "event_weight__ra3d__log10_e__":
-                self.event_weight__ra3d__log10_e__ = table
-
-    def _compute_limits(
-        self, event_weight_array, bin_edges, loss_fraction, axis=0, limit_type="lower"
-    ):
-        """
-        Compute the limits based on the loss fraction.
-
-        Parameters
-        ----------
-        event_weight_array : np.ndarray
-            Array of event weights.
-        bin_edges : np.ndarray
-            Array of bin edges.
-        loss_fraction : float
-            Fraction of events to be lost.
-        axis : int, optional
-            Axis along which to sum the event weights. Default is 0.
-        limit_type : str, optional
-            Type of limit ('lower' or 'upper'). Default is 'lower'.
-
-        Returns
-        -------
-        int
-            Bin index where the threshold is reached.
-        float
-            Bin edge value corresponding to the threshold.
-        """
-        projection = np.sum(event_weight_array, axis=axis)
-        bin_edge_value = None
-        cumulative_sum = None
-        if limit_type == "upper":
-            cumulative_sum = np.cumsum(projection)
-
-        elif limit_type == "lower":
-            cumulative_sum = np.cumsum(projection[::-1])
-
-        total_events = np.sum(projection)
-        threshold = (1 - loss_fraction) * total_events
-        bin_index = np.searchsorted(cumulative_sum, threshold)
-        if limit_type == "upper":
-            bin_edge_value = bin_edges[bin_index]
-        elif limit_type == "lower":
-            bin_edge_value = bin_edges[-bin_index]
-        return bin_index, bin_edge_value
-
-    def get_bin_edges_and_units(self, table, axis="x"):
-        """
-        Extract bin edges and units from the table metadata.
-
-        Parameters
-        ----------
-        table : astropy.table.Table
-            Table containing the event data.
-
-        Returns
-        -------
-        tuple
-            Tuple containing the bin edges and their units.
-        """
-        bin_edges = table.meta[f"{axis}_bin_edges"]
-        try:
-            bin_edges_unit = table.meta[f"{axis}_bin_edges_unit"]
-        except KeyError:
-            bin_edges_unit = ""
-        return bin_edges, bin_edges_unit
-
-    def compute_lower_energy_limit(self, loss_fraction):
-        """
-        Compute the lower energy limit in TeV based on the event loss fraction.
-
-        Parameters
-        ----------
-        loss_fraction : float
-            Fraction of events to be lost.
-
-        Returns
-        -------
-        astropy.units.Quantity
-            Lower energy limit.
-        """
-        event_weight_array = np.column_stack(
-            [
-                self.event_weight__ra3d__log10_e__[name]
-                for name in self.event_weight__ra3d__log10_e__.dtype.names
-            ]
-        )
-        bin_edges, bin_edges_unit = self.get_bin_edges_and_units(
-            self.event_weight__ra3d__log10_e__, axis="y"
-        )
-        if bin_edges_unit == "":
-            bin_edges_unit = "TeV"
-        _, lower_bin_edge_value = self._compute_limits(
-            event_weight_array, bin_edges, loss_fraction, axis=0, limit_type="lower"
-        )
-        return (10**lower_bin_edge_value) * u.Unit(bin_edges_unit)
-
-    def compute_upper_radial_distance(self, loss_fraction):
-        """
-        Compute the upper radial distance based on the event loss fraction.
-
-        Parameters
-        ----------
-        loss_fraction : float
-            Fraction of events to be lost.
-
-        Returns
-        -------
-        astropy.units.Quantity
-            Upper radial distance in m.
-        """
-        event_weight_array = np.column_stack(
-            [
-                self.event_weight__ra3d__log10_e__[name]
-                for name in self.event_weight__ra3d__log10_e__.dtype.names
-            ]
-        )
-        bin_edges, bin_edges_unit = self.get_bin_edges_and_units(
-            self.event_weight__ra3d__log10_e__, axis="x"
-        )
-        if bin_edges_unit == "":
-            bin_edges_unit = "m"
-        _, upper_bin_edge_value = self._compute_limits(
-            event_weight_array, bin_edges, loss_fraction, axis=1, limit_type="upper"
-        )
-        return upper_bin_edge_value * u.Unit(bin_edges_unit)
-
-    def compute_viewcone(self, loss_fraction):
-        """
-        Compute the viewcone based on the event loss fraction.
-
-        Parameters
-        ----------
-        loss_fraction : float
-            Fraction of events to be lost.
-
-        Returns
-        -------
-        astropy.units.Quantity
-            Viewcone radius in degrees.
-        """
-        angle_to_observing_position__triggered_showers = np.column_stack(
-            [
-                self.angle_to_observing_position__triggered_showers_[name]
-                for name in self.angle_to_observing_position__triggered_showers_.dtype.names
-            ]
-        )
-        bin_edges, bin_edges_unit = self.get_bin_edges_and_units(
-            self.angle_to_observing_position__triggered_showers_, axis="x"
-        )
-        if bin_edges_unit == "":
-            bin_edges_unit = "deg"
-        _, upper_bin_edge_value = self._compute_limits(
-            angle_to_observing_position__triggered_showers,
-            bin_edges,
-            loss_fraction,
-            axis=0,
-            limit_type="upper",
-        )
-        return upper_bin_edge_value * u.Unit(bin_edges_unit)