gammasimtools 0.12.0__py3-none-any.whl → 0.14.0__py3-none-any.whl

simtools/testing/configuration.py

@@ -1,6 +1,7 @@
 """Integration test configuration."""
 
 import logging
+import os
 from pathlib import Path
 
 import yaml
@@ -14,6 +15,10 @@ class VersionError(Exception):
     """Raise if model version requested is not supported."""
 
 
+class ProductionDBError(Exception):
+    """Raise if production db is used."""
+
+
 def get_list_of_test_configurations(config_files):
     """
     Return list of test configuration dictionaries or test names.
@@ -100,6 +105,7 @@ def configure(config, tmp_test_directory, request):
 
     if "CONFIGURATION" in config:
         _skip_test_for_model_version(config, model_version_requested)
+        _skip_test_for_production_db(config)
 
         config_file, config_string, config_file_model_version = _prepare_test_options(
             config["CONFIGURATION"],
@@ -130,6 +136,17 @@ def _skip_test_for_model_version(config, model_version_requested):
         )
 
 
+def _skip_test_for_production_db(config):
+    """Skip test if production db is used."""
+    simtools_db_server = os.environ.get("SIMTOOLS_DB_SERVER")
+    if (
+        simtools_db_server
+        and config.get("SKIP_FOR_PRODUCTION_DB")
+        and "db.zeuthen.desy.de" in simtools_db_server
+    ):
+        raise ProductionDBError("Production database used for this test")
+
+
 def _prepare_test_options(config, output_path, model_version=None):
     """
     Prepare test configuration.

simtools/utils/general.py

@@ -2,6 +2,7 @@
 
 import copy
 import datetime
+import glob
 import json
 import logging
 import os
@@ -179,13 +180,17 @@ def collect_data_from_file(file_name, yaml_document=None):
         return collect_data_from_http(file_name)
 
     suffix = Path(file_name).suffix.lower()
-    with open(file_name, encoding="utf-8") as file:
-        if suffix == ".json":
-            return json.load(file)
-        if suffix == ".list":
-            return [line.strip() for line in file.readlines()]
-        if suffix in [".yml", ".yaml"]:
-            return _collect_data_from_yaml_file(file, file_name, yaml_document)
+    try:
+        with open(file_name, encoding="utf-8") as file:
+            if suffix == ".json":
+                return json.load(file)
+            if suffix == ".list":
+                return [line.strip() for line in file.readlines()]
+            if suffix in [".yml", ".yaml"]:
+                return _collect_data_from_yaml_file(file, file_name, yaml_document)
+    # broad exception to catch all possible errors in reading the file
+    except Exception as exc:  # pylint: disable=broad-except
+        raise type(exc)(f"Failed to read file {file_name}: {exc}") from exc
     return None
 
 
@@ -455,6 +460,34 @@ def find_file(name, loc):
     raise FileNotFoundError(msg)
 
 
+def resolve_file_patterns(file_names):
+    """
+    Return a list of files names from string, list, or wildcard pattern.
+
+    Parameters
+    ----------
+    file_names: str, list
+        File names to be searched for (wildcards allowed).
+
+    Returns
+    -------
+    list
+        List of file names found.
+    """
+    if file_names is None:
+        raise ValueError("No file list provided.")
+    if not isinstance(file_names, list):
+        file_names = [file_names]
+
+    _files = []
+    for file_name in file_names:
+        # use glob (and not Path.glob) for easier wildcard handling
+        _files.extend(Path(f) for f in glob.glob(str(file_name), recursive=True))  # noqa: PTH207
+    if not _files:
+        raise FileNotFoundError(f"No files found: {file_names}")
+    return _files
+
+
 def get_log_excerpt(log_file, n_last_lines=30):
     """
     Get an excerpt from a log file, namely the n_last_lines of the file.
@@ -696,16 +729,28 @@ def validate_data_type(reference_dtype, value=None, dtype=None, allow_subtypes=T
     ):
         return True
 
-    if np.issubdtype(dtype, np.bool_) and reference_dtype in ("boolean", "bool"):
-        return True
+    if reference_dtype in ("boolean", "bool"):
+        return _is_valid_boolean_type(dtype, value)
+
+    return _is_valid_numeric_type(dtype, reference_dtype)
 
-    if np.issubdtype(dtype, np.integer) and (
-        np.issubdtype(reference_dtype, np.integer) or np.issubdtype(reference_dtype, np.floating)
-    ):
-        return True
 
-    if np.issubdtype(dtype, np.floating) and np.issubdtype(reference_dtype, np.floating):
+def _is_valid_boolean_type(dtype, value):
+    """Check if dtype or value is a valid boolean type."""
+    if value in {0, 1}:
         return True
+    return np.issubdtype(dtype, np.bool_)
+
+
+def _is_valid_numeric_type(dtype, reference_dtype):
+    """Check if dtype is a valid numeric type compared to reference_dtype."""
+    if np.issubdtype(dtype, np.integer):
+        return np.issubdtype(reference_dtype, np.integer) or np.issubdtype(
+            reference_dtype, np.floating
+        )
+
+    if np.issubdtype(dtype, np.floating):
+        return np.issubdtype(reference_dtype, np.floating)
 
     return False
 
@@ -801,6 +846,28 @@ def _load_yaml_using_astropy(file):
     return astropy_yaml.load(file)
 
 
+def is_utf8_file(file_name):
+    """
+    Check if a file is encoded in UTF-8.
+
+    Parameters
+    ----------
+    file_name: str, Path
+        Name of the file to be checked.
+
+    Returns
+    -------
+    bool
+        True if the file is encoded in UTF-8, False otherwise.
+    """
+    try:
+        with open(file_name, encoding="utf-8") as file:
+            file.read()
+        return True
+    except UnicodeDecodeError:
+        return False
+
+
 def read_file_encoded_in_utf_or_latin(file_name):
     """
     Read a file encoded in UTF-8 or Latin-1.
@@ -851,8 +918,8 @@ def get_structure_array_from_table(table, column_names):
         Structured array containing the table data.
     """
     return np.array(
-        list(zip(*[np.array(table[col]) for col in column_names])),
-        dtype=[(col, np.array(table[col]).dtype) for col in column_names],
+        list(zip(*[np.array(table[col]) for col in column_names if col in table.colnames])),
+        dtype=[(col, np.array(table[col]).dtype) for col in column_names if col in table.colnames],
     )
 
 

simtools/utils/geometry.py

@@ -161,3 +161,22 @@ def rotate(x, y, rotation_around_z_axis, rotation_around_y_axis=0):
     y_trans = x * np.sin(rotation_around_z_axis) + y * np.cos(rotation_around_z_axis)
 
     return x_trans, y_trans
+
+
+def calculate_circular_mean(angles):
+    """
+    Calculate circular mean of angles in radians.
+
+    Parameters
+    ----------
+    angles: numpy.array
+        Array of angles in radians.
+
+    Returns
+    -------
+    float
+        Circular mean of the angles.
+    """
+    sin_sum = np.sum(np.sin(angles))
+    cos_sum = np.sum(np.cos(angles))
+    return np.arctan2(sin_sum, cos_sum)

simtools/utils/names.py

@@ -353,6 +353,8 @@ def get_array_element_type_from_name(array_element_name):
     """
     Get array element type from array element name (e.g "MSTN" from "MSTN-01").
 
+    For sites, return site name.
+
     Parameters
     ----------
     array_element_name: str
@@ -363,7 +365,10 @@ def get_array_element_type_from_name(array_element_name):
     str
         Array element type.
     """
-    return _validate_name(array_element_name.split("-")[0], array_elements())
+    try:  # e.g. instrument is 'North' as given for the site parameters
+        return validate_site_name(array_element_name)
+    except ValueError:  # any other telescope or calibration device
+        return _validate_name(array_element_name.split("-")[0], array_elements())
 
 
 def get_array_element_id_from_name(array_element_name):
@@ -460,7 +465,7 @@ def get_collection_name_from_array_element_name(array_element_name, array_elemen
     """
     try:
         return array_elements()[get_array_element_type_from_name(array_element_name)]["collection"]
-    except ValueError as exc:
+    except (ValueError, KeyError) as exc:
         if array_elements_only:
             raise ValueError(f"Invalid array element name {array_element_name}") from exc
     try:
@@ -508,6 +513,7 @@ def get_collection_name_from_parameter_name(parameter_name):
 def get_simulation_software_name_from_parameter_name(
     parameter_name,
     simulation_software="sim_telarray",
+    set_meta_parameter=False,
 ):
     """
     Get the name used in the given simulation software from the model parameter name.
@@ -521,6 +527,8 @@ def get_simulation_software_name_from_parameter_name(
         Model parameter name.
     simulation_software: str
         Simulation software name.
+    set_meta_parameter: bool
+        If True, return values with 'set_meta_parameter' field set to True.
 
     Returns
     -------
@@ -532,7 +540,10 @@ def get_simulation_software_name_from_parameter_name(
         raise KeyError(f"Parameter {parameter_name} without schema definition")
 
     for software in _parameter.get("simulation_software", []):
-        if software.get("name") == simulation_software:
+        if (
+            software.get("name") == simulation_software
+            and software.get("set_meta_parameter", False) is set_meta_parameter
+        ):
             return software.get("internal_parameter_name", parameter_name)
 
     return None

simtools/visualization/plot_tables.py

@@ -2,11 +2,11 @@
 """Plot tabular data."""
 
 import numpy as np
+from astropy.table import Table
 
 import simtools.utils.general as gen
+from simtools.db import db_handler
 from simtools.io_operations import legacy_data_handler
-from simtools.model.site_model import SiteModel
-from simtools.model.telescope_model import TelescopeModel
 from simtools.visualization import visualize
 
 
@@ -50,11 +50,15 @@ def read_table_data(config, db_config):
         if "parameter" in _config:
             table = _read_table_from_model_database(_config, db_config)
         elif "file_name" in _config:
-            table = legacy_data_handler.read_legacy_data_as_table(
-                _config["file_name"], _config["type"]
-            )
+            if "legacy" in _config.get("type", ""):
+                table = legacy_data_handler.read_legacy_data_as_table(
+                    _config["file_name"], _config["type"]
+                )
+            else:
+                table = Table.read(_config["file_name"], format="ascii.ecsv")
         else:
             raise ValueError("No table data defined in configuration.")
+
         if _config.get("normalize_y"):
             table[_config["column_y"]] = (
                 table[_config["column_y"]] / table[_config["column_y"]].max()
@@ -66,7 +70,13 @@ def read_table_data(config, db_config):
                 _config["select_values"]["value"],
             )
         data[_config["label"]] = gen.get_structure_array_from_table(
-            table, [_config["column_x"], _config["column_y"]]
+            table,
+            [
+                _config["column_x"],
+                _config["column_y"],
+                _config.get("column_x_err"),
+                _config.get("column_y_err"),
+            ],
         )
     return data
 
@@ -85,20 +95,15 @@ def _read_table_from_model_database(table_config, db_config):
     Table
         Astropy table.
     """
-    if "telescope" in table_config:
-        model = TelescopeModel(
-            site=table_config["site"],
-            telescope_name=table_config["telescope"],
-            model_version=table_config["model_version"],
-            mongo_db_config=db_config,
-        )
-    else:
-        model = SiteModel(
-            site=table_config["site"],
-            model_version=table_config["model_version"],
-            mongo_db_config=db_config,
-        )
-    return model.get_model_file_as_table(table_config["parameter"])
+    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+    return db.export_model_file(
+        parameter=table_config["parameter"],
+        site=table_config["site"],
+        array_element_name=table_config.get("telescope"),
+        parameter_version=table_config.get("parameter_version"),
+        model_version=table_config.get("model_version"),
+        export_file_as_table=True,
+    )
 
 
 def _select_values_from_table(table, column_name, value):

simtools/visualization/visualize.py

@@ -144,22 +144,23 @@ def _add_unit(title, array):
     str
         Title with units.
     """
-    unit = ""
-    if isinstance(array, u.Quantity):
-        unit = str(array[0].unit)
-        if len(unit) > 0:
-            unit = f" [{unit}]"
-        if re.search(r"\d", unit):
-            unit = re.sub(r"(\d)", r"^\1", unit)
-            unit = unit.replace("[", r"$[").replace("]", r"]$")
-        if "[" in title and "]" in title:
-            _logger.warning(
-                "Tried to add a unit from astropy.unit, "
-                "but axis already has an explicit unit. Left axis title as is."
-            )
-            unit = ""
+    if title and "[" in title and "]" in title:
+        _logger.warning(
+            "Tried to add a unit from astropy.unit, "
+            "but axis already has an explicit unit. Left axis title as is."
+        )
+        return title
+
+    if not isinstance(array, u.Quantity) or not str(array[0].unit):
+        return title
 
-    return f"{title}{unit}"
+    unit = str(array[0].unit)
+    unit_str = f" [{unit}]"
+    if re.search(r"\d", unit_str):
+        unit_str = re.sub(r"(\d)", r"^\1", unit_str)
+        unit_str = unit_str.replace("[", r"[$").replace("]", r"$]")
+
+    return f"{title}{unit_str}"
 
 
 def set_style(palette="default", big_plot=False):
@@ -196,7 +197,7 @@ def set_style(palette="default", big_plot=False):
         raise KeyError(f"palette must be one of {', '.join(COLORS)}")
 
     fontsize = {"default": 17, "big_plot": 30}
-    markersize = {"default": 8, "big_plot": 18}
+    markersize = {"default": 6, "big_plot": 18}
     plot_size = "big_plot" if big_plot else "default"
 
     plt.rc("lines", linewidth=2, markersize=markersize[plot_size])
@@ -418,27 +419,74 @@ def setup_plot(kwargs, plot_ratio, plot_difference):
     return fig, ax1, gs
 
 
+def _plot_error_plots(kwargs, data_now, x_col, y_col, x_err_col, y_err_col, color):
+    """Plot error plots."""
+    if kwargs.get("error_type") == "fill_between" and y_err_col:
+        plt.fill_between(
+            data_now[x_col],
+            data_now[y_col] - data_now[y_err_col],
+            data_now[y_col] + data_now[y_err_col],
+            alpha=0.2,
+            color=color,
+        )
+
+    if kwargs.get("error_type") == "errorbar" and (x_err_col or y_err_col):
+        plt.errorbar(
+            data_now[x_col],
+            data_now[y_col],
+            xerr=data_now[x_err_col] if x_err_col else None,
+            yerr=data_now[y_err_col] if y_err_col else None,
+            fmt=".",
+            color=color,
+        )
+
+
+def _get_data_columns(data_now):
+    """Return data columns depending on availability."""
+    columns = data_now.dtype.names
+    assert len(columns) >= 2, "Input array must have at least two columns with titles."
+    x_col, y_col = columns[:2]
+    if len(columns) == 3:
+        x_err_col = None
+        y_err_col = columns[2]
+    elif len(columns) == 4:
+        x_err_col = columns[2]
+        y_err_col = columns[3]
+    else:
+        x_err_col = None
+        y_err_col = None
+    return x_col, y_col, x_err_col, y_err_col
+
+
 def plot_main_data(data_dict, kwargs, plot_args):
     """Plot the main data."""
     for label, data_now in data_dict.items():
-        assert len(data_now.dtype.names) == 2, "Input array must have two columns with titles."
-        x_column_name, y_column_name = data_now.dtype.names[0], data_now.dtype.names[1]
-        x_title = kwargs["xtitle"] if kwargs.get("xtitle") else x_column_name
-        y_title = kwargs["ytitle"] if kwargs.get("ytitle") else y_column_name
-        x_title_unit = _add_unit(x_title, data_now[x_column_name])
-        y_title_unit = _add_unit(y_title, data_now[y_column_name])
-        plt.plot(data_now[x_column_name], data_now[y_column_name], label=label, **plot_args)
+        x_col, y_col, x_err_col, y_err_col = _get_data_columns(data_now)
+
+        x_title = kwargs.get("xtitle", x_col)
+        y_title = kwargs.get("ytitle", y_col)
+
+        x_title_unit = _add_unit(x_title, data_now[x_col])
+        y_title_unit = _add_unit(y_title, data_now[y_col])
+
+        (line,) = plt.plot(data_now[x_col], data_now[y_col], label=label, **plot_args)
+        color = line.get_color()
+
+        _plot_error_plots(kwargs, data_now, x_col, y_col, x_err_col, y_err_col, color)
 
     plt.xscale(kwargs["xscale"])
     plt.yscale(kwargs["yscale"])
     plt.xlim(kwargs["xlim"])
     plt.ylim(kwargs["ylim"])
     plt.ylabel(y_title_unit)
+
     if not (kwargs["plot_ratio"] or kwargs["plot_difference"]):
         plt.xlabel(x_title_unit)
+
     if kwargs["title"]:
         plt.title(kwargs["title"], y=1.02)
-    if "_default" not in list(data_dict.keys()) and not kwargs["no_legend"]:
+
+    if "_default" not in data_dict and not kwargs["no_legend"]:
         plt.legend()
 
 
@@ -692,7 +740,7 @@ def plot_array(
 
 
 def initialize_tel_counters():
-    return {one_telescope: 0 for one_telescope in names.get_list_of_array_element_types()}
+    return dict.fromkeys(names.get_list_of_array_element_types(), 0)
 
 
 def get_rotated_positions(telescopes, rotate_angle):

simtools/_dev_version/__init__.py

@@ -1,9 +0,0 @@
-# Try to use setuptools_scm to get the current version; this is only used
-# in development installations from the git repository.
-# see src/simtools/version.py for details
-try:
-    from setuptools_scm import get_version
-
-    version = get_version(root="../../..", relative_to=__file__)
-except Exception as e:
-    raise ImportError(f"setuptools_scm broken or not installed: {e}")

simtools/applications/production_derive_limits.py

@@ -1,95 +0,0 @@
-#!/usr/bin/python3
-
-r"""
-Derives the limits for energy, radial distance, and viewcone to be used in CORSIKA simulations.
-
-The limits are derived based on the event loss fraction specified by the user.
-
-Command line arguments
-----------------------
-event_data_file (str, required)
-    Path to the file containing the event data.
-loss_fraction (float, required)
-    Fraction of events to be lost.
-
-
-Example
--------
-Derive limits for a given file with a specified loss fraction.
-
-.. code-block:: console
-
-    simtools-production-derive-limits\\
-        --event_data_file path/to/event_data_file.hdf5 \\
-        --loss_fraction 1e-6
-"""
-
-import logging
-
-import simtools.utils.general as gen
-from simtools.configuration import configurator
-from simtools.io_operations.hdf5_handler import read_hdf5
-from simtools.production_configuration.limits_calculation import LimitCalculator
-
-_logger = logging.getLogger(__name__)
-
-
-def _parse():
-    """
-    Parse command line configuration.
-
-    Parameters
-    ----------
-    event_data_file: str
-        The event data file.
-    loss_fraction: float
-        Loss fraction of events for limit derivation.
-
-    Returns
-    -------
-    CommandLineParser
-        Command line parser object
-
-    """
-    config = configurator.Configurator(
-        description="Derive limits for energy, radial distance, and viewcone."
-    )
-    config.parser.add_argument(
-        "--event_data_file",
-        type=str,
-        required=True,
-        help="Path to the event data file containing the event data.",
-    )
-    config.parser.add_argument(
-        "--loss_fraction", type=float, required=True, help="Fraction of events to be lost."
-    )
-    return config.initialize(db_config=False)
-
-
-def main():
-    """Derive limits for energy, radial distance, and viewcone."""
-    args_dict, _ = _parse()
-
-    logger = logging.getLogger()
-    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
-
-    event_data_file_path = args_dict["event_data_file"]
-    loss_fraction = args_dict["loss_fraction"]
-
-    _logger.info(f"Loading event data file: {event_data_file_path}")
-    tables = read_hdf5(event_data_file_path)
-
-    calculator = LimitCalculator(tables)
-
-    lower_energy_limit = calculator.compute_lower_energy_limit(loss_fraction)
-    _logger.info(f"Lower energy threshold: {lower_energy_limit}")
-
-    upper_radial_distance = calculator.compute_upper_radial_distance(loss_fraction)
-    _logger.info(f"Upper radius threshold: {upper_radial_distance}")
-
-    viewcone = calculator.compute_viewcone(loss_fraction)
-    _logger.info(f"Viewcone radius: {viewcone}")
-
-
-if __name__ == "__main__":
-    main()