gammasimtools 0.17.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

simtools/utils/general.py

@@ -1,12 +1,10 @@
 """General functions useful across different parts of the code."""
 
-import copy
 import datetime
 import glob
-import json
 import logging
 import os
-import tempfile
+import tarfile
 import time
 import urllib.error
 import urllib.request
@@ -14,54 +12,21 @@ from pathlib import Path
 from urllib.parse import urlparse
 
 import numpy as np
-import yaml
 
 __all__ = [
-    "InvalidConfigDataError",
     "change_dict_keys_case",
     "clear_default_sim_telarray_cfg_directories",
-    "collect_data_from_file",
     "collect_final_lines",
     "collect_kwargs",
     "get_log_excerpt",
     "get_log_level_from_user",
     "remove_substring_recursively_from_dict",
     "set_default_kwargs",
-    "sort_arrays",
 ]
 
 _logger = logging.getLogger(__name__)
 
 
-class InvalidConfigDataError(Exception):
-    """Exception for invalid configuration data."""
-
-
-def join_url_or_path(url_or_path, *args):
-    """
-    Join URL or path with additional subdirectories and file.
-
-    This is the equivalent to Path.join(), with extended functionality
-    working also for URLs.
-
-    Parameters
-    ----------
-    url_or_path: str or Path
-        URL or path to be extended.
-    args: list
-        Additional arguments to be added to the URL or path.
-
-    Returns
-    -------
-    str or Path
-        Extended URL or path.
-
-    """
-    if "://" in str(url_or_path):
-        return "/".join([url_or_path.rstrip("/"), *args])
-    return Path(url_or_path).joinpath(*args)
-
-
 def is_url(url):
     """
     Check if a string is a valid URL.
@@ -106,104 +71,6 @@ def url_exists(url):
         return False
 
 
-def collect_data_from_http(url):
-    """
-    Download yaml or json file from url and return it contents as dict.
-
-    File is downloaded as a temporary file and deleted afterwards.
-
-    Parameters
-    ----------
-    url: str
-        URL of the yaml/json file.
-
-    Returns
-    -------
-    dict
-        Dictionary containing the file content.
-
-    Raises
-    ------
-    TypeError
-        If url is not a valid URL.
-    FileNotFoundError
-        If downloading the yaml file fails.
-
-    """
-    try:
-        with tempfile.NamedTemporaryFile(mode="w+t") as tmp_file:
-            urllib.request.urlretrieve(url, tmp_file.name)
-            data = _collect_data_from_different_file_types(
-                tmp_file, url, Path(url).suffix.lower(), None
-            )
-    except TypeError as exc:
-        raise TypeError(f"Invalid url {url}") from exc
-    except urllib.error.HTTPError as exc:
-        raise FileNotFoundError(f"Failed to download file from {url}") from exc
-
-    _logger.debug(f"Downloaded file from {url}")
-    return data
-
-
-def collect_data_from_file(file_name, yaml_document=None):
-    """
-    Collect data from file based on its extension.
-
-    Parameters
-    ----------
-    file_name: str
-        Name of the yaml/json/ascii file.
-    yaml_document: None, int
-        Return list of yaml documents or a single document (for yaml files with several documents).
-
-    Returns
-    -------
-    data: dict or list
-        Data as dict or list.
-    """
-    if is_url(file_name):
-        return collect_data_from_http(file_name)
-
-    suffix = Path(file_name).suffix.lower()
-    try:
-        with open(file_name, encoding="utf-8") as file:
-            return _collect_data_from_different_file_types(file, file_name, suffix, yaml_document)
-    # broad exception to catch all possible errors in reading the file
-    except Exception as exc:  # pylint: disable=broad-except
-        raise type(exc)(f"Failed to read file {file_name}: {exc}") from exc
-    return None
-
-
-def _collect_data_from_different_file_types(file, file_name, suffix, yaml_document):
-    """Collect data from different file types."""
-    if suffix == ".json":
-        return json.load(file)
-    if suffix in (".list", ".txt"):
-        return [line.strip() for line in file.readlines()]
-    if suffix in [".yml", ".yaml"]:
-        return _collect_data_from_yaml_file(file, file_name, yaml_document)
-    raise TypeError(f"File type {suffix} not supported.")
-
-
-def _collect_data_from_yaml_file(file, file_name, yaml_document):
-    """Collect data from a yaml file (allow for multi-document yaml files)."""
-    try:
-        return yaml.safe_load(file)
-    except yaml.constructor.ConstructorError:
-        return _load_yaml_using_astropy(file)
-    except yaml.composer.ComposerError:
-        pass
-    file.seek(0)
-    if yaml_document is None:
-        return list(yaml.safe_load_all(file))
-    try:
-        return list(yaml.safe_load_all(file))[yaml_document]
-    except IndexError as exc:
-        raise InvalidConfigDataError(
-            f"YAML file {file_name} does not contain {yaml_document} documents."
-        ) from exc
-
-
 def collect_kwargs(label, in_kwargs):
     """
     Collect kwargs of the type label_* and return them as a dict.
@@ -324,10 +191,8 @@ def get_log_level_from_user(log_level):
     possible_levels = {
         "info": logging.INFO,
         "debug": logging.DEBUG,
-        "warn": logging.WARNING,
         "warning": logging.WARNING,
         "error": logging.ERROR,
-        "critical": logging.CRITICAL,
     }
     try:
         log_level_lower = log_level.lower()
@@ -342,26 +207,27 @@ def get_log_level_from_user(log_level):
     return possible_levels[log_level_lower]
 
 
-def copy_as_list(value):
+def ensure_iterable(value):
     """
-    Copy value and, if it is not a list, turn it into a list with a single entry.
+    Return input value as iterable.
+
+    - Single values will return as a list with a single element.
+    - None values will return as empty list.
+    - Values of list or tuple type are not changed.
 
     Parameters
     ----------
-    value single variable of any type or list
+    value: any
+        Input value to be converted to a iterable.
 
     Returns
     -------
-    value: list
-        Copy of value if it is a list of [value] otherwise.
+    list or tuple
+        Converted value as list or tuple.
     """
-    if isinstance(value, str):
-        return [value]
-
-    try:
-        return list(value)
-    except TypeError:
-        return [value]
+    if not value:
+        return []
+    return value if isinstance(value, list | tuple) else [value]
 
 
 def program_is_executable(program):
@@ -398,17 +264,17 @@ def _search_directory(directory, filename, rec=False):
         _logger.debug(f"Directory {directory} does not exist")
         return None
 
-    file = Path(directory).joinpath(filename)
-    if file.exists():
+    _file = Path(directory).joinpath(filename)
+    if _file.exists():
         _logger.debug(f"File {filename} found in {directory}")
-        return file
+        return _file
 
     if rec:
         for subdir in Path(directory).iterdir():
             if subdir.is_dir():
-                file = _search_directory(subdir, filename, True)
-                if file:
-                    return file
+                _file = _search_directory(subdir, filename, True)
+                if _file:
+                    return _file
     return None
 
 
@@ -436,15 +302,15 @@ def find_file(name, loc):
     all_locations = [loc] if not isinstance(loc, list) else loc
 
     # Searching file locally
-    file = _search_directory(".", name)
-    if file:
-        return file
+    _file = _search_directory(".", name)
+    if _file:
+        return _file
 
     # Searching file in given locations
     for location in all_locations:
-        file = _search_directory(location, name, True)
-        if file:
-            return file
+        _file = _search_directory(location, name, True)
+        if _file:
+            return _file
 
     msg = f"File {name} could not be found in {all_locations}"
     _logger.error(msg)
@@ -479,6 +345,29 @@ def resolve_file_patterns(file_names):
     return _files
 
 
+def pack_tar_file(tar_file_name, file_list):
+    """
+    Pack files into a tar.gz archive.
+
+    Parameters
+    ----------
+    tar_file_name: str
+        Name of the output tar.gz file.
+    file_list: list
+        List of files to include in the archive.
+    """
+    file_list = [Path(f) for f in file_list]
+    base = Path(os.path.commonpath([f.resolve() for f in file_list]))
+    base_resolved = base.resolve()
+    for f in file_list:
+        if not f.is_file() or not f.resolve().is_relative_to(base_resolved):
+            raise ValueError(f"Unsafe file path: {f}")
+
+    with tarfile.open(tar_file_name, "w:gz") as tar:
+        for file in file_list:
+            tar.add(file, arcname=file.name)
+
+
 def get_log_excerpt(log_file, n_last_lines=30):
     """
     Get an excerpt from a log file, namely the n_last_lines of the file.
@@ -617,29 +506,6 @@ def remove_substring_recursively_from_dict(data_dict, substring="\n"):
     return data_dict
 
 
-def sort_arrays(*args):
-    """Sort arrays.
-
-    Parameters
-    ----------
-    *args
-        Arguments to be sorted.
-
-    Returns
-    -------
-    list
-        Sorted args.
-    """
-    if len(args) == 0:
-        return args
-    order_array = copy.copy(args[0])
-    new_args = []
-    for arg in args:
-        _, value = zip(*sorted(zip(order_array, arg)))
-        new_args.append(list(value))
-    return new_args
-
-
 def user_confirm():
     """
     Ask the user to enter y or n (case-insensitive) on the command line.
@@ -816,123 +682,125 @@ def convert_string_to_list(data_string, is_float=True, force_comma_separation=Fa
     return data_string
 
 
-def _load_yaml_using_astropy(file):
+def get_structure_array_from_table(table, column_names):
     """
-    Load a yaml file using astropy's yaml loader.
+    Get a structured array from an astropy table for a selected list of columns.
 
     Parameters
     ----------
-    file: file
-        File to be loaded.
+    table: astropy.table.Table
+        Table to be converted.
+    column_names: list
+        List of column names to be included in the structured array.
 
     Returns
     -------
-    dict
-        Dictionary containing the file content.
+    numpy.ndarray
+        Structured array containing the table data.
     """
-    # pylint: disable=import-outside-toplevel
-    import astropy.io.misc.yaml as astropy_yaml
-
-    file.seek(0)
-    return astropy_yaml.load(file)
+    return np.array(
+        list(zip(*[np.array(table[col]) for col in column_names if col in table.colnames])),
+        dtype=[(col, np.array(table[col]).dtype) for col in column_names if col in table.colnames],
+    )
 
 
-def is_utf8_file(file_name):
+def convert_keys_in_dict_to_lowercase(data):
     """
-    Check if a file is encoded in UTF-8.
+    Recursively convert all dictionary keys to lowercase.
 
     Parameters
     ----------
-    file_name: str, Path
-        Name of the file to be checked.
+    data: dict
+        Dictionary to be converted.
 
     Returns
     -------
-    bool
-        True if the file is encoded in UTF-8, False otherwise.
+    dict
+        Dictionary with all keys converted to lowercase.
     """
-    try:
-        with open(file_name, encoding="utf-8") as file:
-            file.read()
-        return True
-    except UnicodeDecodeError:
-        return False
+    if isinstance(data, dict):
+        return {k.lower(): convert_keys_in_dict_to_lowercase(v) for k, v in data.items()}
+    if isinstance(data, list):
+        return [convert_keys_in_dict_to_lowercase(i) for i in data]
+    return data
 
 
-def read_file_encoded_in_utf_or_latin(file_name):
+def remove_key_from_dict(data, key_to_remove):
     """
-    Read a file encoded in UTF-8 or Latin-1.
+    Remove a specific key from a dictionary recursively.
 
     Parameters
     ----------
-    file_name: str
-        Name of the file to be read.
+    data: dict
+        Dictionary to be processed.
+    key_to_remove: str
+        Key to be removed from the dictionary.
 
     Returns
     -------
-    list
-        List of lines read from the file.
-
-    Raises
-    ------
-    UnicodeDecodeError
-        If the file cannot be decoded using UTF-8 or Latin-1.
+    dict
+        Dictionary with the specified key removed.
     """
-    try:
-        with open(file_name, encoding="utf-8") as file:
-            lines = file.readlines()
-    except UnicodeDecodeError:
-        logging.debug("Unable to decode file using UTF-8. Trying Latin-1.")
-        try:
-            with open(file_name, encoding="latin-1") as file:
-                lines = file.readlines()
-        except UnicodeDecodeError as exc:
-            raise UnicodeDecodeError("Unable to decode file using UTF-8 or Latin-1.") from exc
+    if isinstance(data, dict):
+        return {
+            k: remove_key_from_dict(v, key_to_remove) for k, v in data.items() if k != key_to_remove
+        }
+    if isinstance(data, list):
+        return [remove_key_from_dict(i, key_to_remove) for i in data]
+    return data
+
 
-    return lines
+def _find_differences_dict(obj1, obj2, path, diffs):
+    """Recursively find differences between two dictionaries."""
+    for key in sorted(set(obj1) | set(obj2)):
+        subpath = f"{path}['{key}']" if path else f"['{key}']"
+        if key not in obj1:
+            diffs.append(f"{subpath}: added in second object")
+        elif key not in obj2:
+            diffs.append(f"{subpath}: removed in second object")
+        else:
+            diffs.extend(find_differences_in_json_objects(obj1[key], obj2[key], subpath))
 
 
-def get_structure_array_from_table(table, column_names):
+def find_differences_in_json_objects(obj1, obj2, path=""):
     """
-    Get a structured array from an astropy table for a selected list of columns.
+    Recursively find differences between two JSON-like objects.
 
     Parameters
     ----------
-    table: astropy.table.Table
-        Table to be converted.
-    column_names: list
-        List of column names to be included in the structured array.
+    obj1: dict, list, or any
+        First object to compare.
+    obj2: dict, list, or any
+        Second object to compare.
+    path: str
+        Path to the current object in the JSON structure, used for reporting differences.
 
     Returns
     -------
-    numpy.ndarray
-        Structured array containing the table data.
+    list
+        List of differences found between the two objects, with paths indicating where the
+        differences occur.
     """
-    return np.array(
-        list(zip(*[np.array(table[col]) for col in column_names if col in table.colnames])),
-        dtype=[(col, np.array(table[col]).dtype) for col in column_names if col in table.colnames],
-    )
+    diffs = []
 
+    if not isinstance(obj1, type(obj2)):
+        diffs.append(f"{path}: type changed from {type(obj1).__name__} to {type(obj2).__name__}")
+        return diffs
 
-def convert_keys_in_dict_to_lowercase(data):
-    """
-    Recursively convert all dictionary keys to lowercase.
+    if isinstance(obj1, dict):
+        _find_differences_dict(obj1, obj2, path, diffs)
 
-    Parameters
-    ----------
-    data: dict
-        Dictionary to be converted.
+    elif isinstance(obj1, list):
+        if len(obj1) != len(obj2):
+            diffs.append(f"{path}: list length changed from {len(obj1)} to {len(obj2)}")
+        for i, (a, b) in enumerate(zip(obj1, obj2)):
+            subpath = f"{path}[{i}]" if path else f"[{i}]"
+            diffs.extend(find_differences_in_json_objects(a, b, subpath))
 
-    Returns
-    -------
-    dict
-        Dictionary with all keys converted to lowercase.
-    """
-    if isinstance(data, dict):
-        return {k.lower(): convert_keys_in_dict_to_lowercase(v) for k, v in data.items()}
-    if isinstance(data, list):
-        return [convert_keys_in_dict_to_lowercase(i) for i in data]
-    return data
+    elif obj1 != obj2:
+        diffs.append(f"{path}: value changed from {obj1} to {obj2}")
+
+    return diffs
 
 
 def clear_default_sim_telarray_cfg_directories(command):

simtools/utils/geometry.py

@@ -180,3 +180,39 @@ def calculate_circular_mean(angles):
     sin_sum = np.sum(np.sin(angles))
     cos_sum = np.sum(np.cos(angles))
     return np.arctan2(sin_sum, cos_sum)
+
+
+def transform_ground_to_shower_coordinates(x_ground, y_ground, z_ground, azimuth, altitude):
+    """
+    Transform ground to shower coordinates.
+
+    Assume ground to be of type 'North-West-Up' (NWU) coordinates.
+
+    Parameters
+    ----------
+    x_ground: numpy.array
+        Ground x coordinate.
+    y_ground: numpy.array
+        Ground y coordinate.
+    z_ground: numpy.array
+        Ground z coordinate.
+    azimuth: numpy.array
+        Azimuth angle of the shower (in radians).
+    altitude: numpy.array
+        Altitude angle of the shower (in radians).
+
+    Returns
+    -------
+    tuple
+        Transformed shower coordinates (x', y', z').
+    """
+    x, y, z, az, alt = np.broadcast_arrays(x_ground, y_ground, z_ground, azimuth, altitude)
+
+    ca, sa = np.cos(az), np.sin(az)
+    cz, sz = np.sin(alt), np.cos(alt)
+
+    x_s = ca * cz * x - sa * y + ca * sz * z
+    y_s = sa * cz * x + ca * y + sa * sz * z
+    z_s = -sz * x + cz * z
+
+    return x_s, y_s, z_s

simtools/utils/names.py

@@ -12,6 +12,7 @@ Naming in simtools:
 
 """
 
+import json
 import logging
 import re
 from functools import cache
@@ -22,7 +23,7 @@ import yaml
 from simtools.constants import (
     MODEL_PARAMETER_DESCRIPTION_METASCHEMA,
     MODEL_PARAMETER_SCHEMA_PATH,
-    SCHEMA_PATH,
+    RESOURCE_PATH,
 )
 
 _logger = logging.getLogger(__name__)
@@ -59,10 +60,30 @@ def array_elements():
     dict
         Array elements.
     """
-    with open(Path(SCHEMA_PATH) / "array_elements.yml", encoding="utf-8") as file:
+    # for efficiency reason, no functions from simtools.utils.general are used here
+    with open(Path(RESOURCE_PATH) / "array_elements.yml", encoding="utf-8") as file:
         return yaml.safe_load(file)["data"]
 
 
+@cache
+def array_element_common_identifiers():
+    """
+    Get array element IDs from CTAO common identifier.
+
+    Returns
+    -------
+    dict, dict
+        Dictionary mapping array element names to their IDs and vice versa.
+    """
+    # for efficiency reason, no functions from simtools.utils.general are used here
+    id_to_name = {}
+    with open(Path(RESOURCE_PATH) / "array-element-ids.json", encoding="utf-8") as file:
+        data = json.load(file)
+    id_to_name = {e["id"]: e["name"] for e in data["array_elements"]}
+    name_to_id = {e["name"]: e["id"] for e in data["array_elements"]}
+    return id_to_name, name_to_id
+
+
 @cache
 def simulation_software():
     """
@@ -415,6 +436,54 @@ def get_array_element_id_from_name(array_element_name):
         raise ValueError(f"Invalid name {array_element_name}") from exc
 
 
+def get_common_identifier_from_array_element_name(array_element_name, default_return=None):
+    """
+    Get numerical common identifier from array element name as used by CTAO.
+
+    Common identifiers are numerical IDs used by the CTAO ACADA and DPPS systems.
+
+    Parameters
+    ----------
+    array_element_name: str
+        Array element name (e.g. LSTN-01)
+
+    Returns
+    -------
+    int
+        Common identifier.
+    """
+    _, name_to_id = array_element_common_identifiers()
+    try:
+        return name_to_id[array_element_name]
+    except KeyError as exc:
+        if default_return is not None:
+            return default_return
+        raise ValueError(f"Unknown array element name {array_element_name}") from exc
+
+
+def get_array_element_name_from_common_identifier(common_identifier):
+    """
+    Get array element name from common identifier as used by CTAO.
+
+    Common identifiers are numerical IDs used by the CTAO ACADA and DPPS systems.
+
+    Parameters
+    ----------
+    common_identifier: int
+        Common identifier.
+
+    Returns
+    -------
+    str
+        Array element name.
+    """
+    id_to_name, _ = array_element_common_identifiers()
+    try:
+        return id_to_name[common_identifier]
+    except KeyError as exc:
+        raise ValueError(f"Unknown common identifier {common_identifier}") from exc
+
+
 def get_list_of_array_element_types(
     array_element_class="telescopes", site=None, observatory="CTAO"
 ):
@@ -760,6 +829,6 @@ def file_name_with_version(file_name, suffix):
     if file_name is None or suffix is None:
         return None
     file_name = str(file_name)
-    if bool(re.search(r"\d+\.\d+\.\d+$", file_name)):
+    if re.search(r"\d{1,8}\.\d{1,8}\.\d{1,8}\Z", file_name):
         return Path(file_name + suffix)
     return Path(file_name).with_suffix(suffix)