gammasimtools 0.5.1__py3-none-any.whl → 0.6.1__py3-none-any.whl

simtools/utils/general.py

@@ -1,19 +1,26 @@
+"""
+General functions useful across different parts of the code.
+"""
+
 import copy
+import json
 import logging
-import mmap
 import os
 import re
+import tempfile
 import time
+import urllib.error
+import urllib.request
 from collections import namedtuple
 from pathlib import Path
+from urllib.parse import urlparse
 
 import astropy.units as u
-import numpy as np
-from astropy.coordinates.errors import UnitsError
 from astropy.io.misc import yaml
 
 __all__ = [
-    "collect_data_from_yaml_or_dict",
+    "change_dict_keys_case",
+    "collect_data_from_file_or_dict",
     "collect_final_lines",
     "collect_kwargs",
     "InvalidConfigData",
@@ -21,14 +28,12 @@ __all__ = [
     "MissingRequiredConfigEntry",
     "UnableToIdentifyConfigEntry",
     "get_log_level_from_user",
-    "rotate",
+    "remove_substring_recursively_from_dict",
     "separate_args_and_config_data",
     "set_default_kwargs",
-    "sort_arrays",
     "validate_config_data",
     "get_log_excerpt",
-    "convert_2D_to_radial_distr",
-    "save_dict_to_file",
+    "sort_arrays",
 ]
 
 _logger = logging.getLogger(__name__)
@@ -50,33 +55,6 @@ class InvalidConfigData(Exception):
     """Exception for invalid configuration data."""
 
 
-def file_has_text(file, text):
-    """
-    Check whether a file contain a certain piece of text.
-
-    Parameters
-    ----------
-    file: str
-        Path of the file.
-    text: str
-        Piece of text to be searched for.
-
-    Returns
-    -------
-    bool
-        True if file has text.
-    """
-    with open(file, "rb", 0) as string_file, mmap.mmap(
-        string_file.fileno(), 0, access=mmap.ACCESS_READ
-    ) as text_file_input:
-        re_search_1 = re.compile(f"{text}".encode())
-        search_result_1 = re_search_1.search(text_file_input)
-        if search_result_1 is None:
-            return False
-
-        return True
-
-
 def validate_config_data(config_data, parameters, ignore_unidentified=False):
     """
     Validate a generic config_data dict by using the info
@@ -202,7 +180,7 @@ def _validate_and_convert_value_without_units(value, value_keys, par_name, par_i
 
 def _check_value_entry_length(value, par_name, par_info):
     """
-    Validate length of user input parmeters
+    Validate length of user input parameters
 
     Parameters
     ----------
@@ -320,14 +298,85 @@ def _validate_and_convert_value(par_name, par_info, value_in):
     return _validate_and_convert_value_with_units(value, value_keys, par_name, par_info)
 
 
-def collect_data_from_yaml_or_dict(in_yaml, in_dict, allow_empty=False):
+def is_url(url):
+    """
+    Check if a string is a valid URL.
+
+    Parameters
+    ----------
+    url: str
+        String to be checked.
+
+    Returns
+    -------
+    bool
+        True if url is a valid URL.
+
+    """
+
+    try:
+        result = urlparse(url)
+        return all([result.scheme, result.netloc])
+    except AttributeError:
+        return False
+
+
+def collect_data_from_http(url):
+    """
+    Download yaml or json file from url and return it contents as dict.
+    File is downloaded as a temporary file and deleted afterwards.
+
+    Parameters
+    ----------
+    url: str
+        URL of the yaml/json file.
+
+    Returns
+    -------
+    dict
+        Dictionary containing the file content.
+
+    Raises
+    ------
+    TypeError
+        If url is not a valid URL.
+    urllib.error.HTTPError
+        If downloading the yaml file fails.
+
+    """
+
+    _logger.debug(f"Downloaded yaml file from {url}")
+    try:
+        with tempfile.NamedTemporaryFile() as tmp_file:
+            urllib.request.urlretrieve(url, tmp_file.name)
+            if url.endswith("yml") or url.endswith("yaml"):
+                data = yaml.load(tmp_file)
+            elif url.endswith("json"):
+                data = json.load(tmp_file)
+            else:
+                msg = f"File extension of {url} not supported (should be json or yaml)"
+                _logger.error(msg)
+                raise TypeError(msg)
+    except TypeError:
+        msg = "Invalid url {url}"
+        _logger.error(msg)
+        raise
+    except urllib.error.HTTPError:
+        msg = f"Failed to download yaml file from {url}"
+        _logger.error(msg)
+        raise
+
+    return data
+
+
+def collect_data_from_file_or_dict(file_name, in_dict, allow_empty=False):
     """
-    Collect input data that can be given either as a dict or as a yaml file.
+    Collect input data that can be given either as a dict or as a yaml/json file.
 
     Parameters
     ----------
-    in_yaml: str
-        Name of the yaml file.
+    file_name: str
+        Name of the yaml/json file.
     in_dict: dict
         Data as dict.
     allow_empty: bool
@@ -339,60 +388,30 @@ def collect_data_from_yaml_or_dict(in_yaml, in_dict, allow_empty=False):
         Data as dict.
     """
 
-    if in_yaml is not None:
+    if file_name is not None:
         if in_dict is not None:
-            _logger.warning("Both in_dict in_yaml were given - in_yaml will be used")
-        with open(in_yaml, encoding="utf-8") as file:
-            data = yaml.load(file)
+            _logger.warning("Both in_dict and file_name were given - file_name will be used")
+        if is_url(str(file_name)):
+            data = collect_data_from_http(file_name)
+        elif Path(file_name).suffix.lower() == ".json":
+            with open(file_name, encoding="utf-8") as file:
+                data = json.load(file)
+        else:
+            with open(file_name, encoding="utf-8") as file:
+                data = yaml.load(file)
         return data
     if in_dict is not None:
         return dict(in_dict)
 
-    msg = "Input has not been provided (neither by yaml file, nor by dict)"
+    msg = "Input has not been provided (neither by file, nor by dict)"
     if allow_empty:
         _logger.debug(msg)
         return None
 
-    _logger.error(msg)
+    _logger.debug(msg)
     raise InvalidConfigData(msg)
 
 
-def collect_dict_from_file(file_path, file_name=None):
-    """
-    Collect input data from a file.
-
-    File_path can be a yaml file name or a directory.
-    In the latter case, file_name is used to find the file.
-
-    Note that this method returns an empty dict if the file is not found
-    (while gen.collect_data_from_yaml_or_dict returns None).
-
-    Parameters
-    ----------
-    file_path: str
-        Name of the yaml file or directory.
-    file_name: str
-        Name of the file to be found in the directory.
-
-    """
-
-    _dict = {}
-    try:
-        _dict = (
-            collect_data_from_yaml_or_dict(in_yaml=file_path, in_dict=None, allow_empty=True) or {}
-        )
-    except IsADirectoryError:
-        try:
-            _file = Path(file_path).joinpath(file_name)
-            _dict = (
-                collect_data_from_yaml_or_dict(in_yaml=_file, in_dict=None, allow_empty=True) or {}
-            )
-        except (TypeError, KeyError):
-            pass
-
-    return _dict
-
-
 def collect_kwargs(label, in_kwargs):
     """
     Collect kwargs of the type label_* and return them as a dict.
@@ -437,27 +456,6 @@ def set_default_kwargs(in_kwargs, **kwargs):
     return in_kwargs
 
 
-def sort_arrays(*args):
-    """Sort arrays
-
-    Parameters
-    ----------
-    *args
-        Arguments to be sorted.
-    Returns
-    -------
-    list
-        Sorted args.
-    """
-
-    order_array = copy.copy(args[0])
-    new_args = []
-    for arg in args:
-        _, value = zip(*sorted(zip(order_array, arg)))
-        new_args.append(list(value))
-    return new_args
-
-
 def collect_final_lines(file, n_lines):
     """
     Collect final lines.
@@ -477,7 +475,7 @@ def collect_final_lines(file, n_lines):
     list_of_lines = []
 
     if Path(file).suffix == ".gz":
-        import gzip
+        import gzip  # pylint: disable=import-outside-toplevel
 
         file_open_function = gzip.open
     else:
@@ -573,8 +571,8 @@ def copy_as_list(value):
 
 def separate_args_and_config_data(expected_args, **kwargs):
     """
-    Separate kwargs into the arguments expected for instancing a class and the dict to be given as\
-    config_data. This function is specific for methods from_kwargs in classes which use the \
+    Separate kwargs into the arguments expected for instancing a class and the dict to be given as
+    config_data. This function is specific for methods from_kwargs in classes which use the
     validate_config_data system.
 
     Parameters
@@ -621,6 +619,7 @@ def program_is_executable(program):
                 if is_exe(exe_file):
                     return exe_file
         except KeyError:
+            _logger.debug("PATH environment variable is not set.")
             return None
 
     return None
@@ -686,6 +685,47 @@ def find_file(name, loc):
     raise FileNotFoundError(msg)
 
 
+def get_log_excerpt(log_file, n_last_lines=30):
+    """
+    Get an excerpt from a log file, namely the n_last_lines of the file.
+
+    Parameters
+    ----------
+    log_file: str or Path
+        Log file to get the excerpt from.
+    n_last_lines: int
+        Number of last lines of the file to get.
+
+    Returns
+    -------
+    str
+        Excerpt from log file with header/footer
+    """
+
+    return (
+        "\n\nRuntime error - See below the relevant part of the log/err file.\n\n"
+        f"{log_file}\n"
+        "====================================================================\n\n"
+        f"{collect_final_lines(log_file, n_last_lines)}\n\n"
+        "====================================================================\n"
+    )
+
+
+def get_file_age(file_path):
+    """
+    Get the age of a file in seconds since the last modification.
+    """
+    if not Path(file_path).is_file():
+        raise FileNotFoundError(f"'{file_path}' does not exist or is not a file.")
+
+    file_stats = os.stat(file_path)
+    modification_time = file_stats.st_mtime
+    current_time = time.time()
+
+    file_age_minutes = (current_time - modification_time) / 60
+    return file_age_minutes
+
+
 def change_dict_keys_case(data_dict, lower_case=True):
     """
     Change keys of a dictionary to lower or upper case. Crawls through the dictionary and changes\
@@ -719,220 +759,126 @@ def change_dict_keys_case(data_dict, lower_case=True):
             else:
                 _return_dict[_key_changed] = data_dict[key]
     except AttributeError:
-        _logger = logging.getLogger(__name__)
-        _logger.error(f"Invalid method argument: {data_dict}")
+        _logger.error(f"Input is not a proper dictionary: {data_dict}")
         raise
-
     return _return_dict
 
 
-@u.quantity_input(rotation_angle_phi=u.rad, rotation_angle_theta=u.rad)
-def rotate(x, y, rotation_around_z_axis, rotation_around_y_axis=0):
+def remove_substring_recursively_from_dict(data_dict, substring="\n"):
     """
-    Transform the x and y coordinates of the telescopes according to two rotations:
-    `rotation_angle_around_z_axis` gives the rotation on the observation plane (x, y) and
-    `rotation_angle_around_y_axis` allows to rotate the observation plane in space.
-    The function returns the rotated x and y values in the same unit given.
-    The direction of rotation of the elements in the plane is counterclockwise, i.e.,
-    the rotation of the coordinate system is clockwise.
+    Remove substrings from all strings in a dictionary. Recursively crawls through the dictionary
+    This e.g., allows to remove all newline characters from a dictionary.
 
     Parameters
     ----------
-    x: numpy.array or list
-        x positions of the entries (e.g. telescopes), usually in meters.
-    y: numpy.array or list
-        y positions of the entries (e.g. telescopes), usually in meters.
-    rotation_angle_around_z_axis: astropy.units.rad
-        Angle to rotate the array in the observation plane (around z axis) in radians.
-    rotation_angle_around_y_axis: astropy.units.rad
-        Angle to rotate the observation plane around the y axis in radians.
-
-    Returns
-    -------
-    2-tuple of list
-        x and y positions of the rotated entry (e.g. telescopes) positions.
+    data_dict: dict
+        Dictionary to be converted.
+    substring: str
+        Substring to be removed.
 
     Raises
     ------
-    TypeError:
-        If type of x and y parameters are not valid.
-    RuntimeError:
-        If the length of x and y are different.
-    UnitsError:
-        If the unit of x and y are different.
-    """
-
-    allowed_types = (list, np.ndarray, u.Quantity, float, int)
-    if not all(isinstance(variable, allowed_types) for variable in [x, y]):
-        raise TypeError("x and y types are not valid! Cannot perform transformation.")
-
-    if (
-        np.sum(
-            np.array([isinstance(x, type_now) for type_now in allowed_types[:-2]])
-            * np.array([isinstance(y, type_now) for type_now in allowed_types[:-2]])
-        )
-        == 0
-    ):
-        raise TypeError("x and y are not from the same type! Cannot perform transformation.")
-
-    if not isinstance(x, (list, np.ndarray)):
-        x = [x]
-    if not isinstance(y, (list, np.ndarray)):
-        y = [y]
-
-    if len(x) != len(y):
-        raise RuntimeError(
-            "Cannot perform coordinate transformation when x and y have different lengths."
-        )
-    if all(isinstance(variable, (u.Quantity)) for variable in [x, y]):
-        if not isinstance(x[0].unit, type(y[0].unit)):
-            raise UnitsError(
-                "Cannot perform coordinate transformation when x and y have different units."
-            )
-
-    x_trans = np.cos(rotation_around_y_axis) * (
-        x * np.cos(rotation_around_z_axis) - y * np.sin(rotation_around_z_axis)
-    )
-    y_trans = x * np.sin(rotation_around_z_axis) + y * np.cos(rotation_around_z_axis)
-
-    return x_trans, y_trans
+    AttributeError:
+        if input is not a proper dictionary.
+    """
+    try:
+        for key, value in data_dict.items():
+            if isinstance(value, str):
+                data_dict[key] = value.replace(substring, "")
+            elif isinstance(value, list):
+                modified_items = [
+                    item.replace(substring, "") if isinstance(item, str) else item for item in value
+                ]
+                modified_items = [
+                    remove_substring_recursively_from_dict(item, substring)
+                    if isinstance(item, dict)
+                    else item
+                    for item in modified_items
+                ]
+                data_dict[key] = modified_items
+            elif isinstance(value, dict):
+                data_dict[key] = remove_substring_recursively_from_dict(value, substring)
+    except AttributeError:
+        _logger.debug(f"Input is not a dictionary: {data_dict}")
+    return data_dict
 
 
-def get_log_excerpt(log_file, n_last_lines=30):
-    """
-    Get an excerpt from a log file, namely the n_last_lines of the file.
+def sort_arrays(*args):
+    """Sort arrays
 
     Parameters
     ----------
-    log_file: str or Path
-        Log file to get the excerpt from.
-    n_last_lines: int
-        Number of last lines of the file to get.
-
+    *args
+        Arguments to be sorted.
     Returns
     -------
-    str
-        Excerpt from log file with header/footer
+    list
+        Sorted args.
     """
 
-    return (
-        "\n\nRuntime error - See below the relevant part of the log/err file.\n\n"
-        f"{log_file}\n"
-        "====================================================================\n\n"
-        f"{collect_final_lines(log_file, n_last_lines)}\n\n"
-        "====================================================================\n"
-    )
+    if len(args) == 0:
+        return args
+    order_array = copy.copy(args[0])
+    new_args = []
+    for arg in args:
+        _, value = zip(*sorted(zip(order_array, arg)))
+        new_args.append(list(value))
+    return new_args
 
 
-def get_file_age(file_path):
+def extract_type_of_value(value) -> str:
     """
-    Get the age of a file in seconds since the last modification.
+    Extract the string representation of the the type of a value.
+    For example, for a string, it returns 'str' rather than '<class 'str'>'.
+    Take into account also the case where the value is a numpy type.
     """
-    if not Path(file_path).is_file():
-        raise FileNotFoundError(f"'{file_path}' does not exist or is not a file.")
-
-    file_stats = os.stat(file_path)
-    modification_time = file_stats.st_mtime
-    current_time = time.time()
+    _type = str(type(value))
+    if "numpy" in _type:
+        return re.sub(r"\d+", "", _type.split("'")[1].split(".")[-1])
+    if "astropy" in _type:
+        raise NotImplementedError("Astropy types are not supported yet.")
 
-    file_age_minutes = (current_time - modification_time) / 60
-    return file_age_minutes
+    _type = _type.split("'")[1]
+    return _type
 
 
-def convert_2D_to_radial_distr(hist2d, xaxis, yaxis, bins=50, max_dist=1000):
+def get_value_unit_type(value):
     """
-    Convert a 2D histogram of positions, e.g. photon positions on the ground, to a 1D distribution.
+    Get the value, unit and type of a value.
+    The value is stripped of its unit and the unit is returned
+    in its string form (i.e., to_string()).
+    The type is returned as a string representation of the type.
+    For example, for a string, it returns 'str' rather than '<class 'str'>'.
+
+    Note that Quantities are always floats, even if the original value is represented as an int.
 
     Parameters
     ----------
-    hist2d: numpy.ndarray
-        The histogram counts.
-    xaxis: numpy.array
-        The values of the x axis (histogram edges) on the ground.
-    yaxis: numpy.array
-        The values of the y axis (histogram edges) on the ground.
-    bins: float
-        Number of bins in distance.
-    max_dist: float
-       Maximum distance to consider in the 1D histogram, usually in meters.
+    value: str, int, float, bool, u.Quantity
+        Value to be parsed.
 
     Returns
     -------
-    np.array
-        The values of the 1D histogram with size = int(max_dist/bin_size).
-    np.array
-        The edges of the 1D histogram with size = int(max_dist/bin_size) + 1.
-
-    """
-
-    # Check if the histogram will make sense
-    bins_step = 2 * max_dist / bins  # in the 2D array, the positive and negative direction count.
-    for axis in [xaxis, yaxis]:
-        if (bins_step < np.diff(axis)).any():
-            msg = (
-                f"The histogram with number of bins {bins} and maximum distance of {max_dist} "
-                f"resulted in a bin size smaller than the original array. Please adjust those "
-                f"parameters to increase the bin size and avoid nan in the histogram values."
-            )
-            _logger.warning(msg)
-            break
-
-    grid_2d_x, grid_2d_y = np.meshgrid(xaxis[:-1], yaxis[:-1])  # [:-1], since xaxis and yaxis are
-    # the hist edges (n + 1).
-    # radial_distance_map maps the distance to the center from each element in a square matrix.
-    radial_distance_map = np.sqrt(grid_2d_x**2 + grid_2d_y**2)
-    # The sorting and unravel_index give us the two indices for the position of the sorted element
-    # in the original 2d matrix
-    x_indices_sorted, y_indices_sorted = np.unravel_index(
-        np.argsort(radial_distance_map, axis=None), np.shape(radial_distance_map)
-    )
-
-    # We construct a 1D array with the histogram counts sorted according to the distance to the
-    # center.
-    hist_sorted = np.array(
-        [hist2d[i_x, i_y] for i_x, i_y in zip(x_indices_sorted, y_indices_sorted)]
-    )
-    distance_sorted = np.sort(radial_distance_map, axis=None)
-
-    # For larger distances, we have more elements in a slice 'dr' in radius, hence, we need to
-    # acount for it using weights below.
-
-    weights, radial_edges = np.histogram(distance_sorted, bins=bins, range=(0, max_dist))
-    histogram_1D = np.empty_like(weights, dtype=float)
-
-    for i_radial, _ in enumerate(radial_edges[:-1]):
-        # Here we sum all the events within a radial interval 'dr' and then divide by the number of
-        # bins that fit this interval.
-        indices_to_sum = (distance_sorted >= radial_edges[i_radial]) * (
-            distance_sorted < radial_edges[i_radial + 1]
-        )
-        histogram_1D[i_radial] = np.sum(hist_sorted[indices_to_sum]) / weights[i_radial]
-    return histogram_1D, radial_edges
-
-
-def save_dict_to_file(dictionary, file_name):
+    type of value, str, str
+        Value, unit in string representation (to_string())),
+        and string representation of the type of the value.
     """
-    Save dictionary to a file.
 
-    Parameters
-    ----------
-    dictionary: dict
-        Dictionary to be saved into a file.
-    file_name: str or Path
-        Name of file to be saved with path.
-
-    Raises
-    ------
-    IOError:
-        if writing to file_name fails.
-    """
+    base_value = value
+    base_unit = None
+    base_type = ""
+    if isinstance(value, (str, u.Quantity)):
+        try:
+            _quantity_value = u.Quantity(value)
+            base_value = _quantity_value.value
+            base_type = extract_type_of_value(base_value)
+            if _quantity_value.unit.to_string() != "":
+                base_unit = _quantity_value.unit.to_string()
+        except TypeError:
+            base_value = value
+            base_type = "str"
+    else:
+        base_value = value
+        base_type = extract_type_of_value(base_value)
 
-    file_name = Path(file_name).with_suffix(".yml")
-    _logger.info(f"Exporting histogram configuration to {file_name}")
-    try:
-        with open(file_name, "w", encoding="utf-8") as file:
-            yaml.dump(dictionary, file)
-    except IOError:
-        msg = f"Failed to write to {file_name}."
-        _logger.error(msg)
-        raise
+    return base_value, base_unit, base_type