gammasimtools 0.18.0__py3-none-any.whl → 0.20.0__py3-none-any.whl

simtools/dependencies.py

@@ -14,46 +14,75 @@ import re
 import subprocess
 from pathlib import Path
 
-import simtools.utils.general as gen
+import yaml
+
 from simtools.db.db_handler import DatabaseHandler
+from simtools.io import ascii_handler
 
 _logger = logging.getLogger(__name__)
 
 
-def get_version_string(db_config=None):
-    """Print the versions of the dependencies."""
+def get_version_string(db_config=None, run_time=None):
+    """
+    Print the versions of the dependencies.
+
+    Parameters
+    ----------
+    db_config : dict, optional
+        Database configuration dictionary.
+    run_time : list, optional
+        Runtime environment command (e.g., Docker).
+
+    Returns
+    -------
+    str
+        String containing the versions of the dependencies.
+
+    """
     return (
-        f"Database version: {get_database_version(db_config)}\n"
-        f"sim_telarray version: {get_sim_telarray_version()}\n"
-        f"CORSIKA version: {get_corsika_version()}\n"
+        f"Database name: {get_database_version_or_name(db_config, version=False)}\n"
+        f"Database version: {get_database_version_or_name(db_config, version=True)}\n"
+        f"sim_telarray version: {get_sim_telarray_version(run_time)}\n"
+        f"CORSIKA version: {get_corsika_version(run_time)}\n"
+        f"Build options: {get_build_options(run_time)}\n"
+        f"Runtime environment: {run_time if run_time else 'None'}\n"
     )
 
 
-def get_database_version(db_config):
+def get_database_version_or_name(db_config, version=True):
     """
-    Get the version of the simulation model data base used.
+    Get the version or name of the simulation model data base used.
 
     Parameters
     ----------
     db_config : dict
         Dictionary containing the database configuration.
+    version : bool
+        If True, return the version of the database. If False, return the name.
 
     Returns
     -------
     str
-        Version of the simulation model data base used.
+        Version or name of the simulation model data base used.
 
     """
     if db_config is None:
         return None
     db = DatabaseHandler(db_config)
-    return db.mongo_db_config.get("db_simulation_model")
+    return db.mongo_db_config.get(
+        "db_simulation_model_version" if version else "db_simulation_model"
+    )
 
 
-def get_sim_telarray_version():
+def get_sim_telarray_version(run_time):
     """
     Get the version of the sim_telarray package using 'sim_telarray --version'.
 
+    Parameters
+    ----------
+    run_time : list, optional
+        Runtime environment command (e.g., Docker).
+
     Returns
     -------
     str
@@ -65,24 +94,33 @@ def get_sim_telarray_version():
         return None
     sim_telarray_path = Path(sim_telarray_path) / "sim_telarray" / "bin" / "sim_telarray"
 
+    if run_time is None:
+        command = [str(sim_telarray_path), "--version"]
+    else:
+        command = [*run_time, str(sim_telarray_path), "--version"]
+
+    _logger.debug(f"Running command: {command}")
+    result = subprocess.run(command, capture_output=True, text=True, check=False)
+
     # expect stdout with e.g. a line 'Release: 2024.271.0 from 2024-09-27'
-    result = subprocess.run(
-        [sim_telarray_path, "--version"],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
     match = re.search(r"^Release:\s+(.+)", result.stdout, re.MULTILINE)
-
     if match:
         return match.group(1).split()[0]
+
+    _logger.debug(f"Command output stdout: {result.stdout} stderr: {result.stderr}")
+
     raise ValueError(f"sim_telarray release not found in {result.stdout}")
 
 
-def get_corsika_version():
+def get_corsika_version(run_time=None):
     """
     Get the version of the CORSIKA package.
 
+    Parameters
+    ----------
+    run_time : list, optional
+        Runtime environment command (e.g., Docker).
+
     Returns
     -------
     str
@@ -95,10 +133,15 @@ def get_corsika_version():
         return None
     corsika_command = Path(sim_telarray_path) / "corsika-run" / "corsika"
 
+    if run_time is None:
+        command = [str(corsika_command)]
+    else:
+        command = [*run_time, str(corsika_command)]
+
     # Below I do not use the standard context manager because
     # it makes mocking in the tests significantly more difficult
     process = subprocess.Popen(  # pylint: disable=consider-using-with
-        corsika_command,
+        command,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
         stdin=subprocess.PIPE,
@@ -123,25 +166,50 @@ def get_corsika_version():
     if version and re.match(r"\d+\.\d+", version):
         return version
     try:
-        build_opts = get_build_options()
-    except (FileNotFoundError, TypeError):
+        build_opts = get_build_options(run_time)
+    except (FileNotFoundError, TypeError, ValueError):
         _logger.warning("Could not get CORSIKA version.")
         return None
     _logger.debug("Getting the CORSIKA version from the build options.")
     return build_opts.get("corsika_version")
 
 
-def get_build_options():
+def get_build_options(run_time=None):
     """
     Return CORSIKA / sim_telarray build options.
 
     Expects a build_opts.yml file in the sim_telarray directory.
+
+    Parameters
+    ----------
+    run_time : list, optional
+        Runtime environment command (e.g., Docker).
+
+    Returns
+    -------
+    dict
+        Build options from build_opts.yml file.
     """
+    sim_telarray_path = os.getenv("SIMTOOLS_SIMTEL_PATH")
+    if sim_telarray_path is None:
+        raise ValueError("SIMTOOLS_SIMTEL_PATH not defined.")
+
+    build_opts_path = Path(sim_telarray_path) / "build_opts.yml"
+
+    if run_time is None:
+        try:
+            return ascii_handler.collect_data_from_file(build_opts_path)
+        except FileNotFoundError as exc:
+            raise FileNotFoundError("No build_opts.yml file found.") from exc
+
+    command = [*run_time, "cat", str(build_opts_path)]
+    _logger.debug(f"Reading build_opts.yml with command: {command}")
+
+    result = subprocess.run(command, capture_output=True, text=True, check=False)
+    if result.returncode:
+        raise FileNotFoundError(f"No build_opts.yml file found in container: {result.stderr}")
+
     try:
-        return gen.collect_data_from_file(
-            Path(os.getenv("SIMTOOLS_SIMTEL_PATH")) / "build_opts.yml"
-        )
-    except FileNotFoundError as exc:
-        raise FileNotFoundError("No build_opts.yml file found.") from exc
-    except TypeError as exc:
-        raise TypeError("SIMTOOLS_SIMTEL_PATH not defined.") from exc
+        return yaml.safe_load(result.stdout)
+    except yaml.YAMLError as exc:
+        raise ValueError(f"Error parsing build_opts.yml from container: {exc}") from exc

simtools/io/ascii_handler.py

@@ -0,0 +1,279 @@
+"""Helper module for ASCII file operations."""
+
+import json
+import logging
+import tempfile
+import urllib.request
+from pathlib import Path
+
+import astropy.units as u
+import numpy as np
+import yaml
+
+from simtools.utils.general import is_url
+
+_logger = logging.getLogger(__name__)
+
+
+def collect_data_from_file(file_name, yaml_document=None):
+    """
+    Collect data from file based on its extension.
+
+    Parameters
+    ----------
+    file_name: str
+        Name of the yaml/json/ascii file.
+    yaml_document: None, int
+        Return list of yaml documents or a single document (for yaml files with several documents).
+
+    Returns
+    -------
+    data: dict or list
+        Data as dict or list.
+    """
+    if is_url(file_name):
+        return collect_data_from_http(file_name)
+
+    suffix = Path(file_name).suffix.lower()
+    try:
+        with open(file_name, encoding="utf-8") as file:
+            return _collect_data_from_different_file_types(file, file_name, suffix, yaml_document)
+    # broad exception to catch all possible errors in reading the file
+    except Exception as exc:  # pylint: disable=broad-except
+        raise type(exc)(f"Failed to read file {file_name}: {exc}") from exc
+
+
+def _collect_data_from_different_file_types(file, file_name, suffix, yaml_document):
+    """Collect data from different file types."""
+    if suffix == ".json":
+        return json.load(file)
+    if suffix in (".list", ".txt"):
+        return [line.strip() for line in file.readlines()]
+    if suffix in [".yml", ".yaml"]:
+        return _collect_data_from_yaml_file(file, file_name, yaml_document)
+    raise TypeError(f"File type {suffix} not supported.")
+
+
+def _collect_data_from_yaml_file(file, file_name, yaml_document):
+    """Collect data from a yaml file (allow for multi-document yaml files)."""
+    try:
+        return yaml.safe_load(file)
+    except yaml.constructor.ConstructorError:
+        return _load_yaml_using_astropy(file)
+    except yaml.composer.ComposerError:
+        pass
+    file.seek(0)
+    if yaml_document is None:
+        return list(yaml.safe_load_all(file))
+    try:
+        return list(yaml.safe_load_all(file))[yaml_document]
+    except IndexError as exc:
+        raise IndexError(
+            f"Failed to read file {file_name}: YAML document index {yaml_document} is out of range."
+        ) from exc
+
+
+def _load_yaml_using_astropy(file):
+    """
+    Load a yaml file using astropy's yaml loader.
+
+    Parameters
+    ----------
+    file: file
+        File to be loaded.
+
+    Returns
+    -------
+    dict
+        Dictionary containing the file content.
+    """
+    # pylint: disable=import-outside-toplevel
+    import astropy.io.misc.yaml as astropy_yaml
+
+    file.seek(0)
+    return astropy_yaml.load(file)
+
+
+def collect_data_from_http(url):
+    """
+    Download yaml or json file from url and return it contents as dict.
+
+    File is downloaded as a temporary file and deleted afterwards.
+
+    Parameters
+    ----------
+    url: str
+        URL of the yaml/json file.
+
+    Returns
+    -------
+    dict
+        Dictionary containing the file content.
+
+    Raises
+    ------
+    TypeError
+        If url is not a valid URL.
+    FileNotFoundError
+        If downloading the yaml file fails.
+
+    """
+    try:
+        with tempfile.NamedTemporaryFile(mode="w+t") as tmp_file:
+            urllib.request.urlretrieve(url, tmp_file.name)
+            data = _collect_data_from_different_file_types(
+                tmp_file, url, Path(url).suffix.lower(), None
+            )
+    except TypeError as exc:
+        raise TypeError(f"Invalid url {url}") from exc
+    except urllib.error.HTTPError as exc:
+        raise FileNotFoundError(f"Failed to download file from {url}") from exc
+
+    _logger.debug(f"Downloaded file from {url}")
+    return data
+
+
+def read_file_encoded_in_utf_or_latin(file_name):
+    """
+    Read a file encoded in UTF-8 or Latin-1.
+
+    Parameters
+    ----------
+    file_name: str
+        Name of the file to be read.
+
+    Returns
+    -------
+    list
+        List of lines read from the file.
+
+    Raises
+    ------
+    UnicodeDecodeError
+        If the file cannot be decoded using UTF-8 or Latin-1.
+    """
+    try:
+        with open(file_name, encoding="utf-8") as file:
+            lines = file.readlines()
+    except UnicodeDecodeError:
+        _logger.debug("Unable to decode file using UTF-8. Trying Latin-1.")
+        try:
+            with open(file_name, encoding="latin-1") as file:
+                lines = file.readlines()
+        except UnicodeDecodeError as exc:
+            msg = f"Unable to decode file {file_name} using UTF-8 or Latin-1."
+            raise UnicodeDecodeError(exc.encoding, exc.object, exc.start, exc.end, msg) from exc
+
+    return lines
+
+
+def is_utf8_file(file_name):
+    """
+    Check if a file is encoded in UTF-8.
+
+    Parameters
+    ----------
+    file_name: str, Path
+        Name of the file to be checked.
+
+    Returns
+    -------
+    bool
+        True if the file is encoded in UTF-8, False otherwise.
+    """
+    try:
+        with open(file_name, encoding="utf-8") as file:
+            file.read()
+        return True
+    except UnicodeDecodeError:
+        return False
+
+
+def write_data_to_file(data, output_file, sort_keys=False, numpy_types=False):
+    """
+    Write structured data to JSON or YAML file.
+
+    The file type is determined by the file extension.
+
+    Parameters
+    ----------
+    data: dict or list
+        Data to be written to the file.
+    output_file: str or Path
+        Name of the file to be written.
+    sort_keys: bool, optional
+        If True, sort the keys.
+    numpy_types: bool, optional
+        If True, convert numpy types to native Python types.
+    """
+    output_file = Path(output_file)
+    if output_file.suffix.lower() == ".json":
+        return _write_to_json(data, output_file, sort_keys, numpy_types)
+    if output_file.suffix.lower() in [".yml", ".yaml"]:
+        return _write_to_yaml(data, output_file, sort_keys)
+
+    raise ValueError(
+        f"Unsupported file type {output_file.suffix}. Only .json, .yml, and .yaml are supported."
+    )
+
+
+def _write_to_json(data, output_file, sort_keys, numpy_types):
+    """
+    Write data to a JSON file.
+
+    Parameters
+    ----------
+    data: dict or list
+        Data to be written to the file.
+    output_file: Path
+        Name of the file to be written.
+    sort_keys: bool
+        If True, sort the keys.
+    numpy_types: bool
+        If True, convert numpy types to native Python types.
+    """
+    with open(output_file, "w", encoding="utf-8") as file:
+        json.dump(
+            data,
+            file,
+            indent=4,
+            sort_keys=sort_keys,
+            cls=JsonNumpyEncoder if numpy_types else None,
+        )
+        file.write("\n")
+
+
+def _write_to_yaml(data, output_file, sort_keys):
+    """
+    Write data to a YAML file.
+
+    Parameters
+    ----------
+    data: dict or list
+        Data to be written to the file.
+    output_file: Path
+        Name of the file to be written.
+    sort_keys: bool
+        If True, sort the keys.
+
+    """
+    with open(output_file, "w", encoding="utf-8") as file:
+        yaml.dump(data, file, indent=4, sort_keys=sort_keys, explicit_start=True)
+
+
+class JsonNumpyEncoder(json.JSONEncoder):
+    """Convert numpy to python types as accepted by json.dump."""
+
+    def default(self, o):
+        """Return default encoder."""
+        if isinstance(o, np.floating):
+            return float(o)
+        if isinstance(o, np.integer):
+            return int(o)
+        if isinstance(o, np.ndarray):
+            return o.tolist()
+        if isinstance(o, u.core.CompositeUnit | u.core.IrreducibleUnit | u.core.Unit):
+            return str(o) if o != u.dimensionless_unscaled else None
+        if np.issubdtype(type(o), np.bool_):
+            return bool(o)
+        return super().default(o)

simtools/{io_operations → io}/io_handler.py

@@ -97,11 +97,14 @@ class IOHandler(metaclass=IOHandlerSingleton):
                 path.joinpath(label_dir) if sub_dir is None else path.joinpath(label_dir, sub_dir)
             )
 
+        return self._mkdir(path)
+
+    def _mkdir(self, path):
+        """Create a directory and return path."""
         try:
             path.mkdir(parents=True, exist_ok=True)
-        except FileNotFoundError:
-            self._logger.error(f"Error creating directory {path!s}")
-            raise
+        except FileNotFoundError as exc:
+            raise FileNotFoundError(f"Error creating directory {path!s}") from exc
 
         return path.absolute()
 
@@ -156,3 +159,22 @@ class IOHandler(metaclass=IOHandlerSingleton):
         else:
             raise IncompleteIOHandlerInitError
         return file_prefix.joinpath(file_name).absolute()
+
+    def get_model_configuration_directory(self, label, model_version):
+        """
+        Get path of the simulation model configuration directory.
+
+        This is the directory where the sim_telarray configuration files will be stored.
+
+        Parameters
+        ----------
+        label: str
+            Instance label.
+        model_version: str
+            Model version.
+
+        Returns
+        -------
+        Path
+        """
+        return self._mkdir(self.get_output_directory(label=label).joinpath("model", model_version))

simtools/job_execution/htcondor_script_generator.py

@@ -17,13 +17,12 @@ def generate_submission_script(args_dict):
     args_dict: dict
         Arguments dictionary.
     """
-    _logger.info("Generating HT Condor submission scripts ")
-
     work_dir = Path(args_dict["output_path"])
     log_dir = work_dir / "logs"
     work_dir.mkdir(parents=True, exist_ok=True)
     log_dir.mkdir(parents=True, exist_ok=True)
     submit_file_name = "simulate_prod.submit"
+    _logger.info(f"Generating HT Condor submission scripts (path: {work_dir})")
 
     with open(work_dir / f"{submit_file_name}.condor", "w", encoding="utf-8") as submit_file_handle:
         submit_file_handle.write(
@@ -101,9 +100,20 @@ def _get_submit_script(args_dict):
     )
     core_scatter = args_dict["core_scatter"]
     core_scatter_string = f'"{core_scatter[0]} {core_scatter[1].to(u.m).value} m"'
+    view_cone = args_dict["view_cone"]
+    view_cone_string = f'"{view_cone[0].to(u.deg)} {view_cone[1].to(u.deg)}"'
 
     label = args_dict["label"] if args_dict["label"] else "simulate-prod"
 
+    array_layout_name = (
+        args_dict["array_layout_name"][0]
+        if isinstance(args_dict["array_layout_name"], list)
+        and len(args_dict["array_layout_name"]) == 1
+        else args_dict["array_layout_name"]
+    )
+
+    run_number_offset = args_dict["run_number_offset"] or 1
+
     return f"""#!/usr/bin/env bash
 
 # Process ID used to generate run number
@@ -116,15 +126,16 @@ simtools-simulate-prod \\
     --label {label} \\
     --model_version {args_dict["model_version"]} \\
     --site {args_dict["site"]} \\
-    --array_layout_name {args_dict["array_layout_name"]} \\
+    --array_layout_name {array_layout_name} \\
     --primary {args_dict["primary"]} \\
     --azimuth_angle {azimuth_angle_string} \\
     --zenith_angle {zenith_angle_string} \\
     --nshow {args_dict["nshow"]} \\
     --energy_range {energy_range_string} \\
     --core_scatter {core_scatter_string} \\
+    --view_cone {view_cone_string} \\
     --run_number $((process_id)) \\
-    --run_number_offset {args_dict["run_number_offset"]} \\
+    --run_number_offset {run_number_offset} \\
     --number_of_runs 1 \\
     --data_directory /tmp/simtools-data \\
     --output_path /tmp/simtools-output \\

simtools/layout/array_layout.py

@@ -10,7 +10,7 @@ from astropy.table import QTable
 
 import simtools.utils.general as gen
 from simtools.data_model import data_reader, schema
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 from simtools.layout.geo_coordinates import GeoCoordinates
 from simtools.layout.telescope_position import TelescopePosition
 from simtools.model.site_model import SiteModel

simtools/layout/array_layout_utils.py

@@ -7,7 +7,7 @@ import simtools.utils.general as gen
 from simtools.data_model import data_reader
 from simtools.data_model.metadata_collector import MetadataCollector
 from simtools.data_model.model_data_writer import ModelDataWriter
-from simtools.io_operations import io_handler
+from simtools.io import ascii_handler, io_handler
 from simtools.model.array_model import ArrayModel
 from simtools.model.site_model import SiteModel
 from simtools.utils import names
@@ -36,17 +36,19 @@ def retrieve_ctao_array_layouts(site, repository_url, branch_name="main"):
     _logger.info(f"Retrieving array layouts from {repository_url} on branch {branch_name}.")
 
     if gen.is_url(repository_url):
-        array_element_ids = gen.collect_data_from_http(
+        array_element_ids = ascii_handler.collect_data_from_http(
             url=f"{repository_url}/{branch_name}/array-element-ids.json"
         )
-        sub_arrays = gen.collect_data_from_http(
+        sub_arrays = ascii_handler.collect_data_from_http(
             url=f"{repository_url}/{branch_name}/subarray-ids.json"
         )
     else:
-        array_element_ids = gen.collect_data_from_file(
+        array_element_ids = ascii_handler.collect_data_from_file(
             Path(repository_url) / "array-element-ids.json"
         )
-        sub_arrays = gen.collect_data_from_file(Path(repository_url) / "subarray-ids.json")
+        sub_arrays = ascii_handler.collect_data_from_file(
+            Path(repository_url) / "subarray-ids.json"
+        )
 
     return _get_ctao_layouts_per_site(site, sub_arrays, array_element_ids)
 
@@ -232,7 +234,7 @@ def get_array_layouts_from_parameter_file(
     list
         List of dictionaries containing array layout names and their elements.
     """
-    array_layouts = gen.collect_data_from_file(file_path)
+    array_layouts = ascii_handler.collect_data_from_file(file_path)
     try:
         value = array_layouts["value"]
     except KeyError as exc:
@@ -243,7 +245,12 @@ def get_array_layouts_from_parameter_file(
     for layout in value:
         layouts.append(
             _get_array_layout_dict(
-                db_config, model_version, site, None, layout["name"], coordinate_system
+                db_config,
+                model_version,
+                site,
+                layout.get("elements"),
+                layout["name"],
+                coordinate_system,
             )
         )
     return layouts
@@ -275,11 +282,7 @@ def get_array_layouts_from_db(
     """
     layout_names = []
     if layout_name:
-        layout_names.append(
-            layout_name[0]
-            if isinstance(layout_name, list) and len(layout_name) == 1
-            else layout_name
-        )
+        layout_names = gen.ensure_iterable(layout_name)
     else:
         site_model = SiteModel(site=site, model_version=model_version, mongo_db_config=db_config)
         layout_names = site_model.get_list_of_array_layouts()
@@ -387,3 +390,39 @@ def _get_array_layout_dict(
             coordinate_system=coordinate_system
         ),
     }
+
+
+def get_array_elements_from_db_for_layouts(layouts, site, model_version, db_config):
+    """
+    Get list of array elements from the database for given list of layout names.
+
+    Structure of the returned dictionary::
+
+        {
+            "layout_name_1": [telescope_id_1, telescope_id_2, ...],
+            "layout_name_2": [telescope_id_3, telescope_id_4, ...],
+            ...
+        }
+
+    Parameters
+    ----------
+    layouts : list[str]
+        List of layout names to read. If "all", read all available layouts.
+    site : str
+        Site name for the array layouts.
+    model_version : str
+        Model version for the array layouts.
+    db_config : dict
+        Database configuration dictionary.
+
+    Returns
+    -------
+    dict
+        Dictionary mapping layout names to telescope IDs.
+    """
+    site_model = SiteModel(site=site, model_version=model_version, mongo_db_config=db_config)
+    layout_names = site_model.get_list_of_array_layouts() if layouts == ["all"] else layouts
+    layout_dict = {}
+    for layout_name in layout_names:
+        layout_dict[layout_name] = site_model.get_array_elements_for_layout(layout_name)
+    return layout_dict