gammasimtools 0.18.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

simtools/db/db_handler.py

@@ -13,9 +13,9 @@ from packaging.version import Version
 from pymongo import MongoClient
 
 from simtools.data_model import validate_data
-from simtools.io_operations import io_handler
+from simtools.io import ascii_handler, io_handler
 from simtools.simtel import simtel_table_reader
-from simtools.utils import general, names, value_conversion
+from simtools.utils import names, value_conversion
 
 __all__ = ["DatabaseHandler"]
 
@@ -81,6 +81,9 @@ class DatabaseHandler:
 
         self._set_up_connection()
         self._find_latest_simulation_model_db()
+        self.db_name = (
+            self.mongo_db_config.get("db_simulation_model", None) if self.mongo_db_config else None
+        )
 
     def _set_up_connection(self):
         """Open the connection to MongoDB."""
@@ -179,6 +182,28 @@ class DatabaseHandler:
         else:
             raise ValueError("Found LATEST in the DB name but no matching versions found in DB.")
 
+    def generate_compound_indexes(self, db_name=None):
+        """
+        Generate compound indexes for the MongoDB collections.
+
+        Indexes based on the typical query patterns.
+        """
+        db_name = db_name or self.db_name
+        collection_names = [
+            "telescopes",
+            "sites",
+            "configuration_sim_telarray",
+            "configuration_corsika",
+            "calibration_devices",
+        ]
+        for collection_name in collection_names:
+            db_collection = self.get_collection(collection_name, db_name=db_name)
+            db_collection.create_index(
+                [("instrument", 1), ("site", 1), ("parameter", 1), ("parameter_version", 1)]
+            )
+        db_collection = self.get_collection("production_tables", db_name=db_name)
+        db_collection.create_index([("collection", 1), ("model_version", 1)])
+
     def get_model_parameter(
         self,
         parameter,
@@ -333,24 +358,23 @@ class DatabaseHandler:
         )
         return DatabaseHandler.model_parameters_cached[cache_key]
 
-    def get_collection(self, db_name, collection_name):
+    def get_collection(self, collection_name, db_name=None):
         """
         Get a collection from the DB.
 
         Parameters
         ----------
-        db_name: str
-            Name of the DB.
         collection_name: str
             Name of the collection.
+        db_name: str
+            Name of the DB.
 
         Returns
         -------
         pymongo.collection.Collection
             The collection from the DB.
-
         """
-        db_name = self._get_db_name(db_name)
+        db_name = db_name or self.db_name
         return DatabaseHandler.db_client[db_name][collection_name]
 
     def get_collections(self, db_name=None, model_collections_only=False):
@@ -370,7 +394,7 @@ class DatabaseHandler:
             List of collection names
 
         """
-        db_name = db_name or self._get_db_name()
+        db_name = db_name or self.db_name
         if db_name not in self.list_of_collections:
             self.list_of_collections[db_name] = DatabaseHandler.db_client[
                 db_name
@@ -451,7 +475,7 @@ class DatabaseHandler:
         file_id: dict of GridOut._id
             Dict of database IDs of files.
         """
-        db_name = self._get_db_name(db_name)
+        db_name = db_name or self.db_name
 
         if file_names:
             file_names = [file_names] if not isinstance(file_names, list) else file_names
@@ -467,8 +491,8 @@ class DatabaseHandler:
             if Path(dest).joinpath(file_name).exists():
                 instance_ids[file_name] = "file exists"
             else:
-                file_path_instance = self._get_file_mongo_db(self._get_db_name(), file_name)
-                self._write_file_from_mongo_to_disk(self._get_db_name(), dest, file_path_instance)
+                file_path_instance = self._get_file_mongo_db(db_name, file_name)
+                self._write_file_from_mongo_to_disk(db_name, dest, file_path_instance)
                 instance_ids[file_name] = file_path_instance._id  # pylint: disable=protected-access
         return instance_ids
 
@@ -509,8 +533,7 @@ class DatabaseHandler:
         ValueError
             if query returned no results.
         """
-        db_name = self._get_db_name()
-        collection = self.get_collection(db_name, collection_name)
+        collection = self.get_collection(collection_name, db_name=self.db_name)
         posts = list(collection.find(query))
         if not posts:
             raise ValueError(
@@ -547,7 +570,7 @@ class DatabaseHandler:
             pass
 
         query = {"model_version": model_version, "collection": collection_name}
-        collection = self.get_collection(self._get_db_name(), "production_tables")
+        collection = self.get_collection("production_tables", db_name=self.db_name)
         post = collection.find_one(query)
         if not post:
             raise ValueError(f"The following query returned zero results: {query}")
@@ -574,7 +597,7 @@ class DatabaseHandler:
         list
             List of model versions
         """
-        collection = self.get_collection(self._get_db_name(), "production_tables")
+        collection = self.get_collection("production_tables", db_name=self.db_name)
         return sorted(
             {post["model_version"] for post in collection.find({"collection": collection_name})}
         )
@@ -762,8 +785,7 @@ class DatabaseHandler:
         production_table: dict
             The production table to add to the DB.
         """
-        db_name = self._get_db_name(db_name)
-        collection = self.get_collection(db_name, "production_tables")
+        collection = self.get_collection("production_tables", db_name=db_name or self.db_name)
         self._logger.debug(f"Adding production for {production_table.get('collection')} to to DB")
         collection.insert_one(production_table)
         DatabaseHandler.production_table_cached.clear()
@@ -794,8 +816,8 @@ class DatabaseHandler:
         """
         par_dict = validate_data.DataValidator.validate_model_parameter(par_dict)
 
-        db_name = self._get_db_name(db_name)
-        collection = self.get_collection(db_name, collection_name)
+        db_name = db_name or self.db_name
+        collection = self.get_collection(collection_name, db_name=db_name)
 
         par_dict["value"], _base_unit, _ = value_conversion.get_value_unit_type(
             value=par_dict["value"], unit_str=par_dict.get("unit", None)
@@ -810,7 +832,7 @@ class DatabaseHandler:
                     f"corresponding to the {par_dict['parameter']} parameter, must be provided."
                 )
             file_path = Path(file_prefix).joinpath(par_dict["value"])
-            if not general.is_utf8_file(file_path):
+            if not ascii_handler.is_utf8_file(file_path):
                 raise ValueError(f"File is not UTF-8 encoded: {file_path}")
             files_to_add_to_db.add(f"{file_path}")
 
@@ -825,22 +847,6 @@ class DatabaseHandler:
 
         self._reset_parameter_cache()
 
-    def _get_db_name(self, db_name=None):
-        """
-        Return database name. If not provided, return the default database name.
-
-        Parameters
-        ----------
-        db_name: str
-            Database name
-
-        Returns
-        -------
-        str
-            Database name
-        """
-        return self.mongo_db_config["db_simulation_model"] if db_name is None else db_name
-
     def insert_file_to_db(self, file_name, db_name=None, **kwargs):
         """
         Insert a file to the DB.
@@ -863,7 +869,7 @@ class DatabaseHandler:
             "newly created DB GridOut._id.
 
         """
-        db_name = self._get_db_name(db_name)
+        db_name = db_name or self.db_name
         db = DatabaseHandler.db_client[db_name]
         file_system = gridfs.GridFS(db)
 

simtools/db/db_model_upload.py

@@ -3,7 +3,7 @@
 import logging
 from pathlib import Path
 
-import simtools.utils.general as gen
+from simtools.io import ascii_handler
 from simtools.utils import names
 
 logger = logging.getLogger(__name__)
@@ -26,7 +26,7 @@ def add_values_from_json_to_db(file, collection, db, db_name, file_prefix):
     file_prefix : str
         Path to location of all additional files to be uploaded.
     """
-    par_dict = gen.collect_data_from_file(file_name=file)
+    par_dict = ascii_handler.collect_data_from_file(file_name=file)
     logger.debug(
         f"Adding the following parameter to the DB: {par_dict['parameter']} "
         f"version {par_dict['parameter_version']} "
@@ -119,7 +119,7 @@ def _read_production_table(model_dict, file, model_name):
             "design_model": {},
         },
     )
-    parameter_dict = gen.collect_data_from_file(file_name=file)
+    parameter_dict = ascii_handler.collect_data_from_file(file_name=file)
     logger.debug(f"Reading production table for {array_element} (collection {collection})")
     try:
         if array_element in ("configuration_corsika", "configuration_sim_telarray"):

simtools/dependencies.py

@@ -14,18 +14,37 @@ import re
 import subprocess
 from pathlib import Path
 
-import simtools.utils.general as gen
+import yaml
+
 from simtools.db.db_handler import DatabaseHandler
+from simtools.io import ascii_handler
 
 _logger = logging.getLogger(__name__)
 
 
-def get_version_string(db_config=None):
-    """Print the versions of the dependencies."""
+def get_version_string(db_config=None, run_time=None):
+    """
+    Print the versions of the dependencies.
+
+    Parameters
+    ----------
+    db_config : dict, optional
+        Database configuration dictionary.
+    run_time : list, optional
+        Runtime environment command (e.g., Docker).
+
+    Returns
+    -------
+    str
+        String containing the versions of the dependencies.
+
+    """
     return (
         f"Database version: {get_database_version(db_config)}\n"
-        f"sim_telarray version: {get_sim_telarray_version()}\n"
-        f"CORSIKA version: {get_corsika_version()}\n"
+        f"sim_telarray version: {get_sim_telarray_version(run_time)}\n"
+        f"CORSIKA version: {get_corsika_version(run_time)}\n"
+        f"Build options: {get_build_options(run_time)}\n"
+        f"Runtime environment: {run_time if run_time else 'None'}\n"
     )
 
 
@@ -50,10 +69,15 @@ def get_database_version(db_config):
     return db.mongo_db_config.get("db_simulation_model")
 
 
-def get_sim_telarray_version():
+def get_sim_telarray_version(run_time):
     """
     Get the version of the sim_telarray package using 'sim_telarray --version'.
 
+    Parameters
+    ----------
+    run_time : list, optional
+        Runtime environment command (e.g., Docker).
+
     Returns
     -------
     str
@@ -65,24 +89,33 @@ def get_sim_telarray_version():
         return None
     sim_telarray_path = Path(sim_telarray_path) / "sim_telarray" / "bin" / "sim_telarray"
 
+    if run_time is None:
+        command = [str(sim_telarray_path), "--version"]
+    else:
+        command = [*run_time, str(sim_telarray_path), "--version"]
+
+    _logger.debug(f"Running command: {command}")
+    result = subprocess.run(command, capture_output=True, text=True, check=False)
+
     # expect stdout with e.g. a line 'Release: 2024.271.0 from 2024-09-27'
-    result = subprocess.run(
-        [sim_telarray_path, "--version"],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
     match = re.search(r"^Release:\s+(.+)", result.stdout, re.MULTILINE)
-
     if match:
         return match.group(1).split()[0]
+
+    _logger.debug(f"Command output stdout: {result.stdout} stderr: {result.stderr}")
+
     raise ValueError(f"sim_telarray release not found in {result.stdout}")
 
 
-def get_corsika_version():
+def get_corsika_version(run_time=None):
     """
     Get the version of the CORSIKA package.
 
+    Parameters
+    ----------
+    run_time : list, optional
+        Runtime environment command (e.g., Docker).
+
     Returns
     -------
     str
@@ -95,10 +128,15 @@ def get_corsika_version():
         return None
     corsika_command = Path(sim_telarray_path) / "corsika-run" / "corsika"
 
+    if run_time is None:
+        command = [str(corsika_command)]
+    else:
+        command = [*run_time, str(corsika_command)]
+
     # Below I do not use the standard context manager because
     # it makes mocking in the tests significantly more difficult
     process = subprocess.Popen(  # pylint: disable=consider-using-with
-        corsika_command,
+        command,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
         stdin=subprocess.PIPE,
@@ -123,25 +161,50 @@ def get_corsika_version():
     if version and re.match(r"\d+\.\d+", version):
         return version
     try:
-        build_opts = get_build_options()
-    except (FileNotFoundError, TypeError):
+        build_opts = get_build_options(run_time)
+    except (FileNotFoundError, TypeError, ValueError):
         _logger.warning("Could not get CORSIKA version.")
         return None
     _logger.debug("Getting the CORSIKA version from the build options.")
     return build_opts.get("corsika_version")
 
 
-def get_build_options():
+def get_build_options(run_time=None):
     """
     Return CORSIKA / sim_telarray build options.
 
     Expects a build_opts.yml file in the sim_telarray directory.
+
+    Parameters
+    ----------
+    run_time : list, optional
+        Runtime environment command (e.g., Docker).
+
+    Returns
+    -------
+    dict
+        Build options from build_opts.yml file.
     """
+    sim_telarray_path = os.getenv("SIMTOOLS_SIMTEL_PATH")
+    if sim_telarray_path is None:
+        raise ValueError("SIMTOOLS_SIMTEL_PATH not defined.")
+
+    build_opts_path = Path(sim_telarray_path) / "build_opts.yml"
+
+    if run_time is None:
+        try:
+            return ascii_handler.collect_data_from_file(build_opts_path)
+        except FileNotFoundError as exc:
+            raise FileNotFoundError("No build_opts.yml file found.") from exc
+
+    command = [*run_time, "cat", str(build_opts_path)]
+    _logger.debug(f"Reading build_opts.yml with command: {command}")
+
+    result = subprocess.run(command, capture_output=True, text=True, check=False)
+    if result.returncode:
+        raise FileNotFoundError(f"No build_opts.yml file found in container: {result.stderr}")
+
     try:
-        return gen.collect_data_from_file(
-            Path(os.getenv("SIMTOOLS_SIMTEL_PATH")) / "build_opts.yml"
-        )
-    except FileNotFoundError as exc:
-        raise FileNotFoundError("No build_opts.yml file found.") from exc
-    except TypeError as exc:
-        raise TypeError("SIMTOOLS_SIMTEL_PATH not defined.") from exc
+        return yaml.safe_load(result.stdout)
+    except yaml.YAMLError as exc:
+        raise ValueError(f"Error parsing build_opts.yml from container: {exc}") from exc

simtools/io/ascii_handler.py

@@ -0,0 +1,279 @@
+"""Helper module for ASCII file operations."""
+
+import json
+import logging
+import tempfile
+import urllib.request
+from pathlib import Path
+
+import astropy.units as u
+import numpy as np
+import yaml
+
+from simtools.utils.general import is_url
+
+_logger = logging.getLogger(__name__)
+
+
+def collect_data_from_file(file_name, yaml_document=None):
+    """
+    Collect data from file based on its extension.
+
+    Parameters
+    ----------
+    file_name: str
+        Name of the yaml/json/ascii file.
+    yaml_document: None, int
+        Return list of yaml documents or a single document (for yaml files with several documents).
+
+    Returns
+    -------
+    data: dict or list
+        Data as dict or list.
+    """
+    if is_url(file_name):
+        return collect_data_from_http(file_name)
+
+    suffix = Path(file_name).suffix.lower()
+    try:
+        with open(file_name, encoding="utf-8") as file:
+            return _collect_data_from_different_file_types(file, file_name, suffix, yaml_document)
+    # broad exception to catch all possible errors in reading the file
+    except Exception as exc:  # pylint: disable=broad-except
+        raise type(exc)(f"Failed to read file {file_name}: {exc}") from exc
+
+
+def _collect_data_from_different_file_types(file, file_name, suffix, yaml_document):
+    """Collect data from different file types."""
+    if suffix == ".json":
+        return json.load(file)
+    if suffix in (".list", ".txt"):
+        return [line.strip() for line in file.readlines()]
+    if suffix in [".yml", ".yaml"]:
+        return _collect_data_from_yaml_file(file, file_name, yaml_document)
+    raise TypeError(f"File type {suffix} not supported.")
+
+
+def _collect_data_from_yaml_file(file, file_name, yaml_document):
+    """Collect data from a yaml file (allow for multi-document yaml files)."""
+    try:
+        return yaml.safe_load(file)
+    except yaml.constructor.ConstructorError:
+        return _load_yaml_using_astropy(file)
+    except yaml.composer.ComposerError:
+        pass
+    file.seek(0)
+    if yaml_document is None:
+        return list(yaml.safe_load_all(file))
+    try:
+        return list(yaml.safe_load_all(file))[yaml_document]
+    except IndexError as exc:
+        raise IndexError(
+            f"Failed to read file {file_name}: YAML document index {yaml_document} is out of range."
+        ) from exc
+
+
+def _load_yaml_using_astropy(file):
+    """
+    Load a yaml file using astropy's yaml loader.
+
+    Parameters
+    ----------
+    file: file
+        File to be loaded.
+
+    Returns
+    -------
+    dict
+        Dictionary containing the file content.
+    """
+    # pylint: disable=import-outside-toplevel
+    import astropy.io.misc.yaml as astropy_yaml
+
+    file.seek(0)
+    return astropy_yaml.load(file)
+
+
+def collect_data_from_http(url):
+    """
+    Download yaml or json file from url and return it contents as dict.
+
+    File is downloaded as a temporary file and deleted afterwards.
+
+    Parameters
+    ----------
+    url: str
+        URL of the yaml/json file.
+
+    Returns
+    -------
+    dict
+        Dictionary containing the file content.
+
+    Raises
+    ------
+    TypeError
+        If url is not a valid URL.
+    FileNotFoundError
+        If downloading the yaml file fails.
+
+    """
+    try:
+        with tempfile.NamedTemporaryFile(mode="w+t") as tmp_file:
+            urllib.request.urlretrieve(url, tmp_file.name)
+            data = _collect_data_from_different_file_types(
+                tmp_file, url, Path(url).suffix.lower(), None
+            )
+    except TypeError as exc:
+        raise TypeError(f"Invalid url {url}") from exc
+    except urllib.error.HTTPError as exc:
+        raise FileNotFoundError(f"Failed to download file from {url}") from exc
+
+    _logger.debug(f"Downloaded file from {url}")
+    return data
+
+
+def read_file_encoded_in_utf_or_latin(file_name):
+    """
+    Read a file encoded in UTF-8 or Latin-1.
+
+    Parameters
+    ----------
+    file_name: str
+        Name of the file to be read.
+
+    Returns
+    -------
+    list
+        List of lines read from the file.
+
+    Raises
+    ------
+    UnicodeDecodeError
+        If the file cannot be decoded using UTF-8 or Latin-1.
+    """
+    try:
+        with open(file_name, encoding="utf-8") as file:
+            lines = file.readlines()
+    except UnicodeDecodeError:
+        _logger.debug("Unable to decode file using UTF-8. Trying Latin-1.")
+        try:
+            with open(file_name, encoding="latin-1") as file:
+                lines = file.readlines()
+        except UnicodeDecodeError as exc:
+            msg = f"Unable to decode file {file_name} using UTF-8 or Latin-1."
+            raise UnicodeDecodeError(exc.encoding, exc.object, exc.start, exc.end, msg) from exc
+
+    return lines
+
+
+def is_utf8_file(file_name):
+    """
+    Check if a file is encoded in UTF-8.
+
+    Parameters
+    ----------
+    file_name: str, Path
+        Name of the file to be checked.
+
+    Returns
+    -------
+    bool
+        True if the file is encoded in UTF-8, False otherwise.
+    """
+    try:
+        with open(file_name, encoding="utf-8") as file:
+            file.read()
+        return True
+    except UnicodeDecodeError:
+        return False
+
+
+def write_data_to_file(data, output_file, sort_keys=False, numpy_types=False):
+    """
+    Write structured data to JSON or YAML file.
+
+    The file type is determined by the file extension.
+
+    Parameters
+    ----------
+    data: dict or list
+        Data to be written to the file.
+    output_file: str or Path
+        Name of the file to be written.
+    sort_keys: bool, optional
+        If True, sort the keys.
+    numpy_types: bool, optional
+        If True, convert numpy types to native Python types.
+    """
+    output_file = Path(output_file)
+    if output_file.suffix.lower() == ".json":
+        return _write_to_json(data, output_file, sort_keys, numpy_types)
+    if output_file.suffix.lower() in [".yml", ".yaml"]:
+        return _write_to_yaml(data, output_file, sort_keys)
+
+    raise ValueError(
+        f"Unsupported file type {output_file.suffix}. Only .json, .yml, and .yaml are supported."
+    )
+
+
+def _write_to_json(data, output_file, sort_keys, numpy_types):
+    """
+    Write data to a JSON file.
+
+    Parameters
+    ----------
+    data: dict or list
+        Data to be written to the file.
+    output_file: Path
+        Name of the file to be written.
+    sort_keys: bool
+        If True, sort the keys.
+    numpy_types: bool
+        If True, convert numpy types to native Python types.
+    """
+    with open(output_file, "w", encoding="utf-8") as file:
+        json.dump(
+            data,
+            file,
+            indent=4,
+            sort_keys=sort_keys,
+            cls=JsonNumpyEncoder if numpy_types else None,
+        )
+        file.write("\n")
+
+
+def _write_to_yaml(data, output_file, sort_keys):
+    """
+    Write data to a YAML file.
+
+    Parameters
+    ----------
+    data: dict or list
+        Data to be written to the file.
+    output_file: Path
+        Name of the file to be written.
+    sort_keys: bool
+        If True, sort the keys.
+
+    """
+    with open(output_file, "w", encoding="utf-8") as file:
+        yaml.dump(data, file, indent=4, sort_keys=sort_keys, explicit_start=True)
+
+
+class JsonNumpyEncoder(json.JSONEncoder):
+    """Convert numpy to python types as accepted by json.dump."""
+
+    def default(self, o):
+        """Return default encoder."""
+        if isinstance(o, np.floating):
+            return float(o)
+        if isinstance(o, np.integer):
+            return int(o)
+        if isinstance(o, np.ndarray):
+            return o.tolist()
+        if isinstance(o, u.core.CompositeUnit | u.core.IrreducibleUnit | u.core.Unit):
+            return str(o) if o != u.dimensionless_unscaled else None
+        if np.issubdtype(type(o), np.bool_):
+            return bool(o)
+        return super().default(o)