gammasimtools 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

simtools/db/db_model_upload.py

@@ -0,0 +1,139 @@
+"""Upload a simulation model (parameters and production tables) to the database."""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.utils import names
+
+logger = logging.getLogger(__name__)
+
+
+def add_values_from_json_to_db(file, collection, db, db_name, file_prefix):
+    """
+    Upload new model parameter from json files to db.
+
+    Parameters
+    ----------
+    file : list
+        Json file to be uploaded to the DB.
+    collection : str
+        The DB collection to which to add the file.
+    db : DatabaseHandler
+        Database handler object.
+    db_name : str
+        Name of the database to be created.
+    file_prefix : str
+        Path to location of all additional files to be uploaded.
+    """
+    par_dict = gen.collect_data_from_file(file_name=file)
+    logger.info(
+        f"Adding the following parameter to the DB: {par_dict['parameter']} "
+        f"version {par_dict['parameter_version']} "
+        f"(collection {collection} in database {db_name})"
+    )
+
+    db.add_new_parameter(
+        db_name=db_name,
+        par_dict=par_dict,
+        collection_name=collection,
+        file_prefix=file_prefix,
+    )
+
+
+def add_model_parameters_to_db(args_dict, db):
+    """
+    Read model parameters from a directory and upload them to the database.
+
+    Parameters
+    ----------
+    args_dict : dict
+        Command line arguments.
+    db : DatabaseHandler
+        Database handler object.
+    """
+    input_path = Path(args_dict["input_path"])
+    logger.info(f"Reading model parameters from repository path {input_path}")
+    array_elements = [d for d in input_path.iterdir() if d.is_dir()]
+    for element in array_elements:
+        collection = names.get_collection_name_from_array_element_name(element.name, False)
+        if collection == "Files":
+            logger.info("Files (tables) are uploaded with the corresponding model parameters")
+            continue
+        logger.info(f"Reading model parameters for {element.name} into collection {collection}")
+        files_to_insert = list(Path(element).rglob("*json"))
+        for file in files_to_insert:
+            add_values_from_json_to_db(
+                file=file,
+                collection=collection,
+                db=db,
+                db_name=args_dict["db_name"],
+                file_prefix=input_path / "Files",
+            )
+
+
+def add_production_tables_to_db(args_dict, db):
+    """
+    Read production tables from a directory and upload them to the database.
+
+    One dictionary per collection is prepared for each model version, containing
+    tables of all array elements, sites, and configuration parameters.
+
+    Parameters
+    ----------
+    args_dict : dict
+        Command line arguments.
+    db : DatabaseHandler
+        Database handler object.
+    """
+    input_path = Path(args_dict["input_path"])
+    logger.info(f"Reading production tables from repository path {input_path}")
+
+    for model in filter(Path.is_dir, input_path.iterdir()):
+        logger.info(f"Reading production tables for model version {model.name}")
+        model_dict = {}
+        for file in sorted(model.rglob("*json")):
+            _read_production_table(model_dict, file, model.name)
+
+        for collection, data in model_dict.items():
+            if not data["parameters"]:
+                logger.info(f"No production table for {collection} in model version {model.name}")
+                continue
+            logger.info(f"Adding production table for {collection} to the database")
+            db.add_production_table(
+                db_name=args_dict["db_name"],
+                production_table=data,
+            )
+
+
+def _read_production_table(model_dict, file, model_name):
+    """Read a single production table from file."""
+    array_element = file.stem
+    collection = names.get_collection_name_from_array_element_name(array_element, False)
+    model_dict.setdefault(
+        collection,
+        {
+            "collection": collection,
+            "model_version": model_name,
+            "parameters": {},
+            "design_model": {},
+        },
+    )
+    parameter_dict = gen.collect_data_from_file(file_name=file)
+    logger.info(f"Reading production table for {array_element} (collection {collection})")
+    try:
+        if array_element in ("configuration_corsika", "configuration_sim_telarray"):
+            model_dict[collection]["parameters"] = parameter_dict["parameters"]
+        else:
+            model_dict[collection]["parameters"][array_element] = parameter_dict["parameters"][
+                array_element
+            ]
+    except KeyError as exc:
+        logger.error(f"KeyError: {exc}")
+        raise
+    try:
+        model_dict[collection]["design_model"][array_element] = parameter_dict["design_model"][
+            array_element
+        ]
+    except KeyError:
+        pass

simtools/dependencies.py

@@ -0,0 +1,112 @@
+"""
+Simtools dependencies version management.
+
+This modules provides two main functionalities:
+
+- retrieve the versions of simtools dependencies (e.g., databases, sim_telarray, CORSIKA)
+- provide space for future implementations of version management
+
+"""
+
+import logging
+import os
+import re
+import subprocess
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.db.db_handler import DatabaseHandler
+
+_logger = logging.getLogger(__name__)
+
+
+def get_version_string(db_config=None):
+    """Print the versions of the dependencies."""
+    return (
+        f"Database version: {get_database_version(db_config)}\n"
+        f"sim_telarray version: {get_sim_telarray_version()}\n"
+        f"CORSIKA version: {get_corsika_version()}\n"
+    )
+
+
+def get_database_version(db_config):
+    """
+    Get the version of the simulation model data base used.
+
+    Parameters
+    ----------
+    db_config : dict
+        Dictionary containing the database configuration.
+
+    Returns
+    -------
+    str
+        Version of the simulation model data base used.
+
+    """
+    if db_config is None:
+        return None
+    db = DatabaseHandler(db_config)
+    return db.mongo_db_config.get("db_simulation_model")
+
+
+def get_sim_telarray_version():
+    """
+    Get the version of the sim_telarray package using 'sim_telarray --version'.
+
+    Returns
+    -------
+    str
+        Version of the sim_telarray package.
+    """
+    sim_telarray_path = os.getenv("SIMTOOLS_SIMTEL_PATH")
+    if sim_telarray_path is None:
+        _logger.warning("Environment variable SIMTOOLS_SIMTEL_PATH is not set.")
+        return None
+    sim_telarray_path = Path(sim_telarray_path) / "sim_telarray" / "bin" / "sim_telarray"
+
+    # expect stdout with e.g. a line 'Release: 2024.271.0 from 2024-09-27'
+    result = subprocess.run(
+        [sim_telarray_path, "--version"],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    match = re.search(r"^Release:\s+(.+)", result.stdout, re.MULTILINE)
+
+    if match:
+        return match.group(1).split()[0]
+    raise ValueError(f"sim_telarray release not found in {result.stdout}")
+
+
+def get_corsika_version():
+    """
+    Get the version of the corsika package.
+
+    Returns
+    -------
+    str
+        Version of the corsika package.
+    """
+    try:
+        build_opts = get_build_options()
+    except (FileNotFoundError, TypeError):
+        _logger.warning("CORSIKA version not implemented yet.")
+        return None
+    return build_opts.get("corsika_version")
+
+
+def get_build_options():
+    """
+    Return CORSIKA / sim_telarray build options.
+
+    Expects a build_opts.yml file in the sim_telarray directory.
+    """
+    try:
+        return gen.collect_data_from_file(
+            Path(os.getenv("SIMTOOLS_SIMTEL_PATH")) / "build_opts.yml"
+        )
+    except FileNotFoundError as exc:
+        raise FileNotFoundError("No build_opts.yml file found.") from exc
+    except TypeError as exc:
+        raise TypeError("SIMTOOLS_SIMTEL_PATH not defined.") from exc

simtools/io_operations/hdf5_handler.py

@@ -41,40 +41,26 @@ def fill_hdf5_table(hist, x_bin_edges, y_bin_edges, x_label, y_label, meta_data)
     meta_data: dict
         Dictionary with the histogram metadata.
     """
-    # Complement metadata
-    if x_label is not None:
-        meta_data["x_bin_edges"] = sanitize_name(x_label)
+    validate_histogram(hist, y_bin_edges)
+
+    meta_data["x_bin_edges"] = x_bin_edges
     meta_data["x_bin_edges_unit"] = (
         x_bin_edges.unit if isinstance(x_bin_edges, u.Quantity) else u.dimensionless_unscaled
     )
-
     if y_bin_edges is not None:
-        if y_label is not None:
-            meta_data["y_bin_edges"] = sanitize_name(y_label)
-            names = [
-                f"{meta_data['y_bin_edges'].split('__')[0]}_{i}"
-                for i in range(len(y_bin_edges[:-1]))
-            ]
-        else:
-            names = [
-                f"{meta_data['Title'].split('__')[0]}_{i}" for i in range(len(y_bin_edges[:-1]))
-            ]
+        meta_data["y_bin_edges"] = y_bin_edges
         meta_data["y_bin_edges_unit"] = (
             y_bin_edges.unit if isinstance(y_bin_edges, u.Quantity) else u.dimensionless_unscaled
         )
 
-        table = Table(
-            [hist[i, :] for i in range(len(y_bin_edges[:-1]))],
-            names=names,
-            meta=meta_data,
-        )
-
-    else:
+    if hist.ndim == 1:
         if x_label is not None:
-            meta_data["x_bin_edges"] = sanitize_name(x_label)
-            names = meta_data["x_bin_edges"]
-        else:
+            names = sanitize_name(x_label)
+        try:
             names = meta_data["Title"]
+        except KeyError:
+            _logger.warning("Title not found in metadata.")
+
         table = Table(
             [
                 x_bin_edges[:-1],
@@ -83,9 +69,53 @@ def fill_hdf5_table(hist, x_bin_edges, y_bin_edges, x_label, y_label, meta_data)
             names=(names, sanitize_name("Values")),
             meta=meta_data,
         )
+    else:
+        if y_label is not None:
+            names = [
+                f"{sanitize_name(y_label).split('__')[0]}_{i}" for i in range(len(y_bin_edges[:-1]))
+            ]
+        try:
+            names = [
+                f"{(meta_data['Title']).split('__')[0]}_{sanitize_name(y_label)}_{i}"
+                for i in range(len(y_bin_edges[:-1]))
+            ]
+        except KeyError:
+            _logger.warning("Title not found in metadata.")
+            names = [
+                f"{sanitize_name(y_label).split('__')[0]}_{i}" for i in range(len(y_bin_edges[:-1]))
+            ]
+
+        table = Table(
+            [hist[i, :] for i in range(len(y_bin_edges[:-1]))],
+            names=names,
+            meta=meta_data,
+        )
+
     return table
 
 
+def validate_histogram(hist, y_bin_edges):
+    """Validate histogram dimensions and y_bin_edges consistency.
+
+    Parameters
+    ----------
+    hist (np.ndarray): The histogram array, expected to be 1D or 2D.
+    y_bin_edges (array-like or None): Bin edges for the second dimension (if applicable).
+
+    Raises
+    ------
+    ValueError: If histogram dimensions are invalid or inconsistent with y_bin_edges.
+    """
+    if hist.ndim not in (1, 2):
+        raise ValueError("Histogram must be either 1D or 2D.")
+
+    if hist.ndim == 1 and y_bin_edges is not None:
+        raise ValueError("y_bin_edges should be None for 1D histograms.")
+
+    if hist.ndim == 2 and y_bin_edges is None:
+        raise ValueError("y_bin_edges should not be None for 2D histograms.")
+
+
 def read_hdf5(hdf5_file_name):
     """
     Read a hdf5 output file.

simtools/layout/array_layout.py

@@ -9,7 +9,7 @@ import numpy as np
 from astropy.table import QTable
 
 import simtools.utils.general as gen
-from simtools.data_model import data_reader
+from simtools.data_model import data_reader, schema
 from simtools.io_operations import io_handler
 from simtools.layout.geo_coordinates import GeoCoordinates
 from simtools.layout.telescope_position import TelescopePosition
@@ -386,7 +386,9 @@ class ArrayLayout:
         with Path(file_name).open("r", encoding="utf-8") as file:
             data = json.load(file)
 
-        position = gen.convert_string_to_list(data["value"])
+        position = data["value"]
+        if isinstance(position, str):
+            position = gen.convert_string_to_list(position)
         self.site = data.get("site", None)
 
         table = QTable()
@@ -443,9 +445,11 @@ class ArrayLayout:
             )
             try:
                 tel_model = self._get_telescope_model(telescope_name)
-            except ValueError:
+            except ValueError:  # telescope not found in the database revert to design model
                 tel_model = self._get_telescope_model(
-                    names.get_array_element_type_from_name(telescope_name) + "-design",
+                    names.array_element_design_types(
+                        names.get_array_element_type_from_name(telescope_name)
+                    )[0]
                 )
 
             for para in ("telescope_axis_height", "telescope_sphere_radius"):
@@ -579,7 +583,12 @@ class ArrayLayout:
 
         return table
 
-    def export_one_telescope_as_json(self, crs_name):
+    def export_one_telescope_as_json(
+        self,
+        crs_name,
+        parameter_version=None,
+        schema_version=None,
+    ):
         """
         Return a list containing a single telescope in simtools-DB-style json.
 
@@ -587,6 +596,8 @@ class ArrayLayout:
         ----------
         crs_name: str
             Name of coordinate system to be used for export.
+        schema_version: str
+            Version of the schema.
 
         Returns
         -------
@@ -596,43 +607,39 @@ class ArrayLayout:
         table = self.export_telescope_list_table(crs_name)
         if len(table) != 1:
             raise ValueError("Only one telescope can be exported to json")
-        parameter_name = value_string = None
+        parameter_name = value = None
         if crs_name == "ground":
             parameter_name = "array_element_position_ground"
-            value_string = gen.convert_list_to_string(
-                [
-                    table["position_x"][0].value,
-                    table["position_y"][0].value,
-                    table["position_z"][0].value,
-                ]
-            )
+            value = [
+                table["position_x"][0].value,
+                table["position_y"][0].value,
+                table["position_z"][0].value,
+            ]
         elif crs_name == "utm":
             parameter_name = "array_element_position_utm"
-            value_string = gen.convert_list_to_string(
-                [
-                    table["utm_east"][0].value,
-                    table["utm_north"][0].value,
-                    table["altitude"][0].value,
-                ]
-            )
+            value = [
+                table["utm_east"][0].value,
+                table["utm_north"][0].value,
+                table["altitude"][0].value,
+            ]
         elif crs_name == "mercator":
             parameter_name = "array_element_position_mercator"
-            value_string = gen.convert_list_to_string(
-                [
-                    table["latitude"][0].value,
-                    table["longitude"][0].value,
-                    table["altitude"][0].value,
-                ]
-            )
+            value = [
+                table["latitude"][0].value,
+                table["longitude"][0].value,
+                table["altitude"][0].value,
+            ]
+
         return {
+            "schema_version": schema.get_model_parameter_schema_version(schema_version),
             "parameter": parameter_name,
             "instrument": table["telescope_name"][0],
             "site": self.site,
-            "version": self.model_version,
-            "value": value_string,
+            "parameter_version": parameter_version,
+            "unique_id": None,
+            "value": value,
             "unit": "m",
             "type": "float64",
-            "applicable": True,
             "file": False,
         }
 
@@ -711,8 +718,7 @@ class ArrayLayout:
                     set(_telescope_list_from_name + _telescope_list_from_type)
                 )
             self._logger.info(
-                f"Selected {len(self._telescope_list)} telescopes"
-                f" (from originally {_n_telescopes})"
+                f"Selected {len(self._telescope_list)} telescopes (from originally {_n_telescopes})"
             )
         except TypeError:
             self._logger.info("No asset list provided, keeping all telescopes")

simtools/model/array_model.py

@@ -6,8 +6,8 @@ from pathlib import Path
 import astropy.units as u
 from astropy.table import QTable
 
-from simtools.data_model import data_reader
-from simtools.db import db_array_elements, db_handler
+from simtools.data_model import data_reader, schema
+from simtools.db import db_handler
 from simtools.io_operations import io_handler
 from simtools.model.site_model import SiteModel
 from simtools.model.telescope_model import TelescopeModel
@@ -290,7 +290,13 @@ class ArrayModel:
         }
 
     def _get_telescope_position_parameter(
-        self, telescope_name: str, site: str, x: u.Quantity, y: u.Quantity, z: u.Quantity
+        self,
+        telescope_name: str,
+        site: str,
+        x: u.Quantity,
+        y: u.Quantity,
+        z: u.Quantity,
+        parameter_version: str | None = None,
     ) -> dict:
         """
         Return dictionary with telescope position parameters (following DB model database format).
@@ -314,16 +320,17 @@ class ArrayModel:
             Dict with telescope position parameters.
         """
         return {
+            "schema_version": schema.get_model_parameter_schema_version(),
             "parameter": "array_element_position_ground",
             "instrument": telescope_name,
             "site": site,
-            "version": self.model_version,
+            "parameter_version": parameter_version,
+            "unique_id": None,
             "value": general.convert_list_to_string(
                 [x.to("m").value, y.to("m").value, z.to("m").value]
             ),
             "unit": "m",
             "type": "float64",
-            "applicable": True,
             "file": False,
         }
 
@@ -367,9 +374,8 @@ class ArrayModel:
         dict
             Dict with array elements.
         """
-        all_elements = db_array_elements.get_array_elements_of_type(
+        all_elements = self.db.get_array_elements_of_type(
             array_element_type=array_element_type,
-            db=self.db,
             model_version=self.model_version,
             collection="telescopes",
         )