gammasimtools 0.9.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

simtools/db/db_model_upload.py

@@ -0,0 +1,139 @@
+"""Upload a simulation model (parameters and production tables) to the database."""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.utils import names
+
+logger = logging.getLogger(__name__)
+
+
+def add_values_from_json_to_db(file, collection, db, db_name, file_prefix):
+    """
+    Upload new model parameter from json files to db.
+
+    Parameters
+    ----------
+    file : list
+        Json file to be uploaded to the DB.
+    collection : str
+        The DB collection to which to add the file.
+    db : DatabaseHandler
+        Database handler object.
+    db_name : str
+        Name of the database to be created.
+    file_prefix : str
+        Path to location of all additional files to be uploaded.
+    """
+    par_dict = gen.collect_data_from_file(file_name=file)
+    logger.info(
+        f"Adding the following parameter to the DB: {par_dict['parameter']} "
+        f"version {par_dict['parameter_version']} "
+        f"(collection {collection} in database {db_name})"
+    )
+
+    db.add_new_parameter(
+        db_name=db_name,
+        par_dict=par_dict,
+        collection_name=collection,
+        file_prefix=file_prefix,
+    )
+
+
+def add_model_parameters_to_db(args_dict, db):
+    """
+    Read model parameters from a directory and upload them to the database.
+
+    Parameters
+    ----------
+    args_dict : dict
+        Command line arguments.
+    db : DatabaseHandler
+        Database handler object.
+    """
+    input_path = Path(args_dict["input_path"])
+    logger.info(f"Reading model parameters from repository path {input_path}")
+    array_elements = [d for d in input_path.iterdir() if d.is_dir()]
+    for element in array_elements:
+        collection = names.get_collection_name_from_array_element_name(element.name, False)
+        if collection == "Files":
+            logger.info("Files (tables) are uploaded with the corresponding model parameters")
+            continue
+        logger.info(f"Reading model parameters for {element.name} into collection {collection}")
+        files_to_insert = list(Path(element).rglob("*json"))
+        for file in files_to_insert:
+            add_values_from_json_to_db(
+                file=file,
+                collection=collection,
+                db=db,
+                db_name=args_dict["db_name"],
+                file_prefix=input_path / "Files",
+            )
+
+
+def add_production_tables_to_db(args_dict, db):
+    """
+    Read production tables from a directory and upload them to the database.
+
+    One dictionary per collection is prepared for each model version, containing
+    tables of all array elements, sites, and configuration parameters.
+
+    Parameters
+    ----------
+    args_dict : dict
+        Command line arguments.
+    db : DatabaseHandler
+        Database handler object.
+    """
+    input_path = Path(args_dict["input_path"])
+    logger.info(f"Reading production tables from repository path {input_path}")
+
+    for model in filter(Path.is_dir, input_path.iterdir()):
+        logger.info(f"Reading production tables for model version {model.name}")
+        model_dict = {}
+        for file in sorted(model.rglob("*json")):
+            _read_production_table(model_dict, file, model.name)
+
+        for collection, data in model_dict.items():
+            if not data["parameters"]:
+                logger.info(f"No production table for {collection} in model version {model.name}")
+                continue
+            logger.info(f"Adding production table for {collection} to the database")
+            db.add_production_table(
+                db_name=args_dict["db_name"],
+                production_table=data,
+            )
+
+
+def _read_production_table(model_dict, file, model_name):
+    """Read a single production table from file."""
+    array_element = file.stem
+    collection = names.get_collection_name_from_array_element_name(array_element, False)
+    model_dict.setdefault(
+        collection,
+        {
+            "collection": collection,
+            "model_version": model_name,
+            "parameters": {},
+            "design_model": {},
+        },
+    )
+    parameter_dict = gen.collect_data_from_file(file_name=file)
+    logger.info(f"Reading production table for {array_element} (collection {collection})")
+    try:
+        if array_element in ("configuration_corsika", "configuration_sim_telarray"):
+            model_dict[collection]["parameters"] = parameter_dict["parameters"]
+        else:
+            model_dict[collection]["parameters"][array_element] = parameter_dict["parameters"][
+                array_element
+            ]
+    except KeyError as exc:
+        logger.error(f"KeyError: {exc}")
+        raise
+    try:
+        model_dict[collection]["design_model"][array_element] = parameter_dict["design_model"][
+            array_element
+        ]
+    except KeyError:
+        pass

simtools/io_operations/hdf5_handler.py

@@ -41,40 +41,26 @@ def fill_hdf5_table(hist, x_bin_edges, y_bin_edges, x_label, y_label, meta_data)
     meta_data: dict
         Dictionary with the histogram metadata.
     """
-    # Complement metadata
-    if x_label is not None:
-        meta_data["x_bin_edges"] = sanitize_name(x_label)
+    validate_histogram(hist, y_bin_edges)
+
+    meta_data["x_bin_edges"] = x_bin_edges
     meta_data["x_bin_edges_unit"] = (
         x_bin_edges.unit if isinstance(x_bin_edges, u.Quantity) else u.dimensionless_unscaled
     )
-
     if y_bin_edges is not None:
-        if y_label is not None:
-            meta_data["y_bin_edges"] = sanitize_name(y_label)
-            names = [
-                f"{meta_data['y_bin_edges'].split('__')[0]}_{i}"
-                for i in range(len(y_bin_edges[:-1]))
-            ]
-        else:
-            names = [
-                f"{meta_data['Title'].split('__')[0]}_{i}" for i in range(len(y_bin_edges[:-1]))
-            ]
+        meta_data["y_bin_edges"] = y_bin_edges
         meta_data["y_bin_edges_unit"] = (
             y_bin_edges.unit if isinstance(y_bin_edges, u.Quantity) else u.dimensionless_unscaled
         )
 
-        table = Table(
-            [hist[i, :] for i in range(len(y_bin_edges[:-1]))],
-            names=names,
-            meta=meta_data,
-        )
-
-    else:
+    if hist.ndim == 1:
         if x_label is not None:
-            meta_data["x_bin_edges"] = sanitize_name(x_label)
-            names = meta_data["x_bin_edges"]
-        else:
+            names = sanitize_name(x_label)
+        try:
             names = meta_data["Title"]
+        except KeyError:
+            _logger.warning("Title not found in metadata.")
+
         table = Table(
             [
                 x_bin_edges[:-1],
@@ -83,9 +69,53 @@ def fill_hdf5_table(hist, x_bin_edges, y_bin_edges, x_label, y_label, meta_data)
             names=(names, sanitize_name("Values")),
             meta=meta_data,
         )
+    else:
+        if y_label is not None:
+            names = [
+                f"{sanitize_name(y_label).split('__')[0]}_{i}" for i in range(len(y_bin_edges[:-1]))
+            ]
+        try:
+            names = [
+                f"{(meta_data['Title']).split('__')[0]}_{sanitize_name(y_label)}_{i}"
+                for i in range(len(y_bin_edges[:-1]))
+            ]
+        except KeyError:
+            _logger.warning("Title not found in metadata.")
+            names = [
+                f"{sanitize_name(y_label).split('__')[0]}_{i}" for i in range(len(y_bin_edges[:-1]))
+            ]
+
+        table = Table(
+            [hist[i, :] for i in range(len(y_bin_edges[:-1]))],
+            names=names,
+            meta=meta_data,
+        )
+
     return table
 
 
+def validate_histogram(hist, y_bin_edges):
+    """Validate histogram dimensions and y_bin_edges consistency.
+
+    Parameters
+    ----------
+    hist (np.ndarray): The histogram array, expected to be 1D or 2D.
+    y_bin_edges (array-like or None): Bin edges for the second dimension (if applicable).
+
+    Raises
+    ------
+    ValueError: If histogram dimensions are invalid or inconsistent with y_bin_edges.
+    """
+    if hist.ndim not in (1, 2):
+        raise ValueError("Histogram must be either 1D or 2D.")
+
+    if hist.ndim == 1 and y_bin_edges is not None:
+        raise ValueError("y_bin_edges should be None for 1D histograms.")
+
+    if hist.ndim == 2 and y_bin_edges is None:
+        raise ValueError("y_bin_edges should not be None for 2D histograms.")
+
+
 def read_hdf5(hdf5_file_name):
     """
     Read a hdf5 output file.

simtools/layout/array_layout.py

@@ -9,7 +9,7 @@ import numpy as np
 from astropy.table import QTable
 
 import simtools.utils.general as gen
-from simtools.data_model import data_reader
+from simtools.data_model import data_reader, schema
 from simtools.io_operations import io_handler
 from simtools.layout.geo_coordinates import GeoCoordinates
 from simtools.layout.telescope_position import TelescopePosition
@@ -386,7 +386,9 @@ class ArrayLayout:
         with Path(file_name).open("r", encoding="utf-8") as file:
             data = json.load(file)
 
-        position = gen.convert_string_to_list(data["value"])
+        position = data["value"]
+        if isinstance(position, str):
+            position = gen.convert_string_to_list(position)
         self.site = data.get("site", None)
 
         table = QTable()
@@ -579,7 +581,12 @@ class ArrayLayout:
 
         return table
 
-    def export_one_telescope_as_json(self, crs_name):
+    def export_one_telescope_as_json(
+        self,
+        crs_name,
+        parameter_version=None,
+        schema_version=None,
+    ):
         """
         Return a list containing a single telescope in simtools-DB-style json.
 
@@ -587,6 +594,8 @@ class ArrayLayout:
         ----------
         crs_name: str
             Name of coordinate system to be used for export.
+        schema_version: str
+            Version of the schema.
 
         Returns
         -------
@@ -596,43 +605,39 @@ class ArrayLayout:
         table = self.export_telescope_list_table(crs_name)
         if len(table) != 1:
             raise ValueError("Only one telescope can be exported to json")
-        parameter_name = value_string = None
+        parameter_name = value = None
         if crs_name == "ground":
             parameter_name = "array_element_position_ground"
-            value_string = gen.convert_list_to_string(
-                [
-                    table["position_x"][0].value,
-                    table["position_y"][0].value,
-                    table["position_z"][0].value,
-                ]
-            )
+            value = [
+                table["position_x"][0].value,
+                table["position_y"][0].value,
+                table["position_z"][0].value,
+            ]
         elif crs_name == "utm":
             parameter_name = "array_element_position_utm"
-            value_string = gen.convert_list_to_string(
-                [
-                    table["utm_east"][0].value,
-                    table["utm_north"][0].value,
-                    table["altitude"][0].value,
-                ]
-            )
+            value = [
+                table["utm_east"][0].value,
+                table["utm_north"][0].value,
+                table["altitude"][0].value,
+            ]
         elif crs_name == "mercator":
             parameter_name = "array_element_position_mercator"
-            value_string = gen.convert_list_to_string(
-                [
-                    table["latitude"][0].value,
-                    table["longitude"][0].value,
-                    table["altitude"][0].value,
-                ]
-            )
+            value = [
+                table["latitude"][0].value,
+                table["longitude"][0].value,
+                table["altitude"][0].value,
+            ]
+
         return {
+            "schema_version": schema.get_model_parameter_schema_version(schema_version),
             "parameter": parameter_name,
             "instrument": table["telescope_name"][0],
             "site": self.site,
-            "version": self.model_version,
-            "value": value_string,
+            "parameter_version": parameter_version,
+            "unique_id": None,
+            "value": value,
             "unit": "m",
             "type": "float64",
-            "applicable": True,
             "file": False,
         }
 

simtools/model/array_model.py

@@ -6,8 +6,8 @@ from pathlib import Path
 import astropy.units as u
 from astropy.table import QTable
 
-from simtools.data_model import data_reader
-from simtools.db import db_array_elements, db_handler
+from simtools.data_model import data_reader, schema
+from simtools.db import db_handler
 from simtools.io_operations import io_handler
 from simtools.model.site_model import SiteModel
 from simtools.model.telescope_model import TelescopeModel
@@ -290,7 +290,13 @@ class ArrayModel:
         }
 
     def _get_telescope_position_parameter(
-        self, telescope_name: str, site: str, x: u.Quantity, y: u.Quantity, z: u.Quantity
+        self,
+        telescope_name: str,
+        site: str,
+        x: u.Quantity,
+        y: u.Quantity,
+        z: u.Quantity,
+        parameter_version: str | None = None,
     ) -> dict:
         """
         Return dictionary with telescope position parameters (following DB model database format).
@@ -314,16 +320,17 @@ class ArrayModel:
             Dict with telescope position parameters.
         """
         return {
+            "schema_version": schema.get_model_parameter_schema_version(),
             "parameter": "array_element_position_ground",
             "instrument": telescope_name,
             "site": site,
-            "version": self.model_version,
+            "parameter_version": parameter_version,
+            "unique_id": None,
             "value": general.convert_list_to_string(
                 [x.to("m").value, y.to("m").value, z.to("m").value]
             ),
             "unit": "m",
             "type": "float64",
-            "applicable": True,
             "file": False,
         }
 
@@ -367,9 +374,8 @@ class ArrayModel:
         dict
             Dict with array elements.
         """
-        all_elements = db_array_elements.get_array_elements_of_type(
+        all_elements = self.db.get_array_elements_of_type(
             array_element_type=array_element_type,
-            db=self.db,
             model_version=self.model_version,
             collection="telescopes",
         )

simtools/model/model_parameter.py

@@ -67,7 +67,6 @@ class ModelParameter:
 
         self._parameters = {}
         self._simulation_config_parameters = {"corsika": {}, "simtel": {}}
-        self._derived = None
         self.collection = collection
         self.label = label
         self.model_version = model_version
@@ -111,12 +110,8 @@ class ModelParameter:
         """
         try:
             return self._parameters[par_name]
-        except KeyError:
-            pass
-        try:
-            return self.derived[par_name]
         except (KeyError, ValueError) as e:
-            msg = f"Parameter {par_name} was not found in the model"
+            msg = f"Parameter {par_name} was not found in the model {self.name}, {self.site}."
             self._logger.error(msg)
             raise InvalidModelParameterError(msg) from e
 
@@ -180,7 +175,10 @@ class ModelParameter:
         _value = self.get_parameter_value(par_name, _parameter)
 
         try:
-            _unit = [item.strip() for item in _parameter.get("unit").split(",")]
+            if isinstance(_parameter.get("unit"), str):
+                _unit = [item.strip() for item in _parameter.get("unit").split(",")]
+            else:
+                _unit = _parameter.get("unit")
 
             # if there is only one value or the values share one unit
             if (isinstance(_value, (int | float))) or (len(_value) > len(_unit)):
@@ -241,33 +239,6 @@ class ModelParameter:
             self._logger.debug(f"Parameter {par_name} does not have a file associated with it.")
         return False
 
-    @property
-    def derived(self):
-        """Load the derived values and export them if the class instance hasn't done it yet."""
-        if self._derived is None:
-            self._load_derived_values()
-            self._export_derived_files()
-        return self._derived
-
-    def _load_derived_values(self):
-        """Load derived values from the DB."""
-        self._logger.debug("Reading derived values from DB")
-        self._derived = self.db.get_derived_values(
-            self.site,
-            self.name,
-            self.model_version,
-        )
-
-    def _export_derived_files(self):
-        """Write to disk a file from the derived values DB."""
-        for par_now in self.derived.values():
-            if par_now.get("File") or par_now.get("file"):
-                self.db.export_file_db(
-                    db_name=self.db.DB_DERIVED_VALUES,
-                    dest=self.config_file_directory,
-                    file_name=(par_now.get("value") or par_now.get("Value")),
-                )
-
     def print_parameters(self):
         """Print parameters and their values for debugging purposes."""
         for par in self._parameters:
@@ -322,10 +293,11 @@ class ModelParameter:
                         simulation_software=simulation_software,
                     )
                 )
-            except ValueError:
+            except ValueError as exc:
                 self._logger.warning(
                     f"No {simulation_software} parameters found for "
-                    f"{self.site}, {self.name} (model version {self.model_version})."
+                    f"{self.site}, {self.name} (model version {self.model_version}). "
+                    f" (Query {exc})"
                 )
 
     def _load_parameters_from_db(self):
@@ -335,15 +307,18 @@ class ModelParameter:
 
         if self.name is not None:
             self._parameters = self.db.get_model_parameters(
-                self.site, self.name, self.model_version, self.collection, only_applicable=True
+                self.site, self.name, self.collection, self.model_version
             )
 
         if self.site is not None:
-            _site_pars = self.db.get_site_parameters(
-                self.site, self.model_version, only_applicable=True
+            self._parameters.update(
+                self.db.get_model_parameters(
+                    self.site,
+                    None,
+                    "sites",
+                    self.model_version,
+                )
             )
-            self._parameters.update(_site_pars)
-
         self._load_simulation_software_parameter()
 
     def set_extra_label(self, extra_label):
@@ -369,7 +344,7 @@ class ModelParameter:
         """Return the extra label if defined, if not return ''."""
         return self._extra_label if self._extra_label is not None else ""
 
-    def get_simtel_parameters(self, parameters=None, telescope_model=True, site_model=True):
+    def get_simtel_parameters(self, parameters=None):
         """
         Get simtel parameters as name and value pairs.
 
@@ -377,10 +352,6 @@ class ModelParameter:
         ----------
         parameters: dict
             Parameters (simtools) to be renamed (if necessary)
-        telescope_model: bool
-            If True, telescope model parameters are included.
-        site_model: bool
-            If True, site model parameters are included.
 
         Returns
         -------
@@ -394,10 +365,7 @@ class ModelParameter:
         _simtel_parameter_value = {}
         for key in parameters:
             _par_name = names.get_simulation_software_name_from_parameter_name(
-                key,
-                simulation_software="sim_telarray",
-                search_telescope_parameters=telescope_model,
-                search_site_parameters=site_model,
+                key, simulation_software="sim_telarray"
             )
             if _par_name is not None:
                 _simtel_parameter_value[_par_name] = parameters[key].get("value")
@@ -513,7 +481,7 @@ class ModelParameter:
             for par in self._added_parameter_files:
                 pars_from_db.pop(par)
 
-        self.db.export_model_files(pars_from_db, self.config_file_directory)
+        self.db.export_model_files(parameters=pars_from_db, dest=self.config_file_directory)
         self._is_exported_model_files_up_to_date = True
 
     def get_model_file_as_table(self, par_name):
@@ -535,7 +503,7 @@ class ModelParameter:
             _par_entry[par_name] = self._parameters[par_name]
         except KeyError as exc:
             raise ValueError(f"Parameter {par_name} not found in the model.") from exc
-        self.db.export_model_files(_par_entry, self.config_file_directory)
+        self.db.export_model_files(parameters=_par_entry, dest=self.config_file_directory)
         if _par_entry[par_name]["value"].endswith("ecsv"):
             return Table.read(
                 self.config_file_directory.joinpath(_par_entry[par_name]["value"]),
@@ -620,7 +588,7 @@ class ModelParameter:
             Model directory to export the file to.
         """
         self.db.export_model_files(
-            {
+            parameters={
                 "nsb_spectrum_at_2200m": {
                     "value": self._simulation_config_parameters["simtel"][
                         "correct_nsb_spectrum_to_telescope_altitude"
@@ -628,5 +596,5 @@ class ModelParameter:
                     "file": True,
                 }
             },
-            model_directory,
+            dest=model_directory,
         )

simtools/model/site_model.py

@@ -151,11 +151,11 @@ class SiteModel(ModelParameter):
             Model directory to export the file to.
         """
         self.db.export_model_files(
-            {
+            parameters={
                 "atmospheric_transmission_file": {
                     "value": self.get_parameter_value("atmospheric_profile"),
                     "file": True,
                 }
             },
-            model_directory,
+            dest=model_directory,
         )