gammasimtools 0.8.2__py3-none-any.whl → 0.10.0__py3-none-any.whl

simtools/layout/array_layout.py

@@ -9,7 +9,7 @@ import numpy as np
 from astropy.table import QTable
 
 import simtools.utils.general as gen
-from simtools.data_model import data_reader
+from simtools.data_model import data_reader, schema
 from simtools.io_operations import io_handler
 from simtools.layout.geo_coordinates import GeoCoordinates
 from simtools.layout.telescope_position import TelescopePosition
@@ -386,7 +386,9 @@ class ArrayLayout:
         with Path(file_name).open("r", encoding="utf-8") as file:
             data = json.load(file)
 
-        position = gen.convert_string_to_list(data["value"])
+        position = data["value"]
+        if isinstance(position, str):
+            position = gen.convert_string_to_list(position)
         self.site = data.get("site", None)
 
         table = QTable()
@@ -579,7 +581,12 @@ class ArrayLayout:
 
         return table
 
-    def export_one_telescope_as_json(self, crs_name):
+    def export_one_telescope_as_json(
+        self,
+        crs_name,
+        parameter_version=None,
+        schema_version=None,
+    ):
         """
         Return a list containing a single telescope in simtools-DB-style json.
 
@@ -587,6 +594,8 @@ class ArrayLayout:
         ----------
         crs_name: str
             Name of coordinate system to be used for export.
+        schema_version: str
+            Version of the schema.
 
         Returns
         -------
@@ -596,43 +605,39 @@ class ArrayLayout:
         table = self.export_telescope_list_table(crs_name)
         if len(table) != 1:
             raise ValueError("Only one telescope can be exported to json")
-        parameter_name = value_string = None
+        parameter_name = value = None
         if crs_name == "ground":
             parameter_name = "array_element_position_ground"
-            value_string = gen.convert_list_to_string(
-                [
-                    table["position_x"][0].value,
-                    table["position_y"][0].value,
-                    table["position_z"][0].value,
-                ]
-            )
+            value = [
+                table["position_x"][0].value,
+                table["position_y"][0].value,
+                table["position_z"][0].value,
+            ]
         elif crs_name == "utm":
             parameter_name = "array_element_position_utm"
-            value_string = gen.convert_list_to_string(
-                [
-                    table["utm_east"][0].value,
-                    table["utm_north"][0].value,
-                    table["altitude"][0].value,
-                ]
-            )
+            value = [
+                table["utm_east"][0].value,
+                table["utm_north"][0].value,
+                table["altitude"][0].value,
+            ]
         elif crs_name == "mercator":
             parameter_name = "array_element_position_mercator"
-            value_string = gen.convert_list_to_string(
-                [
-                    table["latitude"][0].value,
-                    table["longitude"][0].value,
-                    table["altitude"][0].value,
-                ]
-            )
+            value = [
+                table["latitude"][0].value,
+                table["longitude"][0].value,
+                table["altitude"][0].value,
+            ]
+
         return {
+            "schema_version": schema.get_model_parameter_schema_version(schema_version),
             "parameter": parameter_name,
             "instrument": table["telescope_name"][0],
             "site": self.site,
-            "version": self.model_version,
-            "value": value_string,
+            "parameter_version": parameter_version,
+            "unique_id": None,
+            "value": value,
             "unit": "m",
             "type": "float64",
-            "applicable": True,
             "file": False,
         }
 

simtools/model/array_model.py

@@ -6,8 +6,8 @@ from pathlib import Path
 import astropy.units as u
 from astropy.table import QTable
 
-from simtools.data_model import data_reader
-from simtools.db import db_array_elements, db_handler
+from simtools.data_model import data_reader, schema
+from simtools.db import db_handler
 from simtools.io_operations import io_handler
 from simtools.model.site_model import SiteModel
 from simtools.model.telescope_model import TelescopeModel
@@ -290,7 +290,13 @@ class ArrayModel:
         }
 
     def _get_telescope_position_parameter(
-        self, telescope_name: str, site: str, x: u.Quantity, y: u.Quantity, z: u.Quantity
+        self,
+        telescope_name: str,
+        site: str,
+        x: u.Quantity,
+        y: u.Quantity,
+        z: u.Quantity,
+        parameter_version: str | None = None,
     ) -> dict:
         """
         Return dictionary with telescope position parameters (following DB model database format).
@@ -314,16 +320,17 @@ class ArrayModel:
             Dict with telescope position parameters.
         """
         return {
+            "schema_version": schema.get_model_parameter_schema_version(),
             "parameter": "array_element_position_ground",
             "instrument": telescope_name,
             "site": site,
-            "version": self.model_version,
+            "parameter_version": parameter_version,
+            "unique_id": None,
             "value": general.convert_list_to_string(
                 [x.to("m").value, y.to("m").value, z.to("m").value]
             ),
             "unit": "m",
             "type": "float64",
-            "applicable": True,
             "file": False,
         }
 
@@ -367,9 +374,8 @@ class ArrayModel:
         dict
             Dict with array elements.
         """
-        all_elements = db_array_elements.get_array_elements_of_type(
+        all_elements = self.db.get_array_elements_of_type(
             array_element_type=array_element_type,
-            db=self.db,
             model_version=self.model_version,
             collection="telescopes",
         )

simtools/model/camera.py

@@ -5,8 +5,6 @@ from pathlib import Path
 
 import astropy.units as u
 import numpy as np
-from scipy.spatial import cKDTree as KDTree
-from scipy.spatial import distance
 
 from simtools.utils.geometry import rotate
 
@@ -304,6 +302,8 @@ class Camera:
         float
             The camera fill factor.
         """
+        from scipy.spatial import distance  # pylint: disable=import-outside-toplevel
+
         if self.pixels["pixel_spacing"] == 9999:
             points = np.array([self.pixels["x"], self.pixels["y"]]).T
             pixel_distances = distance.cdist(points, points, "euclidean")
@@ -403,6 +403,8 @@ class Camera:
         list of lists
             Array of neighbor indices in a list for each pixel
         """
+        from scipy.spatial import cKDTree as KDTree  # pylint: disable=import-outside-toplevel
+
         tree = KDTree(np.column_stack([x_pos, y_pos]))
         neighbors = tree.query_ball_tree(tree, radius)
         return [list(np.setdiff1d(neigh, [i])) for i, neigh in enumerate(neighbors)]

simtools/model/model_parameter.py

@@ -6,10 +6,12 @@ import shutil
 from copy import copy
 
 import astropy.units as u
+from astropy.table import Table
 
 import simtools.utils.general as gen
 from simtools.db import db_handler
 from simtools.io_operations import io_handler
+from simtools.simtel import simtel_table_reader
 from simtools.simtel.simtel_config_writer import SimtelConfigWriter
 from simtools.utils import names
 
@@ -65,7 +67,6 @@ class ModelParameter:
 
         self._parameters = {}
         self._simulation_config_parameters = {"corsika": {}, "simtel": {}}
-        self._derived = None
         self.collection = collection
         self.label = label
         self.model_version = model_version
@@ -109,12 +110,8 @@ class ModelParameter:
         """
         try:
             return self._parameters[par_name]
-        except KeyError:
-            pass
-        try:
-            return self.derived[par_name]
         except (KeyError, ValueError) as e:
-            msg = f"Parameter {par_name} was not found in the model"
+            msg = f"Parameter {par_name} was not found in the model {self.name}, {self.site}."
             self._logger.error(msg)
             raise InvalidModelParameterError(msg) from e
 
@@ -178,7 +175,10 @@ class ModelParameter:
         _value = self.get_parameter_value(par_name, _parameter)
 
         try:
-            _unit = [item.strip() for item in _parameter.get("unit").split(",")]
+            if isinstance(_parameter.get("unit"), str):
+                _unit = [item.strip() for item in _parameter.get("unit").split(",")]
+            else:
+                _unit = _parameter.get("unit")
 
             # if there is only one value or the values share one unit
             if (isinstance(_value, (int | float))) or (len(_value) > len(_unit)):
@@ -239,33 +239,6 @@ class ModelParameter:
             self._logger.debug(f"Parameter {par_name} does not have a file associated with it.")
         return False
 
-    @property
-    def derived(self):
-        """Load the derived values and export them if the class instance hasn't done it yet."""
-        if self._derived is None:
-            self._load_derived_values()
-            self._export_derived_files()
-        return self._derived
-
-    def _load_derived_values(self):
-        """Load derived values from the DB."""
-        self._logger.debug("Reading derived values from DB")
-        self._derived = self.db.get_derived_values(
-            self.site,
-            self.name,
-            self.model_version,
-        )
-
-    def _export_derived_files(self):
-        """Write to disk a file from the derived values DB."""
-        for par_now in self.derived.values():
-            if par_now.get("File") or par_now.get("file"):
-                self.db.export_file_db(
-                    db_name=self.db.DB_DERIVED_VALUES,
-                    dest=self.config_file_directory,
-                    file_name=(par_now.get("value") or par_now.get("Value")),
-                )
-
     def print_parameters(self):
         """Print parameters and their values for debugging purposes."""
         for par in self._parameters:
@@ -273,7 +246,7 @@ class ModelParameter:
 
     def _set_config_file_directory_and_name(self):
         """Set and create the directory and the name of the config file."""
-        if self.name is None:
+        if self.name is None and self.site is None:
             return
 
         self._config_file_directory = self.io_handler.get_output_directory(
@@ -281,15 +254,14 @@ class ModelParameter:
         )
 
         # Setting file name and the location
-        if self.site is not None and self.name is not None:
-            config_file_name = names.simtel_config_file_name(
-                self.site,
-                self.model_version,
-                telescope_model_name=self.name,
-                label=self.label,
-                extra_label=self._extra_label,
-            )
-            self._config_file_path = self.config_file_directory.joinpath(config_file_name)
+        config_file_name = names.simtel_config_file_name(
+            self.site,
+            self.model_version,
+            telescope_model_name=self.name,
+            label=self.label,
+            extra_label=self._extra_label,
+        )
+        self._config_file_path = self.config_file_directory.joinpath(config_file_name)
 
         self._logger.debug(f"Config file path: {self._config_file_path}")
 
@@ -321,10 +293,11 @@ class ModelParameter:
                         simulation_software=simulation_software,
                     )
                 )
-            except ValueError:
+            except ValueError as exc:
                 self._logger.warning(
                     f"No {simulation_software} parameters found for "
-                    f"{self.site}, {self.name} (model version {self.model_version})."
+                    f"{self.site}, {self.name} (model version {self.model_version}). "
+                    f" (Query {exc})"
                 )
 
     def _load_parameters_from_db(self):
@@ -334,15 +307,18 @@ class ModelParameter:
 
         if self.name is not None:
             self._parameters = self.db.get_model_parameters(
-                self.site, self.name, self.model_version, self.collection, only_applicable=True
+                self.site, self.name, self.collection, self.model_version
             )
 
         if self.site is not None:
-            _site_pars = self.db.get_site_parameters(
-                self.site, self.model_version, only_applicable=True
+            self._parameters.update(
+                self.db.get_model_parameters(
+                    self.site,
+                    None,
+                    "sites",
+                    self.model_version,
+                )
             )
-            self._parameters.update(_site_pars)
-
         self._load_simulation_software_parameter()
 
     def set_extra_label(self, extra_label):
@@ -368,7 +344,7 @@ class ModelParameter:
         """Return the extra label if defined, if not return ''."""
         return self._extra_label if self._extra_label is not None else ""
 
-    def get_simtel_parameters(self, parameters=None, telescope_model=True, site_model=True):
+    def get_simtel_parameters(self, parameters=None):
         """
         Get simtel parameters as name and value pairs.
 
@@ -376,10 +352,6 @@ class ModelParameter:
         ----------
         parameters: dict
             Parameters (simtools) to be renamed (if necessary)
-        telescope_model: bool
-            If True, telescope model parameters are included.
-        site_model: bool
-            If True, site model parameters are included.
 
         Returns
         -------
@@ -393,10 +365,7 @@ class ModelParameter:
         _simtel_parameter_value = {}
         for key in parameters:
             _par_name = names.get_simulation_software_name_from_parameter_name(
-                key,
-                simulation_software="sim_telarray",
-                search_telescope_parameters=telescope_model,
-                search_site_parameters=site_model,
+                key, simulation_software="sim_telarray"
             )
             if _par_name is not None:
                 _simtel_parameter_value[_par_name] = parameters[key].get("value")
@@ -512,9 +481,38 @@ class ModelParameter:
             for par in self._added_parameter_files:
                 pars_from_db.pop(par)
 
-        self.db.export_model_files(pars_from_db, self.config_file_directory)
+        self.db.export_model_files(parameters=pars_from_db, dest=self.config_file_directory)
         self._is_exported_model_files_up_to_date = True
 
+    def get_model_file_as_table(self, par_name):
+        """
+        Return tabular data from file as astropy table.
+
+        Parameters
+        ----------
+        par_name: str
+            Name of the parameter.
+
+        Returns
+        -------
+        Table
+            Astropy table.
+        """
+        _par_entry = {}
+        try:
+            _par_entry[par_name] = self._parameters[par_name]
+        except KeyError as exc:
+            raise ValueError(f"Parameter {par_name} not found in the model.") from exc
+        self.db.export_model_files(parameters=_par_entry, dest=self.config_file_directory)
+        if _par_entry[par_name]["value"].endswith("ecsv"):
+            return Table.read(
+                self.config_file_directory.joinpath(_par_entry[par_name]["value"]),
+                format="ascii.ecsv",
+            )
+        return simtel_table_reader.read_simtel_table(
+            par_name, self.config_file_directory.joinpath(_par_entry[par_name]["value"])
+        )
+
     def export_config_file(self):
         """Export the config file used by sim_telarray."""
         # Exporting model file
@@ -590,7 +588,7 @@ class ModelParameter:
             Model directory to export the file to.
         """
         self.db.export_model_files(
-            {
+            parameters={
                 "nsb_spectrum_at_2200m": {
                     "value": self._simulation_config_parameters["simtel"][
                         "correct_nsb_spectrum_to_telescope_altitude"
@@ -598,5 +596,5 @@ class ModelParameter:
                     "file": True,
                 }
             },
-            model_directory,
+            dest=model_directory,
         )

simtools/model/site_model.py

@@ -80,7 +80,7 @@ class SiteModel(ModelParameter):
             Site-related CORSIKA parameters as dict
         """
         if config_file_style:
-            model_directory = model_directory or Path("")
+            model_directory = model_directory or Path()
             return {
                 "OBSLEV": [
                     self.get_parameter_value_with_unit("corsika_observation_level").to_value("cm")
@@ -151,11 +151,11 @@ class SiteModel(ModelParameter):
             Model directory to export the file to.
         """
         self.db.export_model_files(
-            {
+            parameters={
                 "atmospheric_transmission_file": {
                     "value": self.get_parameter_value("atmospheric_profile"),
                     "file": True,
                 }
             },
-            model_directory,
+            dest=model_directory,
         )