gammasimtools 0.5.1__py3-none-any.whl → 0.6.1__py3-none-any.whl

simtools/data_model/validate_data.py

@@ -30,7 +30,7 @@ class DataValidator:
 
     """
 
-    def __init__(self, schema_file=None, data_file=None):
+    def __init__(self, schema_file=None, data_file=None, data_table=None):
         """
         Initialize validation class and read required reference data columns
 
@@ -38,11 +38,11 @@ class DataValidator:
 
         self._logger = logging.getLogger(__name__)
 
-        self._data_file_name = data_file
-        self._schema_file_name = schema_file
+        self.data_file_name = data_file
+        self.schema_file_name = schema_file
         self._reference_data_columns = None
         self.data = None
-        self.data_table = None
+        self.data_table = data_table
 
     def validate_and_transform(self):
         """
@@ -53,13 +53,22 @@ class DataValidator:
         data: dict or astropy.table
             Data dict or table
 
+        Raises
+        ------
+        TypeError
+            if no data or data table is available
+
         """
 
-        self.validate_data_file()
+        if self.data_file_name:
+            self.validate_data_file()
         if isinstance(self.data, dict):
             self._validate_data_dict()
-        else:
+        elif isinstance(self.data_table, Table):
             self._validate_data_table()
+        else:
+            self._logger.error("No data or data table to validate")
+            raise TypeError
 
         return self.data_table
 
@@ -72,12 +81,12 @@ class DataValidator:
         """
 
         try:
-            if Path(self._data_file_name).suffix in (".yml", ".yaml"):
-                self.data = gen.collect_data_from_yaml_or_dict(self._data_file_name, None)
-                self._logger.info(f"Reading data from yaml file: {self._data_file_name}")
+            if Path(self.data_file_name).suffix in (".yml", ".yaml"):
+                self.data = gen.collect_data_from_file_or_dict(self.data_file_name, None)
+                self._logger.info(f"Validating data from: {self.data_file_name}")
             else:
-                self.data_table = Table.read(self._data_file_name, guess=True, delimiter=r"\s")
-                self._logger.info(f"Reading tabled data from file: {self._data_file_name}")
+                self.data_table = Table.read(self.data_file_name, guess=True, delimiter=r"\s")
+                self._logger.info(f"Validating tabled data from: {self.data_file_name}")
         except (AttributeError, TypeError):
             pass
 
@@ -90,7 +99,7 @@ class DataValidator:
 
         try:
             self._reference_data_columns = self._read_validation_schema(
-                self._schema_file_name, self.data["name"]
+                self.schema_file_name, self.data["name"]
             )
             _quantities = []
             for value, unit in zip(self.data["value"], self.data["units"]):
@@ -113,11 +122,11 @@ class DataValidator:
         """
 
         try:
-            self._reference_data_columns = self._read_validation_schema(self._schema_file_name)[
+            self._reference_data_columns = self._read_validation_schema(self.schema_file_name)[
                 0
             ].get("table_columns", None)
         except IndexError:
-            self._logger.error(f"Error reading validation schema from {self._schema_file_name}")
+            self._logger.error(f"Error reading validation schema from {self.schema_file_name}")
             raise
 
         if self._reference_data_columns is not None:
@@ -129,8 +138,8 @@ class DataValidator:
         """
         Validate that
         - required data columns are available
-        -  columns are in the correct units (if necessary apply a unit conversion)
-        -  ranges (minimum, maximum) are correct.
+        - columns are in the correct units (if necessary apply a unit conversion)
+        - ranges (minimum, maximum) are correct.
 
         This is not applied to columns of type 'string'.
 
@@ -138,15 +147,17 @@ class DataValidator:
 
         self._check_required_columns()
 
-        for col in self.data_table.itercols():
-            if not self._get_reference_data_column(col.name, status_test=True):
+        for col_name in self.data_table.colnames:
+            col = self.data_table[col_name]
+            if not self._get_reference_data_column(col_name, status_test=True):
                 continue
             if not np.issubdtype(col.dtype, np.number):
                 continue
-            self._check_for_not_a_number(col)
-            self._check_and_convert_units(col)
-            self._check_range(col.name, np.nanmin(col.data), np.nanmax(col.data), "allowed_range")
-            self._check_range(col.name, np.nanmin(col.data), np.nanmax(col.data), "required_range")
+            self._check_for_not_a_number(col, col_name)
+            self._check_data_type(col, col_name)
+            col = self._check_and_convert_units(col, col_name)
+            self._check_range(col_name, np.nanmin(col.data), np.nanmax(col.data), "allowed_range")
+            self._check_range(col_name, np.nanmin(col.data), np.nanmax(col.data), "required_range")
 
     def _check_required_columns(self):
         """
@@ -160,7 +171,7 @@ class DataValidator:
         """
 
         for entry in self._reference_data_columns:
-            if entry.get("required_column", False):
+            if entry.get("required", False):
                 if entry["name"] in self.data_table.columns:
                     self._logger.debug(f"Found required data column {entry['name']}")
                 else:
@@ -231,7 +242,7 @@ class DataValidator:
             else:
                 self._logger.error(
                     "Failed removal of duplication for column "
-                    f"{_column_with_unique_requirement}, values are not unqiue"
+                    f"{_column_with_unique_requirement}, values are not unique"
                 )
                 raise ValueError
 
@@ -283,14 +294,45 @@ class DataValidator:
 
         return u.Unit(reference_unit)
 
-    def _check_for_not_a_number(self, col):
+    def _check_data_type(self, col, column_name):
+        """
+        Check column data type.
+
+        Parameters
+        ----------
+        col: astropy.column or Quantity
+            data column to be converted
+        column_name: str
+            column name
+
+        Raises
+        ------
+        TypeError
+            if data type is not correct
+
+        """
+
+        reference_dtype = self._get_reference_data_column(column_name).get("type", None)
+
+        if not np.issubdtype(col.dtype, reference_dtype):
+            self._logger.error(
+                f"Invalid data type in column '{column_name}'. "
+                f"Expected type '{reference_dtype}', found '{col.dtype}'"
+            )
+            raise TypeError
+
+        self._logger.debug(f"Data column '{column_name}' has correct data type")
+
+    def _check_for_not_a_number(self, col, col_name):
         """
         Check that column values are finite and not NaN.
 
         Parameters
         ----------
-        col: astropy.column
+        col: astropy.column or Quantity
             data column to be converted
+        col_name: str
+            column name
 
         Returns
         -------
@@ -305,11 +347,11 @@ class DataValidator:
         """
 
         if np.isnan(col.data).any():
-            self._logger.info(f"Column {col.name} contains NaN.")
+            self._logger.info(f"Column {col_name} contains NaN.")
         if np.isinf(col.data).any():
-            self._logger.info(f"Column {col.name} contains infinite value.")
+            self._logger.info(f"Column {col_name} contains infinite value.")
 
-        entry = self._get_reference_data_column(col.name)
+        entry = self._get_reference_data_column(col_name)
         if "allow_nan" in entry.get("input_processing", {}):
             return np.isnan(col.data).any() or np.isinf(col.data).any()
 
@@ -319,7 +361,7 @@ class DataValidator:
 
         return False
 
-    def _check_and_convert_units(self, col):
+    def _check_and_convert_units(self, col, col_name):
         """
         Check that all columns have an allowed unit. Convert to reference unit (e.g., Angstrom to
         nm).
@@ -332,8 +374,10 @@ class DataValidator:
 
         Parameters
         ----------
-        col: astropy.column
+        col: astropy.column or Quantity
             data column to be converted
+        col_name: str
+            column name
 
         Returns
         -------
@@ -347,30 +391,26 @@ class DataValidator:
 
         """
 
-        self._logger.debug(f"Checking data column '{col.name}'")
+        self._logger.debug(f"Checking data column '{col_name}'")
 
         try:
-            reference_unit = self._get_reference_unit(col.name)
+            reference_unit = self._get_reference_unit(col_name)
             if col.unit is None or col.unit == "dimensionless":
                 col.unit = u.dimensionless_unscaled
                 return col
 
             self._logger.debug(
-                f"Data column '{col.name}' with reference unit "
+                f"Data column '{col_name}' with reference unit "
                 f"'{reference_unit}' and data unit '{col.unit}'"
             )
-
-            col.convert_unit_to(reference_unit)
-
+            return u.Quantity(col).to(reference_unit)
         except u.core.UnitConversionError:
             self._logger.error(
-                f"Invalid unit in data column '{col.name}'. "
+                f"Invalid unit in data column '{col_name}'. "
                 f"Expected type '{reference_unit}', found '{col.unit}'"
             )
             raise
 
-        return col
-
     def _check_range(self, col_name, col_min, col_max, range_type="allowed_range"):
         """
         Check that column data is within allowed range or required range. Assumes that column and
@@ -434,7 +474,7 @@ class DataValidator:
     def _interval_check(data, axis_range, range_type):
         """
         Check that values are inside allowed range (range_type='allowed_range') or span at least
-         the given inveral (range_type='required_range').
+         the given interval (range_type='required_range').
 
         Parameters
         ----------
@@ -480,15 +520,20 @@ class DataValidator:
         dict
            validation schema
 
+        Raises
+        ------
+        KeyError
+            if 'data' can not be read from dict in schema file
+
         """
 
         try:
             if Path(schema_file).is_dir():
-                return gen.collect_dict_from_file(
-                    file_path=schema_file,
-                    file_name=parameter + ".schema.yml",
+                return gen.collect_data_from_file_or_dict(
+                    file_name=Path(schema_file) / (parameter + ".schema.yml"),
+                    in_dict=None,
                 )["data"]
-            return gen.collect_dict_from_file(schema_file)["data"]
+            return gen.collect_data_from_file_or_dict(file_name=schema_file, in_dict=None)["data"]
         except KeyError:
             self._logger.error(f"Error reading validation schema from {schema_file}")
             raise
@@ -502,11 +547,16 @@ class DataValidator:
         ----------
         column_name: str
             Column name.
+        status_test: bool
+            Test if reference column exists.
 
         Returns
         -------
         dict
-            Reference schema column.
+            Reference schema column (for status_test==False).
+        bool
+            True if reference column exists (for status_test==True).
+
 
         Raises
         ------

simtools/db_handler.py

@@ -12,8 +12,7 @@ from pymongo import MongoClient
 from pymongo.errors import BulkWriteError
 
 import simtools.utils.general as gen
-from simtools import io_handler
-from simtools.model.model_utils import get_telescope_class
+from simtools.io_operations import io_handler
 from simtools.utils import names
 
 __all__ = ["DatabaseHandler"]
@@ -46,7 +45,7 @@ class DatabaseHandler:
     DB_REFERENCE_DATA = "CTA-Reference-Data"
     DB_DERIVED_VALUES = "CTA-Simulation-Model-Derived-Values"
 
-    ALLOWED_FILE_EXTENSIONS = [".dat", ".txt", ".lis", ".cfg", ".yml", ".ecsv"]
+    ALLOWED_FILE_EXTENSIONS = [".dat", ".txt", ".lis", ".cfg", ".yml", ".yaml", ".ecsv"]
 
     db_client = None
 
@@ -69,32 +68,14 @@ class DatabaseHandler:
 
         self._set_up_connection()
 
-    def set_mongo_db_config(self, mongo_db_config):
-        """
-        Set the MongoDB config and open the connection to the DB.
-
-        Parameters
-        ----------
-        mongo_db_config: dict
-            Dictionary with the MongoDB configuration.
-
-        Raises
-        ------
-        KeyError
-            if there is non-valid key in the db_config.
-
-        """
-
-        self.mongo_db_config = mongo_db_config
-        self._set_up_connection()
-
     def _set_up_connection(self):
         """
         Open the connection to MongoDB.
         """
         if self.mongo_db_config:
             if DatabaseHandler.db_client is None:
-                with Lock():
+                lock = Lock()
+                with lock:
                     DatabaseHandler.db_client = self._open_mongo_db()
 
     def _open_mongo_db(self):
@@ -237,6 +218,9 @@ class DatabaseHandler:
             for info in parameters.values():
                 if not info["File"]:
                     continue
+                if Path(dest).joinpath(info["Value"]).exists():
+                    self._logger.debug(f"File {info['Value']} already exists in {dest}")
+                    continue
                 file = self._get_file_mongo_db(
                     DatabaseHandler.DB_CTA_SIMULATION_MODEL, info["Value"]
                 )
@@ -279,7 +263,7 @@ class DatabaseHandler:
 
             raise
 
-        dest_file.write_text(file.read_text())
+        dest_file.write_text(file.read_text(encoding="utf-8"), encoding="utf-8")
 
     def _get_model_parameters_yaml(
         self, telescope_model_name, model_version, only_applicable=False
@@ -302,8 +286,8 @@ class DatabaseHandler:
 
         """
 
-        _tel_class = get_telescope_class(telescope_model_name)
-        _tel_name_converted = names.convert_telescope_model_name_to_yaml(telescope_model_name)
+        _tel_class = names.get_telescope_class(telescope_model_name)
+        _tel_name_converted = names.convert_telescope_model_name_to_yaml_name(telescope_model_name)
 
         if _tel_class == "MST":
             # MST-FlashCam or MST-NectarCam
@@ -361,7 +345,7 @@ class DatabaseHandler:
 
         _site_validated = names.validate_site_name(site)
         _tel_name_db = self._get_telescope_model_name_for_db(_site_validated, telescope_model_name)
-        _tel_class = get_telescope_class(telescope_model_name)
+        _tel_class = names.get_telescope_class(telescope_model_name)
 
         self._logger.debug(f"Tel_name_db: {_tel_name_db}")
         self._logger.debug(f"Tel_class: {_tel_class}")
@@ -881,7 +865,7 @@ class DatabaseHandler:
         try:
             collection.insert_many(db_entries)
         except BulkWriteError as exc:
-            raise BulkWriteError.details from exc
+            raise BulkWriteError(str(exc.details)) from exc
 
     def copy_documents(self, db_name, collection, query, db_to_copy_to, collection_to_copy_to=None):
         """
@@ -931,7 +915,7 @@ class DatabaseHandler:
         try:
             _collection.insert_many(db_entries)
         except BulkWriteError as exc:
-            raise BulkWriteError.details from exc
+            raise BulkWriteError(str(exc.details)) from exc
 
     def delete_query(self, db_name, collection, query):
         """
@@ -1281,8 +1265,12 @@ class DatabaseHandler:
 
         db_entry["Version"] = version
         db_entry["Parameter"] = parameter
-        db_entry["Value"] = value
-        db_entry["Type"] = kwargs["Type"] if "Type" in kwargs else str(type(value))
+
+        _base_value, _base_unit, _base_type = gen.get_value_unit_type(value)
+        db_entry["Value"] = _base_value
+        if _base_unit is not None:
+            db_entry["units"] = _base_unit
+        db_entry["Type"] = kwargs["Type"] if "Type" in kwargs else _base_type
 
         files_to_add_to_db = set()
         db_entry["File"] = False
@@ -1308,24 +1296,24 @@ class DatabaseHandler:
 
     def _convert_version_to_tagged(self, model_version, db_name):
         """Convert to tagged version, if needed."""
-        if model_version in ["Current", "Latest"]:
+        if model_version in ["Released", "Latest"]:
             return self._get_tagged_version(db_name, model_version)
 
         return model_version
 
     @staticmethod
-    def _get_tagged_version(db_name, version="Current"):
+    def _get_tagged_version(db_name, version="Released"):
         """
-        Get the tag of the "Current" or "Latest" version of the MC Model.
-        The "Current" is the latest stable MC Model,
-        the latest is the latest tag (not necessarily stable, but can be equivalent to "Current").
+        Get the tag of the "Released" or "Latest" version of the MC Model.
+        The "Released" is the latest stable MC Model,
+        the latest is the latest tag (not necessarily stable, but can be equivalent to "Released").
 
         Parameters
         ----------
         db_name: str
             the name of the DB
         version: str
-            Can be "Current" or "Latest" (default: "Current").
+            Can be "Released" or "Latest" (default: "Released").
 
         Returns
         -------
@@ -1335,12 +1323,12 @@ class DatabaseHandler:
         Raises
         ------
         ValueError
-            if version not valid. Valid versions are: 'Current' and 'Latest'.
+            if version not valid. Valid versions are: 'Released' and 'Latest'.
 
         """
 
-        if version not in ["Current", "Latest"]:
-            raise ValueError('The only default versions are "Current" or "Latest"')
+        if version not in ["Released", "Latest"]:
+            raise ValueError('The only default versions are "Released" or "Latest"')
 
         collection = DatabaseHandler.db_client[db_name].metadata
         query = {"Entry": "Simulation-Model-Tags"}
@@ -1385,9 +1373,9 @@ class DatabaseHandler:
             self._logger.warning(
                 f"The file {kwargs['filename']} exists in the DB. Returning its ID"
             )
-            return file_system.find_one(
+            return file_system.find_one(  # pylint: disable=protected-access
                 {"filename": kwargs["filename"]}
-            )._id  # pylint: disable=protected-access;
+            )._id
         with open(file_name, "rb") as data_file:
             return file_system.put(data_file, **kwargs)
 
@@ -1453,3 +1441,40 @@ class DatabaseHandler:
             self._logger.warning(f"The query {query} did not return any results. No versions found")
 
         return _all_versions
+
+    def get_all_available_telescopes(
+        self,
+        db_name=DB_CTA_SIMULATION_MODEL,
+        model_version="Released",
+    ):
+        """
+        Get all available telescope names in the collection "telescopes" in the DB.
+
+        Parameters
+        ----------
+        db_name: str
+            the name of the DB
+        model_version: str
+            Which version to get the telescopes of (default: "Released").
+
+        Returns
+        -------
+        all_available_telescopes: list
+            List of all telescope names found
+
+        """
+
+        collection = DatabaseHandler.db_client[db_name]["telescopes"]
+
+        _model_version = self._convert_version_to_tagged(
+            names.validate_model_version_name(model_version),
+            DatabaseHandler.DB_CTA_SIMULATION_MODEL,
+        )
+
+        query = {
+            "Version": _model_version,
+        }
+
+        _all_available_telescopes = collection.find(query).distinct("Telescope")
+
+        return _all_available_telescopes

simtools/io_operations/hdf5_handler.py

@@ -0,0 +1,112 @@
+import logging
+from pathlib import PosixPath
+
+import astropy.units as u
+import tables
+from astropy.table import Table
+from ctapipe.io import read_table
+
+from simtools.utils.names import sanitize_name
+
+__all__ = [
+    "fill_hdf5_table",
+    "read_hdf5",
+]
+
+_logger = logging.getLogger(__name__)
+
+
+def fill_hdf5_table(hist, x_bin_edges, y_bin_edges, x_label, y_label, meta_data):
+    """
+    Create and fill an hdf5 table with the histogram information.
+    It works for both 1D and 2D distributions.
+
+    Parameters
+    ----------
+    hist: numpy.ndarray
+        The counts of the histograms.
+    x_bin_edges: numpy.array
+        The x bin edges of the histograms.
+    y_bin_edges: numpy.array
+        The y bin edges of the histograms.
+        Use None for 1D histograms.
+    x_label: str
+        X bin edges label.
+    y_label: str
+        Y bin edges label.
+        Use None for 1D histograms.
+    meta_data: dict
+        Dictionary with the histogram metadata.
+    """
+
+    # Complement metadata
+    if x_label is not None:
+        meta_data["x_bin_edges"] = sanitize_name(x_label)
+    meta_data["x_bin_edges_unit"] = (
+        x_bin_edges.unit if isinstance(x_bin_edges, u.Quantity) else u.dimensionless_unscaled
+    )
+
+    if y_bin_edges is not None:
+        if y_label is not None:
+            meta_data["y_bin_edges"] = sanitize_name(y_label)
+            names = [
+                f"{meta_data['y_bin_edges'].split('__')[0]}_{i}"
+                for i in range(len(y_bin_edges[:-1]))
+            ]
+        else:
+            names = [
+                f"{meta_data['Title'].split('__')[0]}_{i}" for i in range(len(y_bin_edges[:-1]))
+            ]
+        meta_data["y_bin_edges_unit"] = (
+            y_bin_edges.unit if isinstance(y_bin_edges, u.Quantity) else u.dimensionless_unscaled
+        )
+
+        table = Table(
+            [hist[i, :] for i in range(len(y_bin_edges[:-1]))],
+            names=names,
+            meta=meta_data,
+        )
+
+    else:
+        if x_label is not None:
+            meta_data["x_bin_edges"] = sanitize_name(x_label)
+            names = meta_data["x_bin_edges"]
+        else:
+            names = meta_data["Title"]
+        table = Table(
+            [
+                x_bin_edges[:-1],
+                hist,
+            ],
+            names=(names, sanitize_name("Values")),
+            meta=meta_data,
+        )
+    return table
+
+
+def read_hdf5(hdf5_file_name):
+    """
+    Read a hdf5 output file.
+
+    Parameters
+    ----------
+    hdf5_file_name: str or Path
+        Name or Path of the hdf5 file to read from.
+
+    Returns
+    -------
+    list
+        The list with the astropy.Table instances for the various 1D and 2D histograms saved
+        in the hdf5 file.
+    """
+    if isinstance(hdf5_file_name, PosixPath):
+        hdf5_file_name = hdf5_file_name.absolute().as_posix()
+
+    tables_list = []
+
+    with tables.open_file(hdf5_file_name, mode="r") as file:
+        for node in file.walk_nodes("/", "Table"):
+            table_path = node._v_pathname  # pylint: disable=protected-access
+            table = read_table(hdf5_file_name, table_path)
+            tables_list.append(table)
+    return tables_list