ctao-calibpipe 0.3.0rc2__py3-none-any.whl

calibpipe/database/interfaces/sql_table_info.py

@@ -0,0 +1,131 @@
+"""SQLTableInfo class."""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from sqlalchemy.orm import declarative_base
+from sqlalchemy.schema import (
+    CheckConstraint,
+    ForeignKeyConstraint,
+    PrimaryKeyConstraint,
+    UniqueConstraint,
+)
+
+from ..interfaces import sql_metadata
+from .sql_column_info import SQLColumnInfo
+
+
+class InvalidTableError(Exception):
+    """Raised when a table is invalid e.g. has no primary key."""
+
+
+class SQLTableInfo:
+    """
+    Collection of attributes defining a Table's columns.
+
+    The class contains the column information (`SQLColumnInfo`)
+    and additional arguments required to build the sqlalchemy
+    table when the `get_table()` method is called.
+
+    This class can provide useful information on the corresponding
+    table. For example the primary-key or the list of undeferred
+    and deferred columns, i.e. that must be loaded directly or
+    looked up in a cache system (if implemented) respectively.
+    Note that no cache implementation lies here, only the information
+    that some columns must be deferred if possible.
+
+    The `SQLTableInfo` also can manage several tables of the same type
+    (e.g. for versioning, table_A_v1 && table_A_v2). When calling
+    the `get_table()` method, a custom table name can be given. The
+    object will ensure that only one table is created for a given
+    name (otherwise `sqlalchemy` cannot work properly).
+    """
+
+    table_base_class = declarative_base()
+
+    def __init__(
+        self,
+        table_name: str,
+        metadata: sql_metadata,
+        columns: list[SQLColumnInfo],
+        constraints: list[
+            ForeignKeyConstraint
+            | UniqueConstraint
+            | CheckConstraint
+            | PrimaryKeyConstraint
+        ]
+        | None = None,
+    ) -> None:
+        """Initialize the table data and sqlachemy metadata."""
+        self.table_name = table_name
+        self.metadata = metadata
+        self.columns = columns
+        self.constraints = constraints if constraints else []
+        self._table_instances: dict[str, sa.Table] = {}
+
+    def get_primary_keys(self) -> list[SQLColumnInfo]:
+        """Get list of primary keys for the table.
+
+        Returns
+        -------
+        list
+            list with SQLColumnInfo objects that are the primary keys
+
+        Raises
+        ------
+        InvalidTableError
+            If there are no primary key in the table
+        """
+        pk_columns = []
+        for column in self.columns:
+            if column.is_primary_key():
+                pk_columns.append(column)
+        if pk_columns:
+            return pk_columns
+        raise InvalidTableError(f"Table {self.table_name!r} has no primary key.")
+
+    def get_deferred_columns(self) -> list[SQLColumnInfo]:
+        """
+        Return the columns that must be deferred.
+
+        Deferred columns won't be loaded directly when queried.
+        """
+        return [column for column in self.columns if column.is_deferred]
+
+    def get_undeferred_columns(self) -> list[SQLColumnInfo]:
+        """Return the columns that must not be deferred.
+
+        These columns are loaded directly when queried.
+        """
+        return [column for column in self.columns if not column.is_deferred]
+
+    def get_table(self, table_name: str | None = None) -> sa.Table:
+        """
+        Return a table with a given name, create it if necessary.
+
+        Parameters
+        ----------
+        table_name: str (optional, default=None)
+            Name of the table to create. If not given, the `table_name`
+            attribute is used. If the table with the given name has
+            already been created it is returned and no new table
+            is generated.
+        """
+        table_name = table_name or self.table_name
+        if table_name not in self._table_instances:
+            if table_name in self.metadata.tables:
+                self._table_instances[table_name] = sa.Table(table_name, self.metadata)
+            else:
+                self._table_instances[table_name] = self._generate_table(
+                    table_name=table_name
+                )
+        return self._table_instances[table_name]
+
+    def _generate_table(self, table_name: str) -> sa.Table:
+        """Generate a table corresponding to the info with a specific name."""
+        return sa.Table(
+            table_name,
+            self.metadata,
+            *[col.generate_column() for col in self.columns],
+            *self.constraints,
+        )

calibpipe/database/interfaces/table_handler.py

@@ -0,0 +1,333 @@
+"""Utilities for CalibPipe data."""
+
+from datetime import datetime, timezone
+from typing import Any
+
+import astropy.units as u
+import numpy as np  #  # noqa: F401
+import sqlalchemy as sa
+from astropy.table import QTable
+from ctapipe.core import Container
+
+from ...core.exceptions import DBStorageError
+from ..adapter.adapter import Adapter
+from ..adapter.database_containers.container_map import ContainerMap
+from ..adapter.database_containers.table_version_manager import TableVersionManager
+from ..connections import CalibPipeDatabase
+from ..interfaces import sql_metadata
+
+
+class TableHandler:
+    """
+    Handles tables in CalibPipe DataBase.
+
+    The first method returns a valid insertion for a DB, made by the table instance
+    and the values to be inserted. The second method just insert values in a DB,
+    provided the DB connection, the table and the values.
+
+    """
+
+    @staticmethod
+    def get_database_table_insertion(
+        container: Container,
+        version: str | None = None,
+    ) -> tuple[sa.Table, dict[str, Any]]:
+        """Return a valid insertion for a DB made by the table instance, and the values to insert."""
+        table, kwargs = Adapter.to_postgres(container, version=version)
+        if table is None:
+            raise TypeError(f"Table cannot be created for {type(container)}.")
+        return table, kwargs
+
+    @staticmethod
+    def insert_row_in_database(
+        table: sa.Table,
+        kwargs: dict[str, Any],
+        connection: CalibPipeDatabase,
+    ) -> None:
+        """Insert values in a DB table as a row."""
+        connection.execute(sa.insert(table).values(**kwargs))
+
+    @staticmethod
+    def read_table_from_database(
+        container: Container,
+        connection: CalibPipeDatabase,
+        condition: str | None = None,
+    ) -> QTable:
+        """
+        Read a table from the DB and return it as a QTable object.
+
+        An optional argument `condition` shall have the following form:
+        `c.<column_name> <operator> <value>`
+        or a combination of thereof using `&` and `|` operators.
+        In case of compound condition, every singleton must be contained in parentheses.
+        """
+        table = ContainerMap.map_to_db_container(container).get_table()
+        if condition:
+            query = table.select().where(
+                eval(condition.replace("c.", "table.c."))  # pylint: disable=eval-used
+            )
+        else:
+            query = table.select()
+        rows = connection.execute(query).fetchall()
+        if not rows:
+            return QTable(
+                names=table.columns.keys(),
+                units=[
+                    1 * u.Unit(c.comment) if c.comment else None for c in table.columns
+                ],
+            )
+        return QTable(
+            rows=rows,
+            names=table.columns.keys(),
+            units=[1 * u.Unit(c.comment) if c.comment else None for c in table.columns],
+        )
+
+    @staticmethod
+    def get_compatible_version(
+        version_table: sa.Table,
+        table_name: str,
+        version: str,
+        connection: CalibPipeDatabase,
+    ) -> str:
+        """
+        Get a compatible version for a certain table from the version table.
+
+        If no compatible version of the table is available, the new version
+        the table will be added to the version table.
+        """
+        version_major = version.split(".")[0]
+        query = sa.select(version_table.c.version).where(
+            version_table.c.version.like(f"{version_major}%"),
+            version_table.c.name == table_name,
+        )
+        query_results = connection.execute(query).first()
+        if query_results is None:
+            vals = {
+                "name": table_name,
+                "version": version,
+                "validity_start": datetime(2023, 1, 1, 0, 0, 1, tzinfo=timezone.utc),
+                "validity_end": datetime(2023, 1, 1, 0, 0, 2, tzinfo=timezone.utc),
+            }
+            TableHandler.insert_row_in_database(
+                version_table,
+                vals,
+                connection=connection,
+            )
+            return version
+        comp_version = query_results[0]
+        return comp_version
+
+    @staticmethod
+    def update_tables_info(
+        table: sa.Table,
+        version_table: sa.Table,
+        table_name: str,
+        comp_version: str,
+        table_version: str,
+        connection: CalibPipeDatabase,
+    ) -> str:
+        """
+        Update the tables' info.
+
+        Updated min and max timestamps are taken from the data table,
+        and a check on version is performed to update the version table.
+        Also, the name of the table is updated accordingly if version has changed.
+        """
+        msg = "DB tables have been updated successfully."
+        query = sa.select(
+            sa.func.min(table.c.validity_start).label("min_time"),
+            sa.func.max(table.c.validity_end).label("max_time"),
+        )
+        results = connection.execute(query).first()
+
+        if float(table_version.split(".")[1]) > float(comp_version.split(".")[1]):
+            TableHandler.update_version_table(
+                version_table,
+                table_name,
+                comp_version,
+                table_version,
+                results.min_time,
+                results.max_time,
+                connection,
+            )
+            TableHandler.update_table_name(table, table_version, connection)
+            return (
+                msg
+                + f" Version has been updated from v{comp_version} to v{table_version}."
+            )
+        TableHandler.update_version_table(
+            version_table,
+            table_name,
+            comp_version,
+            comp_version,
+            results.min_time,
+            results.max_time,
+            connection,
+        )
+        return msg
+
+    @staticmethod
+    def update_version_table(
+        version_table: sa.Table,
+        table_name: str,
+        old_version: str,
+        new_version: str,
+        min_time: datetime,
+        max_time: datetime,
+        connection: CalibPipeDatabase,
+    ) -> None:
+        """Update the version of a table with the new version in the version table of the DB."""
+        stmt = (
+            sa.update(version_table)
+            .where(
+                version_table.c.name == table_name,
+                version_table.c.version == old_version,
+            )
+            .values(version=new_version, validity_start=min_time, validity_end=max_time)
+        )
+        connection.execute(stmt)
+
+    @staticmethod
+    def update_table_name(
+        table: sa.Table,
+        version: str,
+        connection: CalibPipeDatabase,
+    ) -> None:
+        """Update the name of a table with the new version."""
+        new_table_name = TableVersionManager.update_version(table.name, version)
+        stmt = sa.text(f"ALTER TABLE {table} RENAME TO {new_table_name};")
+        connection.execute(stmt)
+
+    @staticmethod
+    def prepare_db_tables(containers, db_config):
+        """
+        Create and upload to the CalibPipe DB empty tables for selected calibration containers.
+
+        Parameters
+        ----------
+        containers : list[Container]
+            list of calibpipe containers or ContainerMeta instances
+            that will be created as empty tables in the DB
+
+        config_data : dict
+            Calibpipe configuration with database connection configuration
+        """
+        try:
+            with CalibPipeDatabase(**db_config) as connection:
+                sql_metadata.reflect(bind=connection.engine, extend_existing=True)
+
+                # Create empty main data tables
+                for cp_container in containers:
+                    if isinstance(cp_container, Container):
+                        db_container = ContainerMap.map_to_db_container(
+                            type(cp_container)
+                        )
+                    else:
+                        db_container = ContainerMap.map_to_db_container(cp_container)
+                    if not sa.inspect(connection.engine).has_table(
+                        db_container.table_name
+                    ):
+                        db_container.get_table()
+                sql_metadata.create_all(bind=connection.engine)
+        except sa.exc.DatabaseError:
+            raise DBStorageError("Issues with connection to the CalibPipe DB")
+
+    @staticmethod
+    def upload_data(
+        calibpipe_data_container: Container,
+        metadata: list[Container] | None,
+        connection: CalibPipeDatabase,
+    ) -> None:
+        """
+        Upload data and optional metadata to the database.
+
+        This method uploads the provided data to the main database table and,
+        if provided, associates the metadata with the inserted data row.
+
+        Parameters
+        ----------
+        calibpipe_data_container : ctapipe.core.Container
+            The data container with the data to be uploaded to the main table of the database.
+
+        metadata : list[Container] or None
+            Optional list of metadata containers to be uploaded. Should include
+            a "ReferenceMetadataContainer" if metadata is provided.
+
+        connection : CalibPipeDatabase
+            An active database connection to the CalibPipe database.
+
+        Raises
+        ------
+        DBStorageError
+            If there are issues with the database connection.
+
+        ValueError
+            If the main table does not contain a single autoincremented primary key
+            or if ReferenceMetadataContainer is missing when metadata is provided.
+        """
+        data_db_container = ContainerMap.map_to_db_container(
+            type(calibpipe_data_container)
+        )
+        has_autoincrement_pk = any(
+            col.autoincrement for col in data_db_container.get_table().c
+        )
+        is_single_pk = len(data_db_container.get_primary_keys()) == 1
+
+        if not (has_autoincrement_pk and is_single_pk):
+            raise ValueError(
+                f"Table '{data_db_container.table_name}' "
+                "doesn't contain a single autoincremented primary key."
+            )
+
+        pk_name = data_db_container.get_primary_keys()[0].name
+
+        try:
+            # Insert main data
+            table, values = TableHandler.get_database_table_insertion(
+                calibpipe_data_container
+            )
+            TableHandler.insert_row_in_database(table, values, connection)
+
+            # No metadata to upload
+            if not metadata:
+                return
+
+            # Get the last inserted primary key
+            stmt = sa.select(table).order_by(sa.desc(table.c[pk_name])).limit(1)
+            last_db_record = connection.execute(stmt).fetchone()
+            data_pk_value = last_db_record._asdict()[pk_name]
+
+            # Handle ReferenceMetadataContainer
+            reference_meta_container = next(
+                (
+                    c
+                    for c in metadata
+                    if c.__class__.__name__ == "ReferenceMetadataContainer"
+                ),
+                None,
+            )
+            if reference_meta_container is None:
+                raise ValueError("ReferenceMetadataContainer is required in metadata.")
+
+            reference_meta_container.ID_optical_throughput = data_pk_value
+            ref_table, ref_values = TableHandler.get_database_table_insertion(
+                reference_meta_container
+            )
+            TableHandler.insert_row_in_database(ref_table, ref_values, connection)
+
+            # Get ReferenceMetadata ID to link to other metadata
+            stmt = sa.select(ref_table).order_by(sa.desc(ref_table.c.ID)).limit(1)
+            metadata_id = connection.execute(stmt).fetchone().ID
+
+            # Upload other metadata
+            for container in metadata:
+                if container.__class__.__name__ == "ReferenceMetadataContainer":
+                    continue
+                container.ID = metadata_id
+                meta_table, meta_values = TableHandler.get_database_table_insertion(
+                    container
+                )
+                TableHandler.insert_row_in_database(meta_table, meta_values, connection)
+
+        except sa.exc.DatabaseError:
+            raise DBStorageError("Issues with connection to the CalibPipe DB")

calibpipe/database/interfaces/types.py

@@ -0,0 +1,96 @@
+"""
+Type definitions for SQLAlchemy.
+
+These type definitions allow us to define database fields and
+containers being almost completely decoupled from SQLAlchemy
+(without direct coupling).
+
+In particular, SQLColumnInfo and SQLTableInfo use these generic
+types and not the sqlalchemy types directly.
+
+The NDArray type is defined explicitly to implemented the
+serialization/deserialization np.ndarray <-> bytes and the
+(optional) zlib compression/decompression on the byte data.
+
+"""
+
+import pickle
+import zlib
+
+import numpy as np
+import sqlalchemy as sa
+import sqlalchemy.sql.sqltypes
+from sqlalchemy.dialects.postgresql import ARRAY, DOUBLE_PRECISION
+
+ColumnType = sqlalchemy.sql.sqltypes.TypeEngine
+
+Boolean: ColumnType = sa.Boolean
+
+SmallInteger: ColumnType = sa.SmallInteger
+Integer: ColumnType = sa.Integer
+BigInteger: ColumnType = sa.BigInteger
+Float: ColumnType = sa.Float
+Double: ColumnType = DOUBLE_PRECISION
+Numeric: ColumnType = sa.Numeric
+Binary: ColumnType = sa.types.LargeBinary
+String: ColumnType = sa.String
+
+ArrayF1D: ColumnType = ARRAY(Float, dimensions=1)
+ArrayF2D: ColumnType = ARRAY(Float, dimensions=2)
+ArrayF3D: ColumnType = ARRAY(Float, dimensions=3)
+
+Date: ColumnType = sa.Date
+Time: ColumnType = sa.Time
+DateTime: ColumnType = sa.DateTime
+
+
+class NDArray(sa.types.TypeDecorator):  # pylint: disable=too-many-ancestors
+    """
+    Type for numpy.ndarray binding, include data compression.
+
+    The array is stored as a compressed byte string in the database.
+    The class implements the binding between the `np.ndarray` in the
+    program memory and the byte string stored in the DB.
+
+    Compression can be removed or modified, but the two process methods
+    should be the opposite of each other for the binding to work.
+    Ignoring the dialect parameter that is anyway not used, this means
+    that the following assertion should always pass::
+
+        db_arr: NDArray
+        arr: np.ndarray
+        arr_bytes: bytes = db_arr.process_bind_param(arr)
+        recov_arr: np.ndarray = db_arr.process_result_value(arr_bytes)
+        assert(arr == recov_arr)
+
+    """
+
+    impl = sa.types.LargeBinary  # Byte storage in the DB
+    cache_ok: bool = True  # Results of process methods can be cached
+
+    def process_bind_param(self, value: np.ndarray, dialect) -> bytes:
+        """
+        Serialize a np.ndarray into a byte object to store in the DB.
+
+        The array is first serialized into bytes and compressed using
+        the default zlib compression algorithm.
+        """
+        return zlib.compress(pickle.dumps(value))
+
+    def process_result_value(self, value: bytes, dialect) -> np.ndarray:
+        """
+        Deserialize a np.ndarray from bytes read in the DB.
+
+        The bytes are first decompressed and the array is loaded from
+        the decompressed byte string.
+        """
+        return pickle.loads(zlib.decompress(value))
+
+    def process_literal_param(self, value: np.ndarray, dialect) -> str:
+        """Representation of the NDArray object."""
+        return f"NDArray(shape={value.shape}, dtype={value.dtype})"
+
+    @property
+    def python_type(self) -> type:
+        """Return the python type of the underlying object represented by the byte string."""
+        return np.ndarray

calibpipe/telescope/throughput/containers.py

@@ -0,0 +1,66 @@
+"""Containers to keep optical throughput data and metadata."""
+
+import numpy as np
+from astropy.time import Time
+from ctapipe.core import Container, Field
+
+NAN_TIME = Time(0, format="mjd", scale="tai")
+
+
+class OpticalThoughtputContainer(Container):
+    """Optical throughput calibration coefficient and analysis results for a single telescope."""
+
+    mean = Field(
+        np.nan,
+        "Mean optical throughput from the selected calibration method",
+        type=np.float64,
+        allow_none=False,
+    )
+    median = Field(
+        np.nan,
+        "Optical throughput from the selected calibration method",
+        type=np.float64,
+        allow_none=False,
+    )
+    std = Field(
+        np.nan,
+        "Optical throughput from the selected calibration method",
+        type=np.float64,
+        allow_none=False,
+    )
+    sem = Field(
+        np.nan,
+        "Standard error of the mean optical throughput from the selected calibration method",
+        type=np.float64,
+        allow_none=False,
+    )
+    method = Field(
+        "None",
+        "Calibration method used",
+        type=str,
+        allow_none=False,
+    )
+    time_start = Field(
+        NAN_TIME,
+        description="Starting timestamp of validity for the selected throughput.",
+        type=Time,
+        allow_none=False,
+    )
+    time_end = Field(
+        NAN_TIME,
+        description="Ending timestamp of validity for the selected throughput.",
+        type=Time,
+        allow_none=False,
+    )
+    obs_id = Field(
+        -1,
+        description="ID of the observation block for validity",
+        type=np.int32,
+        allow_none=False,
+    )
+    n_events = Field(
+        0,
+        description="Number of muon rings used to calculate the throughput",
+        type=np.int64,
+        allow_none=False,
+    )