ctao-calibpipe 0.1.0__py3-none-any.whl

calibpipe/database/connections/postgres_utils.py

@@ -0,0 +1,97 @@
+"""Adapter for psycopg types and database uri."""
+
+from typing import Any
+
+import numpy as np
+from psycopg import adapters
+from psycopg.adapt import Buffer, Dumper
+from psycopg.errors import DataError
+from psycopg.postgres import types as _types
+from psycopg.types.bool import BoolDumper
+from psycopg.types.numeric import Float4Dumper, FloatDumper
+
+
+def get_postgres_uri(
+    user: str,
+    database: str,
+    passwd: str,
+    host: str = "postgres",
+    port: str | None = None,
+) -> str:
+    """Generate a valid uri to connect to the postgres+psycopg database."""
+    port_str = f":{port}" if port else ""
+    return f"postgresql+psycopg://{user}:{passwd}@{host}{port_str}/{database}"
+
+
+# np.int dumpers
+class _NPIntDumper(Dumper):
+    def dump(self, obj: Any) -> Buffer:
+        t = type(obj)
+        allowed_types = [
+            np.int8,
+            np.int16,
+            np.int32,
+            np.int64,
+            np.longlong,
+            np.uint8,
+            np.uint16,
+            np.uint32,
+            np.uint64,
+            np.ulonglong,
+        ]
+        if t not in allowed_types:
+            raise DataError(f"Numpy integer expected, got {type(obj).__name__!r}")
+        return str(obj).encode()
+
+
+class NPInt16Dumper(_NPIntDumper):
+    """Numpy int16 dumper."""
+
+    oid = _types["int2"].oid
+
+
+class NPInt32Dumper(_NPIntDumper):
+    """Numpy int32 dumper."""
+
+    oid = _types["int4"].oid
+
+
+class NPInt64Dumper(_NPIntDumper):
+    """Numpy int64 dumper."""
+
+    oid = _types["int8"].oid
+
+
+class NPNumericDumper(_NPIntDumper):
+    """Numpy numeric dumper."""
+
+    oid = _types["numeric"].oid
+
+
+def adapt_psycopg() -> None:
+    # pylint: disable=line-too-long
+    """
+    Adapt numpy numerical types for psycopg3.
+
+    .. note::
+        Required for psycopg3 < 3.2. Until the pyscopg-3.2.0 is released, we borrow their code.
+        See `this PR <https://github.com/psycopg/psycopg/pull/332/files#diff-6d04f11a711cbef8ea32bd1479af4a79b402e213559d8b66359a6b871c5bdd28>`_ for details
+    """
+    # pylint: enable=line-too-long
+    adapters.register_dumper("numpy.int8", NPInt16Dumper)
+    adapters.register_dumper("numpy.int16", NPInt16Dumper)
+    adapters.register_dumper("numpy.int32", NPInt32Dumper)
+    adapters.register_dumper("numpy.int64", NPInt64Dumper)
+    adapters.register_dumper("numpy.longlong", NPInt64Dumper)
+    adapters.register_dumper("numpy.bool_", BoolDumper)
+    adapters.register_dumper("numpy.uint8", NPInt16Dumper)
+    adapters.register_dumper("numpy.uint16", NPInt32Dumper)
+    adapters.register_dumper("numpy.uint32", NPInt64Dumper)
+    adapters.register_dumper("numpy.uint64", NPNumericDumper)
+    adapters.register_dumper("numpy.ulonglong", NPNumericDumper)
+    adapters.register_dumper("numpy.float16", Float4Dumper)
+    adapters.register_dumper("numpy.float32", Float4Dumper)
+    adapters.register_dumper("numpy.float64", FloatDumper)
+
+
+adapt_psycopg()

calibpipe/database/connections/sql_connection.py

@@ -0,0 +1,103 @@
+"""Interface to connect to the calibration DB stored in a SQL DB."""
+
+import sqlalchemy as sa
+from sqlalchemy.engine import Engine, Result
+from sqlalchemy.orm import Session
+
+
+class SQLConnection:
+    """
+    Interface to communicate with a SQL database.
+
+    Once an uri (`str`) has been generated, the connection can be
+    open in a context to ensure proper closing (and commit if
+    required)::
+
+        uri: str = get_postgres_uri(user='api-owner', database='calibration')
+        with SQLConnection(uri=uri, autocommit=True) as connection:
+            # e.g.
+            # connection.execute(...)
+
+    Attributes
+    ----------
+    autocommit: bool
+        Tell if the database changes must be committed automatically
+        when closing the connection (can be done manually by calling
+        the :meth:`commit` method).
+
+    uri: str
+        Uri used to connect to the database. This attribute is not
+        used anymore once the connection is open.
+
+    engine: sqlalchemy.engine.Engine
+        Engine used for the database connection. It can be of several
+        kinds, the default one is `postgres + psycopg`. The engine
+        is automatically connected at the initialization step.
+
+    session: sqlalchemy.orm.Session
+        Session (use the :attr:`engine`) used to execute
+        queries to the database.
+    """
+
+    def __init__(self, uri: str, autocommit: bool = False) -> None:
+        """
+        Initialize the session and engine, connect to the database.
+
+        Parameters
+        ----------
+        uri: str
+            uri to connect to the database. See the
+            :func:`calibpipe.database.connections.get_postgres_uri`
+            to generate the uri connecting to a Postgres database.
+
+        autocommit: bool, optional (default=False)
+            Determines if the connection commits changes when the
+            :meth:`__exit__`
+            method is called. If set to `False` (default), changes will not be
+            committed and it is necessary to call :meth:`commit` after
+            modifications have been done.
+        """
+        self.autocommit = autocommit
+        self.uri: str = uri
+        self.engine: Engine = sa.create_engine(self.uri, echo=True, future=True)
+        self.engine.connect()
+        self.session: Session = Session(self.engine)
+
+    def __enter__(self) -> "SQLConnection":
+        """Enter a new context."""
+        return self
+
+    def __exit__(self, *args) -> None:
+        """
+        Exit the context and close the connection.
+
+        This method simply call `close()`.
+        """
+        self.close()
+
+    def close(self) -> None:
+        """
+        Close the session.
+
+        If the autocommit attribute is True, changes are committed before
+        closing the connection.
+        """
+        if self.session:
+            if self.autocommit:
+                self.commit()
+            self.session.close()
+
+    def commit(self) -> None:
+        """Commit changes to the database."""
+        self.session.commit()
+
+    def execute(self, *args) -> Result:
+        """
+        Execute a query in the SQL session.
+
+        This methods forwards the arguments to the
+        :meth:`sqlalchemy.orm.Session.execute` method of
+        :attr:`session`.
+        Refer to the documentation of `sqlalchemy` to use queries.
+        """
+        return self.session.execute(*args)

calibpipe/database/connections/user_confirmation.py

@@ -0,0 +1,19 @@
+"""Function to ask for a user confirmation."""
+
+
+def get_user_confirmation(prompt: str) -> bool:
+    """Ask a confirmation from the user by displaying a prompt and asking yes or no.
+
+    Parameters
+    ----------
+    prompt: str
+        Prompt to display
+
+    Returns
+    -------
+    bool
+        Answer from the user
+    """
+    print(f"{prompt} [y/n, default: no]")
+    user_input = input()
+    return user_input.lower() == "y"

calibpipe/database/interfaces/__init__.py

@@ -0,0 +1,71 @@
+"""
+Various interfaces for database access.
+
+This module contains field, column, row and table interfaces
+to interact with a database. In general the module is tightly
+coupled to sqlalchemy though some parts have been preserved
+from an explicit dependency.
+
+Module content:
+  - `types.py`: Type definition for DB fields using sqlalchemy
+    types and defining the specific case of the `numpy.ndarray`.
+  - `sql_metadata.py`: Simple metadata variable definition for
+    sqlalchemy.
+  - `sql_column_info.py`: Container for column information, useful
+    to e.g. define the corresponding LST fields at the DB level.
+  - `sql_table_info.py`: Container for table information, used in
+    particular to create sqlalchemy table objects from a list
+    of column information.
+  - `hashable_row_data.py`: Hashable container of a (single) row
+    information (table_name + primary_key value). This can be
+    (and is) used to index data using the row to which they belong
+    and create a cache for data retrieved from the database.
+  - `table_handler.py`: Container for functions to handle
+    tables in the DB.
+  - `queries.py`: Built-in queries that can be used to retrieve camera calibration data.
+
+"""
+
+from .hashable_row_data import HashableRowData
+from .queries import query_from_date, query_from_run, query_full_table
+from .sql_column_info import SQLColumnInfo
+from .sql_metadata import sql_metadata
+from .sql_table_info import SQLTableInfo
+from .table_handler import TableHandler
+from .types import (
+    BigInteger,
+    Boolean,
+    Date,
+    DateTime,
+    Double,
+    Float,
+    Integer,
+    NDArray,
+    Numeric,
+    SmallInteger,
+    String,
+    Time,
+)
+
+__all__ = [
+    "Boolean",
+    "SmallInteger",
+    "Integer",
+    "BigInteger",
+    "Float",
+    "Double",
+    "Numeric",
+    "String",
+    "Date",
+    "Time",
+    "DateTime",
+    "NDArray",
+    "sql_metadata",
+    "SQLTableInfo",
+    "SQLColumnInfo",
+    "HashableRowData",
+    "TableHandler",
+    "query_full_table",
+    "query_from_date",
+    "query_from_run",
+]

calibpipe/database/interfaces/hashable_row_data.py

@@ -0,0 +1,54 @@
+"""HashableRowData class."""
+
+from collections.abc import Hashable
+
+
+class HashableRowData:
+    """
+    Contain hashable row information (table and primary key).
+
+    A table name and a primary key **value** is enough information
+    to uniquely identify a row inside the entire database. This
+    information can therefore be hashed to create a map indexed
+    by a row.
+
+    Attributes
+    ----------
+    table_name: str
+        Name of the table in which the row is stored.
+
+    primary_key: Hashable
+        Any python object that can be hashed, must contain the primary
+        key value of the row.
+    """
+
+    table_name: str
+    primary_key: Hashable  # Any hashable value
+
+    def __init__(self, table_name: str, primary_key: Hashable) -> None:
+        """Initialize hashable table."""
+        self.table_name = table_name
+        self.primary_key = primary_key
+
+    def __eq__(self, object_: object) -> bool:
+        """Compare two HashableRowData objects."""
+        if not isinstance(object_, HashableRowData):
+            return False
+        return (
+            self.table_name == object_.table_name
+            and self.primary_key == object_.primary_key
+        )
+
+    def __hash__(self) -> int:
+        """
+        Hash function.
+
+        The xor (operator ^) seems ok because the table name and primary_key
+        will in general be different, or at least it should be extremely
+        rare and performance should not be affected.
+        """
+        return hash(self.table_name) ^ hash(self.primary_key)
+
+    def __str__(self) -> str:
+        """Generate a string representation for HashableRowData object. Unicity is guaranteed."""
+        return f"{self.primary_key}_{self.table_name}"

calibpipe/database/interfaces/queries.py

@@ -0,0 +1,180 @@
+"""
+Built-in queries for camera calibration data.
+
+All built-in queries are obtained by calling functions that
+return a tuple of query and a list of strings, e.g.::
+
+    light_query, deferred_column_names = some_builtin_query()
+
+Each time, the query object can be directly sent to an `execute()`
+SQLConnection method and it will retrieve from the DB the primary
+key and all light fields (by default non-array types, see the
+deferred property of `SQLColumnInfo`). This can be done e.g. using::
+
+    db.execute(light_query)
+
+The list of strings returned in the
+tuple is the list of deferred fields, that must be queried
+separately later on, e.g. using::
+
+    query = sa.select(deferred_column_names)
+    res = db.execute(query)
+
+The built-in queries are:
+
+ - `query_full_table()`: Return the query utilities
+   to retrieve a full table from the database
+   (used e.g. to retrieve the run metadata table).
+ - `query_from_run()`: Return the query utilities
+   to retrieve data of a given run in a given table.
+ - `query_from_date()`: Return the query utilities
+   to retrieve data at a given date in a given table.
+
+The queries have an undefined type (internal to sqlalchemy and
+not fully defined), an alias to `Any` is used in this file to express
+what objects are SQL queries.
+"""
+
+import datetime
+from typing import Any
+
+import sqlalchemy as sa
+
+from ..adapter.database_containers.table_version_manager import (
+    TableVersionManager,
+)
+from .sql_table_info import SQLTableInfo
+
+Query = Any
+""" Alias to express which objects are queries as `sqlalchemy` does not define it. """
+
+
+def _get_deferred_column(table: sa.Table, info: SQLTableInfo) -> list[str]:
+    """Return the names of deferred columns to use in a select statement."""
+    return [getattr(table.c, column.name) for column in info.get_deferred_columns()]
+
+
+def _get_undeferred_column(table: sa.Table, info: SQLTableInfo) -> list[str]:
+    """Return the names of undeferred columns to use in a select statement."""
+    return [getattr(table.c, column.name) for column in info.get_undeferred_columns()]
+
+
+def _process_table_info(
+    table_info: SQLTableInfo, version: str
+) -> tuple[sa.Table, list[str], list[str]]:
+    """Return objects necessary to build queries from a table info."""
+    table = TableVersionManager.apply_version(table_info=table_info, version=version)
+    deferred_columns = _get_deferred_column(table=table, info=table_info)
+    undeferred_columns = _get_undeferred_column(table=table, info=table_info)
+    return table, deferred_columns, undeferred_columns
+
+
+def _select_and_filter(column: list[Any], condition: Any) -> Query | None:
+    """Return a select clause on several columns using a simple filter."""
+    if not column:
+        return None
+    return sa.select(*column).filter(condition)
+
+
+def _select_by_date(
+    table: sa.Table, column: list[str], date: datetime.date
+) -> Query | None:
+    """Return a query selecting a list of columns from a date."""
+    return _select_and_filter(
+        column=column,
+        condition=(table.c.date == date),  # pylint: disable=superfluous-parens
+    )
+
+
+def _select_by_run(table: sa.Table, column: list[str], run: int) -> Query | None:
+    """Return a query selecting a list of columns from a run."""
+    return _select_and_filter(
+        column=column,
+        condition=(table.c.run == run),  # pylint: disable=superfluous-parens
+    )
+
+
+def query_full_table(
+    table_info: SQLTableInfo, version: str | None = None
+) -> tuple[Query, list[str]]:
+    """
+    Return a query for a complete table.
+
+    Parameters
+    ----------
+    table_info: SQLTableInfo
+        Table to which the query must be built.
+
+    version: Optional[str], default=None
+        Software version of the data to retrieve. If `None` is given, the
+        `_pro` version will be used i.e. the latest available.
+
+    Returns
+    -------
+    tuple[Query, list[str]]
+        A tuple containing the light query to retrieve small fields and the list
+        of field names for deferred fields (cached and loaded later).
+    """
+    _unused_table, deferred_columns, undeferred_columns = _process_table_info(
+        table_info, version=version
+    )
+    light_query = sa.select(undeferred_columns)
+    return light_query, deferred_columns
+
+
+def query_from_date(
+    table_info: SQLTableInfo, version: str, date: datetime.date
+) -> tuple[Query, list[str]]:
+    """
+    Return a query from a date.
+
+    Parameters
+    ----------
+    table_info: SQLTableInfo
+        Table to which the query must be built.
+
+    version: Optional[str], default=None
+        Software version of the data to retrieve. If `None` is given, the
+        `_pro` version will be used i.e. the latest available.
+
+    Returns
+    -------
+    tuple[Query, list[str]]
+        A tuple containing the light query to retrieve small fields and the list
+        of field names for deferred fields (cached and loaded later).
+    """
+    table, deferred_columns, undeferred_columns = _process_table_info(
+        table_info, version=version
+    )
+    light_query = _select_by_date(table, undeferred_columns, date)
+
+    return light_query, deferred_columns
+
+
+def query_from_run(
+    table_info: SQLTableInfo, version: str, run: int
+) -> tuple[Query, list[str]]:
+    """
+    Return a query from a run.
+
+    Parameters
+    ----------
+    table_info: SQLTableInfo
+        Table to which the query must be built.
+
+    version: Optional[str], default=None
+        Software version of the data to retrieve. If `None` is given, the
+        `_pro` version will be used i.e. the latest available.
+
+    Returns
+    -------
+    tuple[Query, list[str]]
+        A tuple containing the light query to retrieve small fields and the list
+        of field names for deferred fields (cached and loaded later).
+    """
+    table, deferred_columns, undeferred_columns = _process_table_info(
+        table_info, version=version
+    )
+    light_query = _select_by_run(table, undeferred_columns, run)
+
+    return light_query, deferred_columns

calibpipe/database/interfaces/sql_column_info.py

@@ -0,0 +1,67 @@
+"""SQLColumnInfo class."""
+
+import astropy.units as u
+import sqlalchemy as sa
+from astropy.units.cds import ppm
+
+from .types import ColumnType, NDArray
+
+u.add_enabled_units([ppm])
+
+
+class SQLColumnInfo:
+    """
+    Contain info required to create a `sa.Column` object.
+
+    The data representing the column is system-independent,
+    using in particular the generic types in `.types`,
+    only the `generate_column()` method is specialized for
+    `sqlalchemy` (returning a `sa.Column` object).
+
+    Attributes
+    ----------
+    name: str
+        Field name
+
+    field_type: ColumnType
+        Field `type.` See the `.types` import for possible types
+
+    is_deferred: bool (optional, default=None)
+        If given, tell if the field must be deferred i.e. loaded
+        only later (when queried) if a cache system is in place.
+        If not given, only `NDArray` objects are deferred.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        field_type: ColumnType,
+        unit: str | None = "",
+        is_deferred: bool | None = None,
+        **kwargs,
+    ) -> None:
+        """
+        Initialize the column data.
+
+        Any keyword argument required to build the final Column object
+        (here sa.Column for sqlachemy) in the generate_column() method
+        can be given to the initializer.
+        """
+        self.name = name
+        self.type = field_type
+        self.unit = u.Unit(unit)
+        if is_deferred is not None:
+            self.is_deferred = is_deferred
+        else:
+            # If not specified defer automatically arrays and arrays only
+            self.is_deferred = field_type == NDArray
+        self.kwargs = {**kwargs}
+
+    def generate_column(self) -> sa.Column:
+        """Generate a new corresponding sa.Column object."""
+        column = sa.Column(self.name, self.type, comment=str(self.unit), **self.kwargs)
+        return column
+
+    def is_primary_key(self) -> bool:
+        """Check if the column is a primary key."""
+        return "primary_key" in self.kwargs and self.kwargs["primary_key"]

calibpipe/database/interfaces/sql_metadata.py

@@ -0,0 +1,6 @@
+"""SQL meta-data variable."""
+
+import sqlalchemy as sa
+
+sql_metadata = sa.MetaData()
+""" Metadata variable used for `sqlalchemy` objects. """