ctao-calibpipe 0.3.0rc2__py3-none-any.whl

calibpipe/database/adapter/database_containers/atmosphere.py

@@ -0,0 +1,199 @@
+"""Atmospheric SQL table info."""
+
+from sqlalchemy.schema import ForeignKeyConstraint
+
+from ....atmosphere.atmosphere_containers import (
+    AtmosphericModelContainer,
+    MacobacContainer,
+    MolecularAtmosphericProfileContainer,
+    MolecularAtmosphericProfileMetaContainer,
+    MolecularDensityContainer,
+    RayleighExtinctionContainer,
+    SelectedAtmosphericModelContainer,
+)
+from ...interfaces.sql_column_info import SQLColumnInfo
+from ...interfaces.sql_metadata import sql_metadata
+from ...interfaces.sql_table_info import SQLTableInfo
+from ...interfaces.types import (
+    ArrayF1D,
+    ArrayF2D,
+    Boolean,
+    Date,
+    Float,
+    Integer,
+    String,
+)
+from .container_map import ContainerMap
+
+atmospheric_model_sql_info = SQLTableInfo(
+    table_name="AtmosphericModel",
+    metadata=sql_metadata,
+    columns=[
+        SQLColumnInfo("start", Date),
+        SQLColumnInfo("stop", Date),
+        SQLColumnInfo("version", String, primary_key=True),
+        SQLColumnInfo("current", Boolean),
+        SQLColumnInfo("season", String),
+        SQLColumnInfo("name_Observatory", String, nullable=False),
+        SQLColumnInfo("version_Observatory", Integer, nullable=False),
+    ],
+    constraints=[
+        ForeignKeyConstraint(
+            ["name_Observatory", "version_Observatory"],
+            ["Observatory.name", "Observatory.version"],
+        )
+    ],
+)
+
+mdp_sql_info = SQLTableInfo(
+    table_name="MolecularDensity",
+    metadata=sql_metadata,
+    columns=[
+        SQLColumnInfo("season", String),
+        SQLColumnInfo("density", Float, unit="1/cm^3"),
+        SQLColumnInfo("version", String, primary_key=True),
+    ],
+    constraints=[
+        ForeignKeyConstraint(
+            [
+                "version",
+            ],
+            [
+                "AtmosphericModel.version",
+            ],
+        )
+    ],
+)
+
+map_meta_sql_info = SQLTableInfo(
+    table_name="MolecularAtmosphericProfileMeta",
+    metadata=sql_metadata,
+    columns=[
+        SQLColumnInfo("data_assimilation_system", String),
+        SQLColumnInfo("dataset", String),
+        SQLColumnInfo("description", String),
+        SQLColumnInfo("version", String, primary_key=True),
+    ],
+    constraints=[
+        ForeignKeyConstraint(
+            [
+                "version",
+            ],
+            [
+                "AtmosphericModel.version",
+            ],
+        )
+    ],
+)
+
+map_sql_info = SQLTableInfo(
+    table_name="MolecularAtmosphericProfile",
+    metadata=sql_metadata,
+    columns=[
+        SQLColumnInfo("altitude", ArrayF1D, unit="km"),
+        SQLColumnInfo("pressure", ArrayF1D, unit="hPa"),
+        SQLColumnInfo("temperature", ArrayF1D, unit="K"),
+        SQLColumnInfo("partial_water_pressure", ArrayF1D),
+        SQLColumnInfo("refractive_index_m_1", ArrayF1D),
+        SQLColumnInfo("atmospheric_density", ArrayF1D, unit="g/cm^3"),
+        SQLColumnInfo("atmospheric_thickness", ArrayF1D, unit="g/cm^2"),
+        SQLColumnInfo("version", String, primary_key=True),
+    ],
+    constraints=[
+        ForeignKeyConstraint(
+            [
+                "version",
+            ],
+            [
+                "AtmosphericModel.version",
+            ],
+        )
+    ],
+)
+
+macobac_sql_info = SQLTableInfo(
+    table_name="MACOBAC",
+    metadata=sql_metadata,
+    columns=[
+        SQLColumnInfo("co2_concentration", Float, unit="ppm"),
+        SQLColumnInfo("estimation_date", Date),
+        SQLColumnInfo("version", String, primary_key=True),
+    ],
+    constraints=[
+        ForeignKeyConstraint(
+            [
+                "version",
+            ],
+            [
+                "AtmosphericModel.version",
+            ],
+        )
+    ],
+)
+
+rayleigh_extinction_sql_info = SQLTableInfo(
+    table_name="RayleighExtinction",
+    metadata=sql_metadata,
+    columns=[
+        SQLColumnInfo("wavelength", ArrayF1D, unit="nm"),
+        SQLColumnInfo("altitude", ArrayF2D, unit="km"),
+        SQLColumnInfo("AOD", ArrayF2D),
+        SQLColumnInfo("version", String, primary_key=True),
+    ],
+    constraints=[
+        ForeignKeyConstraint(
+            [
+                "version",
+            ],
+            [
+                "AtmosphericModel.version",
+            ],
+        )
+    ],
+)
+
+selected_model_sql_info = SQLTableInfo(
+    table_name="SelectedAtmosphericModel",
+    metadata=sql_metadata,
+    columns=[
+        SQLColumnInfo("date", Date),
+        SQLColumnInfo("provenance", String),
+        SQLColumnInfo("season", String),
+        SQLColumnInfo("site", String),
+        SQLColumnInfo("version", String),
+    ],
+)
+
+ContainerMap.register_container_pair(
+    cp_container=AtmosphericModelContainer,
+    db_container=atmospheric_model_sql_info,
+)
+
+ContainerMap.register_container_pair(
+    cp_container=MolecularAtmosphericProfileMetaContainer,
+    db_container=map_meta_sql_info,
+)
+
+ContainerMap.register_container_pair(
+    cp_container=MolecularAtmosphericProfileContainer,
+    db_container=map_sql_info,
+)
+
+ContainerMap.register_container_pair(
+    cp_container=MolecularDensityContainer,
+    db_container=mdp_sql_info,
+)
+
+ContainerMap.register_container_pair(
+    cp_container=MacobacContainer,
+    db_container=macobac_sql_info,
+)
+
+ContainerMap.register_container_pair(
+    cp_container=RayleighExtinctionContainer,
+    db_container=rayleigh_extinction_sql_info,
+)
+ContainerMap.register_container_pair(
+    cp_container=SelectedAtmosphericModelContainer,
+    db_container=selected_model_sql_info,
+)

calibpipe/database/adapter/database_containers/common_metadata.py

@@ -0,0 +1,150 @@
+"""Common Metadata SQL info."""
+
+from sqlalchemy.schema import ForeignKeyConstraint, UniqueConstraint
+
+from ....core.common_metadata_containers import (
+    ActivityReferenceMetadataContainer,
+    ContactReferenceMetadataContainer,
+    InstrumentReferenceMetadataContainer,
+    ProcessReferenceMetadataContainer,
+    ProductReferenceMetadataContainer,
+    ReferenceMetadataContainer,
+)
+from ...interfaces.sql_column_info import SQLColumnInfo
+from ...interfaces.sql_metadata import sql_metadata
+from ...interfaces.sql_table_info import SQLTableInfo
+from ...interfaces.types import Integer, String
+from .container_map import ContainerMap
+
+reference_metadata_sql_info = SQLTableInfo(
+    table_name="reference_metadata",
+    metadata=sql_metadata,
+    columns=[
+        SQLColumnInfo("ID", Integer, unique=True, primary_key=True, autoincrement=True),
+        SQLColumnInfo("ID_optical_throughput", Integer),
+        SQLColumnInfo("version_atmospheric_model", String),
+        SQLColumnInfo("version", String),
+    ],
+    constraints=[
+        ForeignKeyConstraint(["ID_optical_throughput"], ["optical_throughput.ID"]),
+        ForeignKeyConstraint(
+            ["version_atmospheric_model"], ["AtmosphericModel.version"]
+        ),
+        UniqueConstraint("ID"),
+    ],
+)
+
+product_reference_metadata_sql_info = SQLTableInfo(
+    table_name="product_reference_metadata",
+    metadata=sql_metadata,
+    columns=[
+        SQLColumnInfo("ID", Integer, unique=True, primary_key=True),
+        SQLColumnInfo("description", String),
+        SQLColumnInfo("creation_time", String),
+        SQLColumnInfo("product_id", String),
+        SQLColumnInfo("data_category", String),
+        SQLColumnInfo("data_level", String),
+        SQLColumnInfo("data_association", String),
+        SQLColumnInfo("data_type", String),
+        SQLColumnInfo("data_model_name", String),
+        SQLColumnInfo("data_model_version", String),
+        SQLColumnInfo("data_model_url", String),
+        SQLColumnInfo("format", String),
+    ],
+    constraints=[
+        ForeignKeyConstraint(["ID"], ["reference_metadata.ID"]),
+        UniqueConstraint("ID"),
+    ],
+)
+
+contact_reference_metadata_sql_info = SQLTableInfo(
+    table_name="contact_reference_metadata",
+    metadata=sql_metadata,
+    columns=[
+        SQLColumnInfo("ID", Integer, unique=True, primary_key=True),
+        SQLColumnInfo("organization", String),
+        SQLColumnInfo("name", String),
+        SQLColumnInfo("email", String),
+    ],
+    constraints=[
+        ForeignKeyConstraint(["ID"], ["reference_metadata.ID"]),
+        UniqueConstraint("ID"),
+    ],
+)
+
+activity_reference_metadata_sql_info = SQLTableInfo(
+    table_name="activity_reference_metadata",
+    metadata=sql_metadata,
+    columns=[
+        SQLColumnInfo("ID", Integer, unique=True, primary_key=True),
+        SQLColumnInfo("activity_id", String),
+        SQLColumnInfo("name", String),
+        SQLColumnInfo("type", String),
+        SQLColumnInfo("start", String),
+        SQLColumnInfo("end", String),
+        SQLColumnInfo("software_name", String),
+        SQLColumnInfo("software_version", String),
+    ],
+    constraints=[
+        ForeignKeyConstraint(["ID"], ["reference_metadata.ID"]),
+        UniqueConstraint("ID"),
+    ],
+)
+
+process_reference_metadata_sql_info = SQLTableInfo(
+    table_name="process_reference_metadata",
+    metadata=sql_metadata,
+    columns=[
+        SQLColumnInfo("ID", Integer, unique=True, primary_key=True),
+        SQLColumnInfo("type", String),
+        SQLColumnInfo("subtype", String),
+        SQLColumnInfo("subtype_id", String),
+    ],
+    constraints=[
+        ForeignKeyConstraint(["ID"], ["activity_reference_metadata.ID"]),
+        UniqueConstraint("ID"),
+    ],
+)
+
+instrument_reference_metadata_sql_info = SQLTableInfo(
+    table_name="instrument_reference_metadata",
+    metadata=sql_metadata,
+    columns=[
+        SQLColumnInfo("ID", Integer, unique=True, primary_key=True),
+        SQLColumnInfo("site", String),
+        SQLColumnInfo("type", String),
+        SQLColumnInfo("subtype", String),
+        SQLColumnInfo("instrument_id", String),
+    ],
+    constraints=[ForeignKeyConstraint(["ID"], ["process_reference_metadata.ID"])],
+)
+
+ContainerMap.register_container_pair(
+    cp_container=ReferenceMetadataContainer,
+    db_container=reference_metadata_sql_info,
+)
+
+ContainerMap.register_container_pair(
+    cp_container=ProductReferenceMetadataContainer,
+    db_container=product_reference_metadata_sql_info,
+)
+
+ContainerMap.register_container_pair(
+    cp_container=ContactReferenceMetadataContainer,
+    db_container=contact_reference_metadata_sql_info,
+)
+
+ContainerMap.register_container_pair(
+    cp_container=ActivityReferenceMetadataContainer,
+    db_container=activity_reference_metadata_sql_info,
+)
+
+ContainerMap.register_container_pair(
+    cp_container=ProcessReferenceMetadataContainer,
+    db_container=process_reference_metadata_sql_info,
+)
+
+ContainerMap.register_container_pair(
+    cp_container=InstrumentReferenceMetadataContainer,
+    db_container=instrument_reference_metadata_sql_info,
+)

calibpipe/database/adapter/database_containers/container_map.py

@@ -0,0 +1,59 @@
+"""ContainerMap class."""
+
+from ctapipe.core import Container
+
+from ...interfaces.sql_table_info import SQLTableInfo
+
+
+class ContainerMap:
+    """
+    Map CalibPipe and database containers.
+
+    The correspondence must be set using the `register_container_pair()`
+    method, called automatically for the built-in CalibPipe containers. Once
+    set, this map allows us to quickly get the DB container associated
+    to a CalilbPipe container (or contrary) to switch between the two worlds.
+    """
+
+    cp_container = Container
+
+    db_container_info = SQLTableInfo
+    cp_container_type = type[cp_container]
+
+    _cp_containers: dict[db_container_info, cp_container_type] = {}
+    _db_containers: dict[cp_container_type, db_container_info] = {}
+
+    @staticmethod
+    def map_to_cp_container(
+        db_container: db_container_info,
+    ) -> cp_container_type:
+        """Return the CalibPipe container corresponding to a DB container."""
+        return ContainerMap._cp_containers[db_container]
+
+    @staticmethod
+    def map_to_db_container(
+        cp_container: cp_container_type,
+    ) -> db_container_info:
+        """Return the DB container corresponding to a CalibPipe container."""
+        return ContainerMap._db_containers[cp_container]
+
+    @staticmethod
+    def get_cp_containers() -> list:
+        """Return the list of registered CalibPipe containers."""
+        return [*ContainerMap._db_containers]
+
+    @staticmethod
+    def register_container_pair(
+        cp_container: cp_container_type, db_container: db_container_info
+    ) -> None:
+        """Associate a CalibPipe container and a DB container."""
+        ContainerMap._cp_containers[db_container] = cp_container
+        ContainerMap._db_containers[cp_container] = db_container
+
+    @staticmethod
+    def unregister_container_pair(
+        cp_container: cp_container_type, db_container: db_container_info
+    ) -> None:
+        """Deassociate a CalibPipe container and a DB container."""
+        ContainerMap._db_containers[cp_container].pop(db_container, None)
+        ContainerMap._cp_containers[db_container].pop(cp_container, None)

calibpipe/database/adapter/database_containers/observatory.py

@@ -0,0 +1,61 @@
+"""Observatory SQL table info."""
+
+from sqlalchemy.schema import ForeignKeyConstraint, UniqueConstraint
+
+from ....utils.observatory_containers import ObservatoryContainer, SeasonContainer
+from ...interfaces.sql_column_info import SQLColumnInfo
+from ...interfaces.sql_metadata import sql_metadata
+from ...interfaces.sql_table_info import SQLTableInfo
+from ...interfaces.types import Date, Float, Integer, String
+from .container_map import ContainerMap
+
+observatory_sql_info = SQLTableInfo(
+    table_name="Observatory",
+    metadata=sql_metadata,
+    columns=[
+        SQLColumnInfo("name", String, primary_key=True),
+        SQLColumnInfo("latitude", Float, unit="deg"),
+        SQLColumnInfo("longitude", Float, unit="deg"),
+        SQLColumnInfo("elevation", Integer, unit="m"),
+        SQLColumnInfo("version", Integer, primary_key=True),
+    ],
+    constraints=[
+        UniqueConstraint("name", "version", name="observatory_name_version_unique"),
+    ],
+)
+
+season_sql_info = SQLTableInfo(
+    table_name="Season",
+    metadata=sql_metadata,
+    columns=[
+        SQLColumnInfo("start", Date),
+        SQLColumnInfo("stop", Date),
+        SQLColumnInfo("name", String),
+        SQLColumnInfo("alias", String),
+        SQLColumnInfo("name_Observatory", String, nullable=False),
+        SQLColumnInfo("version_Observatory", Integer, nullable=False),
+    ],
+    constraints=[
+        ForeignKeyConstraint(
+            ["name_Observatory", "version_Observatory"],
+            ["Observatory.name", "Observatory.version"],
+        ),
+        UniqueConstraint(
+            "start",
+            "stop",
+            "name_Observatory",
+            "version_Observatory",
+            name="season_unique",
+        ),
+    ],
+)
+
+ContainerMap.register_container_pair(
+    cp_container=ObservatoryContainer,
+    db_container=observatory_sql_info,
+)
+
+ContainerMap.register_container_pair(
+    cp_container=SeasonContainer,
+    db_container=season_sql_info,
+)

calibpipe/database/adapter/database_containers/table_version_manager.py

@@ -0,0 +1,39 @@
+"""TableVersionManager class."""
+
+from sqlalchemy import Table
+
+from ...interfaces.sql_table_info import SQLTableInfo
+
+
+class TableVersionManager:
+    """
+    Create versioned tables for CalibPipe.
+
+    The main method is `apply_version()` that returns a table given
+    a table information and a version. The returned table will
+    contain the same data as the table information and have
+    a unique name containing the common table name and the
+    software version.
+    """
+
+    @staticmethod
+    def get_safe_version_string(version: str) -> str:
+        """Create a string from a version that is safe for table names."""
+        return version.replace("-", "_").replace(".", "_")
+
+    @staticmethod
+    def update_version(old_name: str, version: str) -> str:
+        """Update the DB object name with a new version number."""
+        safe_version = TableVersionManager.get_safe_version_string(version)
+        root_name = old_name.rsplit("_v")[0]
+        new_name = "".join([root_name, "_v", safe_version])
+        return new_name
+
+    @staticmethod
+    def apply_version(table_info: SQLTableInfo, version: str | None = None) -> Table:
+        """Create a DB object class with a particular version number."""
+        if not version:
+            return table_info.get_table()
+        safe_version = TableVersionManager.get_safe_version_string(version)
+        table_name = "".join([table_info.table_name, "_v", safe_version])
+        return table_info.get_table(table_name=table_name)

calibpipe/database/adapter/database_containers/throughput.py

@@ -0,0 +1,30 @@
+"""OpticalThroughput SQL info."""
+
+from calibpipe.telescope.throughput.containers import OpticalThoughtputContainer
+
+from ...interfaces.sql_column_info import SQLColumnInfo
+from ...interfaces.sql_metadata import sql_metadata
+from ...interfaces.sql_table_info import SQLTableInfo
+from ...interfaces.types import DateTime, Float, Integer, String
+from .container_map import ContainerMap
+
+optical_throughput_sql_info = SQLTableInfo(
+    table_name="optical_throughput",
+    metadata=sql_metadata,
+    columns=[
+        SQLColumnInfo("ID", Integer, primary_key=True, autoincrement=True),
+        SQLColumnInfo("tel_id", Integer),
+        SQLColumnInfo("obs_id", Integer),
+        SQLColumnInfo("validity_start", DateTime(timezone=True)),
+        SQLColumnInfo("validity_end", DateTime(timezone=True)),
+        SQLColumnInfo("optical_throughput_coefficient", Float),
+        SQLColumnInfo("optical_throughput_coefficient_std", Float),
+        SQLColumnInfo("method", String),
+        SQLColumnInfo("n_events", Integer),
+    ],
+)
+
+ContainerMap.register_container_pair(
+    cp_container=OpticalThoughtputContainer,
+    db_container=optical_throughput_sql_info,
+)

calibpipe/database/adapter/database_containers/version_control.py

@@ -0,0 +1,17 @@
+"""SQL info for run metadata."""
+
+from ...interfaces.sql_column_info import SQLColumnInfo
+from ...interfaces.sql_metadata import sql_metadata
+from ...interfaces.sql_table_info import SQLTableInfo
+from ...interfaces.types import DateTime, String
+
+version_control_sql_info = SQLTableInfo(
+    table_name="version_control_table",
+    metadata=sql_metadata,
+    columns=[
+        SQLColumnInfo("name", String),
+        SQLColumnInfo("version", String),
+        SQLColumnInfo("validity_start", DateTime(timezone=True)),
+        SQLColumnInfo("validity_end", DateTime(timezone=True)),
+    ],
+)

calibpipe/database/connections/__init__.py

@@ -0,0 +1,28 @@
+"""
+Connection utilities for the database.
+
+The framework used here is `sqlachemy` that can be used with
+different engines and dialects. For now the calibration data
+is stored in a `PostgreSQL` database and accessed using the
+`psycopg` dialect.
+
+The main connection object is the :class:`SQLConnection` that provides the
+interface to a SQL database, not knowing which engine it is
+(can be `Postgres`, `MySQL`, `Oracle` etc.).
+
+To use the :class:`SQLConnection` with a different DB system or dialect,
+it is enough to change the uri and generate the relevant one
+following the example of the postgres uri.
+"""
+
+from .calibpipe_database import CalibPipeDatabase
+from .postgres_utils import get_postgres_uri
+from .sql_connection import SQLConnection
+from .user_confirmation import get_user_confirmation
+
+__all__ = [
+    "SQLConnection",
+    "get_user_confirmation",
+    "get_postgres_uri",
+    "CalibPipeDatabase",
+]

calibpipe/database/connections/calibpipe_database.py

@@ -0,0 +1,60 @@
+"""CalibPipeDatabase class."""
+
+from .postgres_utils import (
+    get_postgres_uri,
+)
+from .sql_connection import SQLConnection
+
+
+class CalibPipeDatabase(SQLConnection):
+    """
+    CalibPipeDatabase connection. For now `SQLConnection` (`PostgreSQL+psycopg`).
+
+    This class simply creates a valid URI from named parameters to create the
+    particular instance of DB used for CalibPipe data and provides no additional
+    interface.
+
+    A few built-in queries can be found in the module
+    :mod:`queries<calibpipe.database.interfaces.queries>`.
+
+    Attributes
+    ----------
+    user: str
+        Username used to connect to the database.
+
+    database: str
+        Name of the database with which the connection must be established.
+
+    password: str
+        Password for the given user.
+
+    host: str, default=`localhost`
+        Database host.
+
+    port: Optional[int], default=None
+        Database port.
+
+    autocommit: bool, default=False
+        Tell if the modifications to the DB must be committed automatically when the
+        connection closes. Default is `False`, in this case the `commit()` method
+        has to be called explicitly.
+
+    """
+
+    def __init__(
+        self,
+        user: str,
+        database: str,
+        password: str,
+        host: str = "localhost",
+        port: int | None = None,
+        autocommit: bool = False,
+    ) -> None:
+        """Initialize the database connection."""
+        uri = get_postgres_uri(
+            user=user, database=database, passwd=password, host=host, port=port
+        )
+        super().__init__(
+            uri=uri,
+            autocommit=autocommit,
+        )