ctao-calibpipe 0.1.0rc8__py3-none-any.whl → 0.2.0rc1__py3-none-any.whl

calibpipe/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.0rc8'
-__version_tuple__ = version_tuple = (0, 1, 0, 'rc8')
+__version__ = version = '0.2.0rc1'
+__version_tuple__ = version_tuple = (0, 2, 0, 'rc1')

calibpipe/core/common_metadata_containers.py

@@ -22,6 +22,9 @@ class ReferenceMetadataContainer(Container):
         description="Version of the reference metadata schema used in the data product",
         type=int,
     )
+    ID_optical_throughput = Field(
+        None, description="Optical throughput ID to serve as a foreign key", type=int
+    )
 
 
 class ProductReferenceMetadataContainer(Container):

calibpipe/database/adapter/adapter.py

@@ -36,7 +36,7 @@ class Adapter:
         Returns
         -------
         Optional[tuple[sa.Table, dict[str, Any]]]
-            None if to DB table information is associated to the container.
+            None if no DB table information is associated to the container.
             A tuple containing the SQL table corresponding to the container,
             and a dictionary with the valued attributes contained in the
             CalibPipe container.

calibpipe/database/adapter/database_containers/__init__.py

@@ -37,6 +37,7 @@ from .common_metadata import (
 from .container_map import ContainerMap
 from .observatory import observatory_sql_info, season_sql_info
 from .table_version_manager import TableVersionManager
+from .throughput import optical_throughput_sql_info
 from .version_control import version_control_sql_info
 
 __all__ = [
@@ -58,4 +59,5 @@ __all__ = [
     "activity_reference_metadata_sql_info",
     "process_reference_metadata_sql_info",
     "instrument_reference_metadata_sql_info",
+    "optical_throughput_sql_info",
 ]

calibpipe/database/adapter/database_containers/common_metadata.py

@@ -21,10 +21,12 @@ reference_metadata_sql_info = SQLTableInfo(
     metadata=sql_metadata,
     columns=[
         SQLColumnInfo("ID", Integer, unique=True, primary_key=True, autoincrement=True),
+        SQLColumnInfo("ID_optical_throughput", Integer),
         SQLColumnInfo("version_atmospheric_model", String),
         SQLColumnInfo("version", String),
     ],
     constraints=[
+        ForeignKeyConstraint(["ID_optical_throughput"], ["optical_throughput.ID"]),
         ForeignKeyConstraint(
             ["version_atmospheric_model"], ["AtmosphericModel.version"]
         ),

calibpipe/database/adapter/database_containers/throughput.py

@@ -0,0 +1,30 @@
+"""OpticalThroughput SQL info."""
+
+from calibpipe.telescope.throughput.containers import OpticalThoughtputContainer
+
+from ...interfaces.sql_column_info import SQLColumnInfo
+from ...interfaces.sql_metadata import sql_metadata
+from ...interfaces.sql_table_info import SQLTableInfo
+from ...interfaces.types import DateTime, Float, Integer, String
+from .container_map import ContainerMap
+
+optical_throughput_sql_info = SQLTableInfo(
+    table_name="optical_throughput",
+    metadata=sql_metadata,
+    columns=[
+        SQLColumnInfo("ID", Integer, primary_key=True, autoincrement=True),
+        SQLColumnInfo("tel_id", Integer),
+        SQLColumnInfo("obs_id", Integer),
+        SQLColumnInfo("validity_start", DateTime(timezone=True)),
+        SQLColumnInfo("validity_end", DateTime(timezone=True)),
+        SQLColumnInfo("optical_throughput_coefficient", Float),
+        SQLColumnInfo("optical_throughput_coefficient_std", Float),
+        SQLColumnInfo("method", String),
+        SQLColumnInfo("n_events", Integer),
+    ],
+)
+
+ContainerMap.register_container_pair(
+    cp_container=OpticalThoughtputContainer,
+    db_container=optical_throughput_sql_info,
+)

calibpipe/database/interfaces/table_handler.py

@@ -9,8 +9,6 @@ import sqlalchemy as sa
 from astropy.table import QTable
 from ctapipe.core import Container
 
-import calibpipe.core.common_metadata_containers as common_metadata_module
-
 from ...core.exceptions import DBStorageError
 from ..adapter.adapter import Adapter
 from ..adapter.database_containers.container_map import ContainerMap
@@ -235,36 +233,38 @@ class TableHandler:
             raise DBStorageError("Issues with connection to the CalibPipe DB")
 
     @staticmethod
-    def upload_data(calibpipe_data_container, config_data):
+    def upload_data(
+        calibpipe_data_container: Container,
+        metadata: list[Container] | None,
+        connection: CalibPipeDatabase,
+    ) -> None:
         """
-        Universal function to upload data and metadata to the DB.
+        Upload data and optional metadata to the database.
 
-        Metadata is uploaded based on values in the dictionary config_data.
-        It is possible to update fields in the dictionary while performing calibration,
-        and transfer the final metadata collection to this function.
+        This method uploads the provided data to the main database table and,
+        if provided, associates the metadata with the inserted data row.
 
         Parameters
         ----------
-        calibpipe_data_container : ctapipe.container
-            calibpipe container with data that will be uploaded to the main table of DB
+        calibpipe_data_container : ctapipe.core.Container
+            The data container with the data to be uploaded to the main table of the database.
 
-        config_data : dict
-            dictionary with configurable values,
-            should contain at least DB configuration
-            and metadata information for each metadata table.
-
-        Returns
-        -------
-        insertion_list : list
-            list of metadata dictionaries that were uploaded to DB
-        """
-        insertion_list = []
-        metadata_dict = {
-            container: values
-            for container, values in config_data.items()
-            if "Reference" in container
-        }
+        metadata : list[Container] or None
+            Optional list of metadata containers to be uploaded. Should include
+            a "ReferenceMetadataContainer" if metadata is provided.
+
+        connection : CalibPipeDatabase
+            An active database connection to the CalibPipe database.
 
+        Raises
+        ------
+        DBStorageError
+            If there are issues with the database connection.
+
+        ValueError
+            If the main table does not contain a single autoincremented primary key
+            or if ReferenceMetadataContainer is missing when metadata is provided.
+        """
         data_db_container = ContainerMap.map_to_db_container(
             type(calibpipe_data_container)
         )
@@ -272,80 +272,62 @@ class TableHandler:
             col.autoincrement for col in data_db_container.get_table().c
         )
         is_single_pk = len(data_db_container.get_primary_keys()) == 1
-        # Check if there are only one autoincremented pk in the table
-        if has_autoincrement_pk and is_single_pk:
-            pk_name = data_db_container.get_primary_keys()[0].name
-            try:
-                with CalibPipeDatabase(
-                    **config_data["database_configuration"]
-                ) as connection:
-                    TableHandler.insert_row_in_database(
-                        data_db_container.get_table(),
-                        calibpipe_data_container,
-                        connection,
-                    )
-                    # Get the last uploaded DB record,
-                    # to which all metadata will be attached
-                    stmt = (
-                        sa.select(data_db_container.get_table())
-                        .order_by(sa.desc(data_db_container.get_table().c[pk_name]))
-                        .limit(1)
-                    )
-                    last_db_record = connection.execute(stmt).fetchone()
-                    data_pk_value = last_db_record._asdict()[pk_name]
-
-                    # We should process Reference metadata separately,
-                    # because it contains autoincremented PK
-                    # to which all other metadata are connected
-                    cp_container = getattr(
-                        common_metadata_module, "ReferenceMetadataContainer"
-                    )
-                    db_container = ContainerMap.map_to_db_container(cp_container)
-                    reference_meta_insertion = cp_container(
-                        ID_optical_throughput=data_pk_value,
-                        **config_data["ReferenceMetadataContainer"],
-                    )
-                    TableHandler.insert_row_in_database(
-                        db_container.get_table(), reference_meta_insertion, connection
-                    )
-
-                    # Extract value of the Reference metadata PK,
-                    # and connect to it all other metadata tables
-                    stmt = (
-                        sa.select(db_container.get_table())
-                        .order_by(sa.desc(db_container.get_table().c.ID))
-                        .limit(1)
-                    )
-                    metadata_id = connection.execute(stmt).fetchone()
-
-                    # Remove Reference metadata from the dict to not process it second time
-                    metadata_dict.pop("ReferenceMetadataContainer", None)
-
-                    # Create list with values that should be inserted
-                    # to the metadata tables in the DB
-                    for container in metadata_dict.keys():
-                        cp_container = getattr(common_metadata_module, container)
-                        insertion_list.append(
-                            cp_container(ID=metadata_id.ID, **config_data[container])
-                        )
 
-                    # Upload metadata values to the DB
-                    for insertion, container in zip(
-                        insertion_list, metadata_dict.keys()
-                    ):
-                        cp_container = getattr(common_metadata_module, container)
-                        db_container = ContainerMap.map_to_db_container(cp_container)
-                        TableHandler.insert_row_in_database(
-                            db_container.get_table(), insertion, connection
-                        )
-
-                    insertion_list = [reference_meta_insertion] + insertion_list
-            except sa.exc.DatabaseError:
-                raise DBStorageError("Issues with connection to the CalibPipe DB")
-        else:
+        if not (has_autoincrement_pk and is_single_pk):
             raise ValueError(
                 f"Table '{data_db_container.table_name}' "
-                "doesn't contain single autoincremented primary key."
+                "doesn't contain a single autoincremented primary key."
             )
 
-        return insertion_list
+        pk_name = data_db_container.get_primary_keys()[0].name
+
+        try:
+            # Insert main data
+            table, values = TableHandler.get_database_table_insertion(
+                calibpipe_data_container
+            )
+            TableHandler.insert_row_in_database(table, values, connection)
+
+            # No metadata to upload
+            if not metadata:
+                return
+
+            # Get the last inserted primary key
+            stmt = sa.select(table).order_by(sa.desc(table.c[pk_name])).limit(1)
+            last_db_record = connection.execute(stmt).fetchone()
+            data_pk_value = last_db_record._asdict()[pk_name]
+
+            # Handle ReferenceMetadataContainer
+            reference_meta_container = next(
+                (
+                    c
+                    for c in metadata
+                    if c.__class__.__name__ == "ReferenceMetadataContainer"
+                ),
+                None,
+            )
+            if reference_meta_container is None:
+                raise ValueError("ReferenceMetadataContainer is required in metadata.")
+
+            reference_meta_container.ID_optical_throughput = data_pk_value
+            ref_table, ref_values = TableHandler.get_database_table_insertion(
+                reference_meta_container
+            )
+            TableHandler.insert_row_in_database(ref_table, ref_values, connection)
+
+            # Get ReferenceMetadata ID to link to other metadata
+            stmt = sa.select(ref_table).order_by(sa.desc(ref_table.c.ID)).limit(1)
+            metadata_id = connection.execute(stmt).fetchone().ID
+
+            # Upload other metadata
+            for container in metadata:
+                if container.__class__.__name__ == "ReferenceMetadataContainer":
+                    continue
+                container.ID = metadata_id
+                meta_table, meta_values = TableHandler.get_database_table_insertion(
+                    container
+                )
+                TableHandler.insert_row_in_database(meta_table, meta_values, connection)
+
+        except sa.exc.DatabaseError:
+            raise DBStorageError("Issues with connection to the CalibPipe DB")

calibpipe/telescope/throughput/containers.py

@@ -0,0 +1,59 @@
+"""Containers to keep optical throughput data and metadata."""
+
+import datetime
+
+import numpy as np
+from ctapipe.core import Container, Field
+
+# UNIX_TIME_ZERO value corresponds to "epoch" special time from PostgreSQL documentation:
+# https://www.postgresql.org/docs/current/datatype-datetime.html#DATATYPE-DATETIME-SPECIAL-TABLE
+
+UNIX_TIME_ZERO = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
+
+
+class OpticalThoughtputContainer(Container):
+    """Optical throughput calibration coefficient and analysis results for a single telescope."""
+
+    optical_throughput_coefficient = Field(
+        np.nan,
+        "Optical throughput from the selected calibration method",
+        type=np.float64,
+        allow_none=False,
+    )
+    optical_throughput_coefficient_std = Field(
+        np.nan,
+        "Optical throughput from the selected calibration method",
+        type=np.float64,
+        allow_none=False,
+    )
+    method = Field(
+        "None",
+        "Calibration method used to fill optical_throughput_coefficient",
+        type=str,
+        allow_none=False,
+    )
+    validity_start = Field(
+        UNIX_TIME_ZERO,
+        description="Starting timestamp of validity for the selected throughput.",
+        type=datetime.datetime,
+        allow_none=False,
+    )
+    validity_end = Field(
+        UNIX_TIME_ZERO,
+        description="Ending timestamp of validity for the selected throughput.",
+        type=datetime.datetime,
+        allow_none=False,
+    )
+    obs_id = Field(
+        -1,
+        description="ID of the observation block for validity",
+        type=np.int32,
+        allow_none=False,
+    )
+    tel_id = Field(-1, description="Telescope ID", type=np.int32, allow_none=False)
+    n_events = Field(
+        0,
+        description="Number of muon rings used to calculate the throughput",
+        type=np.int32,
+        allow_none=False,
+    )