ctao-calibpipe 0.1.0__py3-none-any.whl

calibpipe/atmosphere/templates/request_templates/__init__.py

@@ -0,0 +1 @@
+"""Templates for DAS data requests."""

calibpipe/atmosphere/templates/request_templates/copernicus.json

@@ -0,0 +1,11 @@
+{
+    "product_type": "reanalysis",
+    "variable": ["divergence", "geopotential", "ozone_mass_mixing_ratio", "potential_vorticity", "relative_humidity", "temperature", "u_component_of_wind", "v_component_of_wind", "vertical_velocity"],
+    "pressure_level": ["1", "2", "3", "5", "7", "10", "20", "30", "50", "70", "100", "125", "150", "175", "200", "225", "250", "300", "350", "400", "450", "500", "550", "600", "650", "700", "750", "775", "800", "825", "850", "875", "900", "925", "950", "975", "1000"],
+    "year": [2023],
+    "month": [1],
+    "day": [20],
+    "time": ["19:00", "20:00", "21:00", "22:00", "23:00"],
+    "area": [28.74, -18.01, 28.76, -17.99],
+    "format": "grib"
+}

calibpipe/atmosphere/templates/request_templates/gdas.json

@@ -0,0 +1,12 @@
+{
+	"dataset": "ds083.2",
+	"date": "202301260600/to/202301270600",
+	"datetype": "init",
+	"param": "HGT/PRES/TMP/R H/P WAT/A PCP/U GRD/V GRD/T CDC/LANDN/TOZNE",
+	"level": "ISBL:1000/975/950/925/900/850/800/750/700/650/600/550/500/450/400/350/300/250/200/150/100/50/20",
+	"nlat": -24,
+	"slat": -24,
+	"elon": -70,
+	"wlon": -70,
+	"product": "Analysis"
+}

calibpipe/core/__init__.py

@@ -0,0 +1,39 @@
+"""Provide core classes and methods for CalibPipe."""
+
+from .common_metadata_containers import (
+    ActivityReferenceMetadataContainer,
+    ContactReferenceMetadataContainer,
+    InstrumentReferenceMetadataContainer,
+    ProcessReferenceMetadataContainer,
+    ProductReferenceMetadataContainer,
+    ReferenceMetadataContainer,
+)
+from .exceptions import (
+    CalibrationStorageError,
+    ConfigurationError,
+    CorruptedInputDataError,
+    DBStorageError,
+    FileStorageError,
+    InsufficientStatisticsError,
+    IntermittentError,
+    MissingInputDataError,
+)
+
+__all__ = [
+    # exceptions
+    "CorruptedInputDataError",
+    "ConfigurationError",
+    "MissingInputDataError",
+    "InsufficientStatisticsError",
+    "IntermittentError",
+    "DBStorageError",
+    "FileStorageError",
+    "CalibrationStorageError",
+    # common_metadata_containers
+    "ReferenceMetadataContainer",
+    "ProductReferenceMetadataContainer",
+    "ContactReferenceMetadataContainer",
+    "ProcessReferenceMetadataContainer",
+    "ActivityReferenceMetadataContainer",
+    "InstrumentReferenceMetadataContainer",
+]

calibpipe/core/common_metadata_containers.py

@@ -0,0 +1,195 @@
+"""Common metadata containers."""
+
+from ctapipe.core import Container, Field
+
+
+class ReferenceMetadataContainer(Container):
+    """
+    Container to store reference metadata.
+
+    Serves as a central node for other metadata containers.
+    This is the essential metadata that must be attached
+    to all data products to uniquely describe the data product.
+    """
+
+    version_atmospheric_model = Field(
+        None,
+        description="Atmospheric Model version value to serve as a foreign key",
+        type=str,
+    )
+    version = Field(
+        None,
+        description="Version of the reference metadata schema used in the data product",
+        type=int,
+    )
+
+
+class ProductReferenceMetadataContainer(Container):
+    """
+    Container to store product-related reference metadata.
+
+    Describes the details of the data product, including its type and links to the data model definition.
+    """
+
+    ID = Field(
+        None,
+        description="Autoincremented value to serve as primary/foreign key",
+        type=int,
+    )
+    description = Field(
+        None, description="Human-readable description of data product", type=str
+    )
+    creation_time = Field(
+        None,
+        description="Human-readable date and time of file creation, in ISO format, UTC",
+        type=str,
+    )
+    product_id = Field(
+        None,
+        description="A fixed-id to identify this product, e.g. UUID or VFN",
+        type=str,
+    )
+    data_category = Field(None, description="A,B,C", type=str)
+    data_level = Field(None, description="R0, R1, DL0, DL1, etc", type=str)
+    data_association = Field(
+        None, description="CTAO, Site, Subarray, Telescope, etc", type=str
+    )
+    data_type = Field(None, description="Event, Monitoring, Service, etc", type=str)
+    data_model_name = Field(
+        None, description="Identifying name of the data model used", type=str
+    )
+    data_model_version = Field(
+        None, description="Version of the data model used", type=str
+    )
+    data_model_url = Field(
+        None,
+        description="Link to definition of data model, if available, and preferably a DOI",
+        type=str,
+    )
+    format = Field(None, description="File format of the data product", type=str)
+
+
+class ContactReferenceMetadataContainer(Container):
+    """
+    Container to store contact reference metadata.
+
+    Describes the person or institution that is responsible for this data product.
+    """
+
+    ID = Field(
+        None,
+        description="Autoincremented value to serve as primary/foreign key",
+        type=int,
+    )
+    organization = Field(
+        None,
+        description="Organization to which this data product is associated",
+        type=str,
+    )
+    name = Field(None, description="Name of contact within organization", type=str)
+    email = Field(None, description="Contact email address", type=str)
+
+
+class ProcessReferenceMetadataContainer(Container):
+    """
+    Container to store process reference metadata.
+
+    Describes the top-level process
+    to which the activity that generated this product belongs.
+    """
+
+    ID = Field(
+        None,
+        description="Autoincremented value to serve as primary/foreign key",
+        type=int,
+    )
+    type = Field(None, description="General type of the process", type=str)
+    subtype = Field(
+        None,
+        description="More specific class of the process if the class is not sufficient to describe it",
+        type=str,
+    )
+    subtype_id = Field(
+        None,
+        description="Unique identifier of the process, e.g. if the type is observation this is the obs_id",
+        type=str,
+    )
+
+
+class ActivityReferenceMetadataContainer(Container):
+    """
+    Container to store activity reference metadata.
+
+    Describes the specific software
+    or task that generated this particular data product.
+    """
+
+    ID = Field(
+        None,
+        description="Autoincremented value to serve as primary/foreign key",
+        type=int,
+    )
+    activity_id = Field(
+        None,
+        description="Unique identifier of the instance of this activity, if software a UUID",
+        type=str,
+    )
+    name = Field(
+        None,
+        description="Name of activity that produced this data product, e.g. the software/script name",
+        type=str,
+    )
+    type = Field(None, description="General type of the activity", type=str)
+    start = Field(
+        None, description="Starting date/time of activity, in ISO format, UTC", type=str
+    )
+    end = Field(
+        None, description="Ending date/time of activity, in ISO format, UTC", type=str
+    )
+    software_name = Field(
+        None,
+        description="Name of software framework/library that was used if the activity if it involved software",
+        type=str,
+    )
+    software_version = Field(None, description="Version of software used", type=str)
+
+
+class InstrumentReferenceMetadataContainer(Container):
+    """
+    Container to store instrument reference metadata.
+
+    Describes the subset of CTAO Instrument Description to which this data product is associated,
+    which could be e.g. a Sub-array, or a small part such as a photo-sensor.
+    """
+
+    ID = Field(
+        None,
+        description="Autoincremented value to serve as primary/foreign key",
+        type=int,
+    )
+    site = Field(
+        None,
+        description="CTAO-South, CTAO-North, or other site associated with the data product",
+        type=str,
+    )
+    type = Field(
+        None,
+        description=(
+            "The specific type of instrument in the class. E.g. if class=camera, ",
+            "the type might be CHEC, or if class=telescope, the type might be 'SST'",
+        ),
+        type=str,
+    )
+    subtype = Field(
+        None,
+        description="Sub-type of the instrument. For example if the type is MST, this might be 'NectarCAM' or 'FlashCAM'",
+        type=str,
+    )
+    instrument_id = Field(
+        None,
+        description=(
+            "The unique ID of the specific instrument in the class. Depending on the instrument class,"
+            "this might be for example is a telescope id, nominal subarray name, camera_id, or part serial number"
+        ),
+        type=str,
+    )

calibpipe/core/exceptions.py

@@ -0,0 +1,87 @@
+"""
+Calibpipe-specific exceptions.
+
+Exit code values are following the ICD_.
+
+.. _ICD: https://gitlab.cta-observatory.org/cta-computing/dpps/dpps-project-management/icd-pipelines-workload/-/jobs/artifacts/main/raw/build/pipelines-workload.pdf?job=build
+"""
+
+
+class CalibrationError(Exception):
+    """Base class for all CalibPipe exceptions."""
+
+    exit_code = 1
+
+
+class IntermittentError(CalibrationError):
+    """Indicate an intermittent known issue that might be resolved by re-submitting the job."""
+
+    exit_code = 100
+
+    def __init__(self, message="Intermittent runtime error."):
+        super().__init__(message)
+
+
+class CorruptedInputDataError(CalibrationError):
+    """Raise when the input data file is available but cannot be correctly processed."""
+
+    exit_code = 104  # Activate an alternative scenario
+
+    def __init__(self, message="Input data is corrupted."):
+        super().__init__(message)
+
+
+class MissingInputDataError(CalibrationError):
+    """Raise when expected input data is missing."""
+
+    exit_code = 105  # Activate an alternative scenario
+
+    def __init__(self, message="Input data is missing."):
+        super().__init__(message)
+
+
+class InsufficientStatisticsError(CalibrationError):
+    """Raise when data quality is found to be insufficient after processing."""
+
+    exit_code = 106  # Activate an alternative scenario
+
+    def __init__(self, message="Quality of input data is not sufficient."):
+        super().__init__(message)
+
+
+class ConfigurationError(CalibrationError):
+    """Raise for issues with the configuration files."""
+
+    exit_code = 101  # Configuration problem exit code due to IOError
+
+    def __init__(self, message="Problem with the input configuration file.", *args):
+        if args:
+            message = message % args
+        super().__init__(message)
+
+
+class CalibrationStorageError(CalibrationError):
+    """Raise for database/ECSV table storage related exceptions."""
+
+    exit_code = 102  # Runtime error exit code due to IOError
+
+    def __init__(self, message="Calibration coefficient cannot be stored."):
+        super().__init__(message)
+
+
+class DBStorageError(CalibrationStorageError):
+    """Raise for issues with data storage in the database."""
+
+    exit_code = 102  # Runtime error exit code due to IOError
+
+    def __init__(self, message="Storage in the database cannot be performed."):
+        super().__init__(message)
+
+
+class FileStorageError(CalibrationStorageError):
+    """Raise for issues with data storage in ECSV files."""
+
+    exit_code = 102  # Runtime error exit code due to IOError
+
+    def __init__(self, message="Storage in the output file cannot be performed."):
+        super().__init__(message)

calibpipe/database/__init__.py

@@ -0,0 +1,24 @@
+"""
+Database utilities.
+
+This sub-package contains general purpose utilities for database management,
+and also the specialization for LST. LST-related code is contained in
+the sub-packages starting with `lst_`, the rest is decoupled from it.
+
+*Content:*
+ - :mod:`connections <calibpipe.database.connections>`:
+
+ General purpose connection (`SQLConnection`) using sqlalchemy
+ and specialisation for a `PostgreSQL+psycopg` database.
+
+ - :mod:`interfaces <calibpipe.database.interfaces>`:
+
+ General utilities to create and manipulate database objects
+ (fields, columns, tables, and the hashable row information used for cache
+ indexing, queries).
+
+ - :mod:`adapter <calibpipe.database.adapter>`:
+
+All containers and utilities to convert ctapipe containers to tables that fill
+the CalibPipe postgres database.
+"""

calibpipe/database/adapter/__init__.py

@@ -0,0 +1,23 @@
+"""
+Adapter sub-package.
+
+Contains all utilities to adapt CalibPipe data to a DB and use it.
+
+Sub-package content:
+
+ - :mod:`database_containers<calibpipe.database.adapter.database_containers>`:
+   DB containers (in correspondence with CalibPipe containers)
+   to connect CalibPipe data and the DB system.
+ - :mod:`adapter<calibpipe.database.adapter.adapter>`:
+   Small class used to translate CalibPipe container data
+   to DB-level data that can be directly inserted in a DB.
+
+"""
+
+from . import database_containers
+from .adapter import Adapter
+
+__all__ = [
+    "database_containers",
+    "Adapter",
+]

calibpipe/database/adapter/adapter.py

@@ -0,0 +1,80 @@
+"""Adapter class."""
+
+from typing import Any
+
+import astropy.units
+import sqlalchemy as sa
+
+from .database_containers.container_map import ContainerMap
+from .database_containers.table_version_manager import TableVersionManager
+
+
+class Adapter:
+    """
+    Adapt CalibPipe containers into DB objects.
+
+    The main method is to_postgres(), converting a CalibPipe container
+    to a tuple containing the objects required to interact with the
+    corresponding data in the database.
+    """
+
+    @staticmethod
+    def to_postgres(
+        container: ContainerMap.cp_container, version: str | None = None
+    ) -> tuple[sa.Table, dict[str, Any]] | None:
+        """
+        Convert a CalibPipe container in DB data that can be inserted or queried.
+
+        Parameters
+        ----------
+        container: ContainerMap.cp_container
+            CalibPipe container (containing data) to convert to DB-level data
+
+        version: str
+            Software version corresponding to the data
+
+        Returns
+        -------
+        Optional[tuple[sa.Table, dict[str, Any]]]
+            None if to DB table information is associated to the container.
+            A tuple containing the SQL table corresponding to the container,
+            and a dictionary with the valued attributes contained in the
+            CalibPipe container.
+        """
+        try:
+            table_info = ContainerMap.map_to_db_container(type(container))
+            table = TableVersionManager.apply_version(
+                table_info=table_info, version=version
+            )
+            return table, Adapter.get_db_kwargs(container, table_info)
+        except KeyError:
+            return None
+
+    @staticmethod
+    def get_db_kwargs(
+        container: ContainerMap.cp_container,
+        table_info: ContainerMap.db_container_info,
+    ) -> dict[str, Any]:
+        """Get kwargs required to insert a row from a CalibPipe container."""
+        return {
+            name: Adapter._remove_astropy_unit(value, col.unit)
+            for (name, value), col in zip(container.items(), table_info.columns)
+        }
+
+    @staticmethod
+    def _remove_astropy_unit(value, unit):
+        """
+        Remove the astropy unit from a value if necessary.
+
+        Astropy units cannot be stored in a database, or at least not
+        in a single field. Units are therefore removed and the value is
+        converted to either units from the column description
+        or to SI units for consistency.
+        """
+        if isinstance(value, astropy.units.Quantity):
+            if unit is not None:
+                return (
+                    value.decompose().to(unit).value
+                )  # Value only, in the units from column description
+            return value.decompose().value  # Value only, in S.I. units
+        return value

calibpipe/database/adapter/database_containers/__init__.py

@@ -0,0 +1,61 @@
+"""
+Contain database containers for CalibPipe and the table version manager.
+
+All CalibPipe data is defined at the database level
+in this sub-package. In particular, a `SQLTableInformation` can be
+found for each CalibPipe container.
+
+In addition, two utilities are present here to ease the use of DB
+containers:
+
+- `TableVersionManager`: Creates tables objects specific to a version number
+  (needed for CalibPipe data).
+- `ContainerMap`: Maps CalibPipe containers (ultimately `ctapipe.core.container`)
+  to database table information. The mapping is built-in,
+  and allows users to quickly access the DB container
+  corresponding to a CalibPipe container (or the contrary).
+
+"""
+
+from .atmosphere import (
+    atmospheric_model_sql_info,
+    macobac_sql_info,
+    map_meta_sql_info,
+    map_sql_info,
+    mdp_sql_info,
+    rayleigh_extinction_sql_info,
+    selected_model_sql_info,
+)
+from .common_metadata import (
+    activity_reference_metadata_sql_info,
+    contact_reference_metadata_sql_info,
+    instrument_reference_metadata_sql_info,
+    process_reference_metadata_sql_info,
+    product_reference_metadata_sql_info,
+    reference_metadata_sql_info,
+)
+from .container_map import ContainerMap
+from .observatory import observatory_sql_info, season_sql_info
+from .table_version_manager import TableVersionManager
+from .version_control import version_control_sql_info
+
+__all__ = [
+    "atmospheric_model_sql_info",
+    "map_meta_sql_info",
+    "map_sql_info",
+    "mdp_sql_info",
+    "macobac_sql_info",
+    "selected_model_sql_info",
+    "observatory_sql_info",
+    "rayleigh_extinction_sql_info",
+    "season_sql_info",
+    "version_control_sql_info",
+    "TableVersionManager",
+    "ContainerMap",
+    "reference_metadata_sql_info",
+    "product_reference_metadata_sql_info",
+    "contact_reference_metadata_sql_info",
+    "activity_reference_metadata_sql_info",
+    "process_reference_metadata_sql_info",
+    "instrument_reference_metadata_sql_info",
+]