ctao-calibpipe 0.3.0rc2__py3-none-any.whl

calibpipe/tests/unittests/test_bootstrap_db.py

@@ -0,0 +1,79 @@
+from copy import deepcopy
+from pathlib import Path
+
+import pytest
+import yaml
+from calibpipe.tools.atmospheric_model_db_loader import UploadAtmosphericModel
+from calibpipe.tools.init_db import CalibPipeDatabaseInitialization
+from calibpipe.tools.observatory_data_db_loader import UploadObservatoryData
+from ctapipe.core import run_tool
+from traitlets.config import Config
+
+
+@pytest.mark.db()
+@pytest.mark.order(1)
+def test_init_database():
+    """
+    Fixture to initialize the database and upload required data.
+    This runs before any other database-related tests.
+    """
+    # Paths to configuration files
+    db_config_path = (
+        Path(__file__).parent.parent.parent.parent.parent
+        / "docs/source/user_guide/utils/configuration/db_config.yaml"
+    )
+
+    # Initialize the database
+    with open(db_config_path) as file:
+        db_config = Config(yaml.load(file, Loader=yaml.SafeLoader))
+    tool = CalibPipeDatabaseInitialization(config=db_config)
+    run_tool(tool)
+
+
+@pytest.mark.db()
+@pytest.mark.order(2)
+def test_upload_observatory():
+    db_config_path = (
+        Path(__file__).parent.parent.parent.parent.parent
+        / "docs/source/user_guide/utils/configuration/db_config.yaml"
+    )
+    with open(db_config_path) as file:
+        db_config = Config(yaml.load(file, Loader=yaml.SafeLoader))
+
+    observatory_data_config_path = (
+        Path(__file__).parent.parent.parent.parent.parent
+        / "docs/source/user_guide/utils/configuration/upload_observatory_data_db.yaml"
+    )
+    # Upload observatory data
+    config = deepcopy(db_config)
+    with open(observatory_data_config_path) as file:
+        observatory_data_config = Config(yaml.load(file, Loader=yaml.SafeLoader))
+    config.update(observatory_data_config)
+    tool = UploadObservatoryData(config=config)
+    run_tool(tool)
+
+
+@pytest.mark.db()
+@pytest.mark.order(3)
+def test_upload_atmospheric_models():
+    db_config_path = (
+        Path(__file__).parent.parent.parent.parent.parent
+        / "docs/source/user_guide/utils/configuration/db_config.yaml"
+    )
+
+    with open(db_config_path) as file:
+        db_config = Config(yaml.load(file, Loader=yaml.SafeLoader))
+
+    atmospheric_model_configs = list(
+        Path(__file__).parent.parent.parent.parent.parent.glob(
+            "docs/source/user_guide/utils/configuration/upload_atmospheric*.yaml"
+        )
+    )
+    # Upload atmospheric model data
+    for config_path in atmospheric_model_configs:
+        config = deepcopy(db_config)
+        with open(config_path) as file:
+            atmospheric_model_config = Config(yaml.load(file, Loader=yaml.SafeLoader))
+        config.update(atmospheric_model_config)
+        tool = UploadAtmosphericModel(config=config)
+        run_tool(tool)

calibpipe/tests/unittests/utils/test_observatory.py

@@ -0,0 +1,309 @@
+# Python built-in imports
+import importlib.resources
+from copy import deepcopy
+from datetime import date, datetime, timezone
+
+import astropy.units as u
+import numpy as np
+import pytest
+import sqlalchemy as sa
+
+# Third-party imports
+from astropy.coordinates import Latitude, Longitude
+from astropy.table import Table
+from calibpipe.database.adapter.database_containers import ContainerMap
+from calibpipe.database.connections import CalibPipeDatabase
+from calibpipe.database.interfaces import TableHandler
+from calibpipe.tests.data import utils
+
+# Internal imports
+from calibpipe.utils.observatory import (
+    Observatory,
+    ObservatoryContainer,
+    Season,
+    SeasonContainer,
+)
+from sqlalchemy.exc import IntegrityError
+from traitlets.config import Config
+
+
+class TestSeason:
+    # test season configuration shall contain new year to test the start>stop condition
+    test_season_configuration = {
+        "Season": {
+            "name": "winter",
+            "start_month": 11,
+            "start_day": 16,
+            "stop_month": 4,
+            "stop_day": 30,
+        }
+    }
+
+    @pytest.fixture()
+    def test_season(self):
+        return Season(config=Config(self.test_season_configuration))
+
+    def test_contains_timestamp(self, test_season):
+        test_timestamp = datetime(
+            year=2020, month=2, day=29, hour=0, minute=0, second=0, microsecond=0
+        )
+        assert test_timestamp in test_season
+        test_date = date(year=2020, month=2, day=29)
+        assert test_date in test_season
+        test_date_begin = date(year=2020, month=11, day=16)
+        test_date_end = date(year=2020, month=4, day=30)
+        assert test_date_begin in test_season
+        assert test_date_end in test_season
+        test_date_out = date(year=2020, month=7, day=29)
+        assert test_date_out not in test_season
+
+    def test_properties(self, test_season):
+        assert test_season.start == (11, 16)
+        assert test_season.stop == (4, 30)
+        assert test_season.months == [11, 12, 1, 2, 3, 4]
+        assert test_season.reference_dates == (date(1999, 11, 16), date(2000, 4, 30))
+
+
+class TestObservatory:
+    cta_north_cfg = {
+        "Observatory": {
+            "name": "CTAO-North",
+            "latitude": 28.761795,
+            "longitude": -17.890701,
+            "elevation": 2150,
+            "seasons": [
+                {
+                    "Season": {
+                        "name": "spring",
+                        "start_month": 5,
+                        "start_day": 1,
+                        "stop_month": 6,
+                        "stop_day": 20,
+                    }
+                },
+                {
+                    "Season": {
+                        "name": "summer",
+                        "start_month": 6,
+                        "start_day": 21,
+                        "stop_month": 10,
+                        "stop_day": 4,
+                    }
+                },
+                {
+                    "Season": {
+                        "name": "winter",
+                        "start_month": 11,
+                        "start_day": 16,
+                        "stop_month": 4,
+                    }
+                },
+                {
+                    "Season": {
+                        "name": "fall",
+                        "start_month": 10,
+                        "start_day": 5,
+                        "stop_month": 11,
+                        "stop_day": 15,
+                    }
+                },
+            ],
+        }
+    }
+
+    cta_south_cfg = {
+        "Observatory": {
+            "name": "CTAO-South",
+            "latitude": -24.6272,
+            "longitude": -70.4039,
+            "elevation": 2200,
+            "seasons": [
+                {
+                    "Season": {
+                        "name": "summer",
+                        "start_month": 11,
+                        "start_day": 1,
+                        "stop_month": 5,
+                        "stop_day": 1,
+                    }
+                },
+                {
+                    "Season": {
+                        "name": "winter",
+                        "start_month": 5,
+                        "start_day": 2,
+                        "stop_month": 10,
+                        "stop_day": 31,
+                    }
+                },
+            ],
+        }
+    }
+
+    # CI testing configuration
+    db_config = {
+        "user": "TEST_CALIBPIPE_DB_USER",
+        "password": "DUMMY_PSWD",
+        "database": "TEST_CALIBPIPE_DB",
+        "host": "postgres",
+        "autocommit": True,
+    }
+
+    @pytest.fixture()
+    def cta_north(self):
+        return Observatory(config=Config(self.cta_north_cfg))
+
+    @pytest.fixture()
+    def cta_south(self):
+        return Observatory(config=Config(self.cta_south_cfg))
+
+    def test_check_seasons(self):
+        test_cfg = deepcopy(self.cta_north_cfg)
+        test_cfg["Observatory"]["seasons"][2]["Season"]["start_day"] = 17
+        with pytest.raises(ValueError, match="The seasons don't cover a year"):
+            _ = Observatory(config=Config(test_cfg))
+        test_cfg["Observatory"]["seasons"][2]["Season"]["start_day"] = 16
+        test_cfg["Observatory"]["seasons"][0]["Season"]["start_day"] = 2
+        with pytest.raises(
+            ValueError, match="The season coverage has gaps or overlaps!"
+        ):
+            _ = Observatory(config=Config(test_cfg))
+
+    def test_properties(self, cta_north):
+        assert cta_north.name == "CTAO-NORTH"
+        assert cta_north.coordinates == (
+            Latitude(
+                angle=self.cta_north_cfg["Observatory"]["latitude"],
+                unit=u.deg,
+            ),
+            Longitude(
+                angle=self.cta_north_cfg["Observatory"]["longitude"],
+                unit=u.deg,
+                wrap_angle=180 * u.deg,
+            ),
+        )
+
+    def test_get_astronomical_night(self, cta_north):
+        test_timestamp = datetime(2020, 11, 22, 1, 0, 0, 0)
+        test_dusk = datetime(2020, 11, 21, 19, 44, 47, 805822, tzinfo=timezone.utc)
+        test_dawn = datetime(2020, 11, 22, 6, 10, 44, 48436, tzinfo=timezone.utc)
+        assert cta_north.get_astronomical_night(test_timestamp) == (
+            test_dusk,
+            test_dawn,
+        )
+        test_timestamp_daytime = datetime(2020, 11, 22, 12, 0, 0, 0)
+        with pytest.raises(
+            ValueError, match=r"The provided timestamp .* corresponds to daytime"
+        ):
+            dusk, down = cta_north.get_astronomical_night(test_timestamp_daytime)
+
+    def test_get_season_from_timestamp(self, cta_north):
+        test_timestamp = datetime(2020, 11, 22, 1, 0, 0, 0)
+        assert cta_north.get_season_from_timestamp(test_timestamp).upper() == "WINTER"
+
+    def test_select_season_data(self, cta_north):
+        test_data = Table.read(
+            importlib.resources.files(utils).joinpath(
+                "meteo_data_winter_and_summer.ecsv"
+            )
+        )
+        summer_data = cta_north.select_season_data(test_data, "SUMMER")
+        assert np.all(
+            np.vectorize(lambda x: x.month)(summer_data["Timestamp"].to_datetime()) == 7
+        )
+        winter_data = cta_north.select_season_data(test_data, "WINTER")
+        assert np.all(
+            np.vectorize(lambda x: x.month)(winter_data["Timestamp"].to_datetime())
+            == 12
+        )
+
+    @pytest.mark.db()
+    @pytest.mark.observatory()
+    def test_db_write(self, cta_north, cta_south):
+        with CalibPipeDatabase(
+            **self.db_config,
+        ) as connection:
+            table, insertion = TableHandler.get_database_table_insertion(
+                cta_north.containers[0],  # Observatory table
+            )
+
+            # Check if the Observatory table exists, create one if needed
+            if not sa.inspect(connection.engine).has_table(table.name):
+                table.create(bind=connection.engine)
+
+        with CalibPipeDatabase(
+            **self.db_config,
+        ) as connection:
+            table, insertion = TableHandler.get_database_table_insertion(
+                cta_north.containers[1],  # Season table
+            )
+
+            # Check if the Season table exists, create one if needed
+            if not sa.inspect(connection.engine).has_table(table.name):
+                table.create(bind=connection.engine)
+
+        with CalibPipeDatabase(
+            **self.db_config,
+        ) as connection:
+            for test_observatory in [cta_north, cta_south]:
+                for container in test_observatory.containers:
+                    table, insertion = TableHandler.get_database_table_insertion(
+                        container,
+                    )
+                    try:
+                        TableHandler.insert_row_in_database(
+                            table, insertion, connection
+                        )
+                    except IntegrityError as e:
+                        if "duplicate key value violates unique constraint" in str(e):
+                            connection.session.rollback()  # Unique constraint violation is expected in some tests due to previous DB setup
+                        else:
+                            raise e
+
+    @pytest.mark.db()
+    @pytest.mark.observatory()
+    def test_db_read_raw(self):
+        observatory_table = ContainerMap.map_to_db_container(
+            ObservatoryContainer
+        ).get_table()
+        season_table = ContainerMap.map_to_db_container(SeasonContainer).get_table()
+        query_observatory = observatory_table.select().where(
+            observatory_table.c.name == "CTAO-NORTH"
+        )
+        query_season = season_table.select().where(
+            season_table.c.name_Observatory == "CTAO-NORTH"
+        )
+        with CalibPipeDatabase(
+            **self.db_config,
+        ) as connection:
+            read_observatory = connection.execute(query_observatory).fetchall()
+            assert len(read_observatory) == 1
+            read_season = connection.execute(query_season).fetchall()
+            assert len(read_season) == 4
+
+    @pytest.mark.db()
+    @pytest.mark.observatory()
+    def test_db_table_read_simple(self):
+        with CalibPipeDatabase(
+            **self.db_config,
+        ) as connection:
+            observatory_qtable = TableHandler.read_table_from_database(
+                container=ObservatoryContainer,
+                connection=connection,
+            )
+            # DEBUG purposes only. Use pytest -s -v in order to see this printout
+            print(observatory_qtable)
+
+    @pytest.mark.db()
+    @pytest.mark.observatory()
+    def test_db_table_read_conditional(self):
+        with CalibPipeDatabase(
+            **self.db_config,
+        ) as connection:
+            season_qtable = TableHandler.read_table_from_database(
+                container=SeasonContainer,
+                connection=connection,
+                condition="(c.name_Observatory == 'CTAO-NORTH') & (c.alias == 'INTERMEDIATE')",
+            )
+            # DEBUG purposes only. Use pytest -s -v in order to see this printout
+            print(season_qtable)

calibpipe/tools/atmospheric_base_tool.py

@@ -0,0 +1,78 @@
+from datetime import datetime  # noqa: D100
+
+from ctapipe.core.traits import (
+    AstroTime,
+    CaselessStrEnum,
+    Dict,
+    Integer,
+    Path,
+    Unicode,
+)
+
+from ..atmosphere.meteo_data_handlers import (
+    MeteoDataHandler,
+)
+from .basic_tool_with_db import BasicToolWithDB
+
+
+class AtmosphericBaseTool(BasicToolWithDB):
+    """Basic tool for atmospheric data processing."""
+
+    meteo_data_handler = CaselessStrEnum(
+        values=["GDASDataHandler", "ECMWFDataHandler"],
+        default_value="ECMWFDataHandler",
+        help="Meteorological data handler name",
+    ).tag(config=True)
+
+    observatory = Dict(
+        per_key_traits={
+            "name": Unicode(),
+            "version": Integer(),
+        },
+        default_value={
+            "name": "CTAO-NORTH",
+            "version": 1,
+        },
+        help="Observatory name and configuration version",
+    ).tag(config=True)
+
+    timestamp = AstroTime(
+        allow_none=False,
+        help="A timestamp used to retrieve meteorological data. "
+        "Should correspond to a night time of the observatory. ",
+    ).tag(config=True)
+
+    output_path = Path(
+        "./",
+        help="Path to the output folder where the atmospheric model files will be saved",
+        allow_none=False,
+        directory_ok=True,
+        file_ok=False,
+    ).tag(config=True)
+
+    output_format = Unicode(
+        "ascii.ecsv",
+        help="Output files format",
+        allow_none=False,
+    ).tag(config=True)
+
+    DEFAULT_METEO_COLUMNS = [
+        "Pressure",
+        "Altitude",
+        "Density",
+        "Temperature",
+        "Wind Speed",
+        "Wind Direction",
+        "Relative humidity",
+        "Exponential Density",
+    ]
+
+    classes = [MeteoDataHandler]
+
+    def setup(self):
+        """Set up the tool."""
+        super().setup()
+        self.data_handler = MeteoDataHandler.from_name(
+            self.meteo_data_handler, parent=self
+        )
+        self._timestamp = datetime.fromisoformat(self.timestamp.iso)

calibpipe/tools/atmospheric_model_db_loader.py

@@ -0,0 +1,181 @@
+"""Tool to upload atmospheric models to the CalibPipe DB."""
+
+import astropy.units as u
+import numpy as np
+from astropy.table import QTable
+from astropy.units import Quantity
+from astropy.units.cds import ppm
+
+# Internal imports
+from calibpipe.atmosphere.atmosphere_containers import (
+    AtmosphericModelContainer,
+    MacobacContainer,
+    MolecularAtmosphericProfileContainer,
+    MolecularAtmosphericProfileMetaContainer,
+    MolecularDensityContainer,
+    RayleighExtinctionContainer,
+)
+from calibpipe.database.connections import CalibPipeDatabase
+from calibpipe.database.interfaces import TableHandler
+
+# CTA-related imports
+from ctapipe.core.traits import AstroTime, Bool, Dict, Integer, Path, Unicode
+
+from .basic_tool_with_db import BasicToolWithDB
+
+u.add_enabled_units([ppm])
+
+
+class UploadAtmosphericModel(BasicToolWithDB):
+    """
+    Upload a (reference) atmospheric model to the calibpipe database.
+
+    For the time being the model consists of
+    - a molecular atmospheric profile;
+    - a molecular number density at 15km a.s.l.;
+    - a 12MACOBAC value;
+    - a Rayleigh extinction table - TBA.
+    The input data  for the atmospheric profile is provided in ecsv data format.
+
+    """
+
+    name = Unicode("UploadAtmosphericModel")
+    description = "Upload an atmospheric model to the calibpipe database"
+
+    atmospheric_model = Dict(
+        per_key_traits={
+            "start": AstroTime(),
+            "stop": AstroTime(),
+            "version": Unicode(),
+            "current": Bool(),
+            "season": Unicode(),
+            "name_Observatory": Unicode(),
+            "version_Observatory": Integer(),
+        },
+        help="Atmospheric model metadata",
+    ).tag(config=True)
+
+    macobac_data_path = Path(
+        help="Path to an ecsv file with macobac data that contains the 12-MACOBAC value and estimation date",
+        directory_ok=False,
+        allow_none=False,
+    ).tag(config=True)
+    molecular_density_data_path = Path(
+        help="Path to an ecsv file that contains the molecular number density at 15km a.s.l. for a given atmospheric model",
+        directory_ok=False,
+        allow_none=False,
+    ).tag(config=True)
+
+    molecular_atmospheric_profile = Dict(
+        per_key_traits={
+            "data_assimilation_system": Unicode(),
+            "dataset": Unicode(),
+            "description": Unicode(),
+            "data_path": Path(allow_none=False),
+        },
+        default_value={
+            "data_assimilation_system": "GDAS",
+            "dataset": "ds.083.2",
+            "description": "Test",
+            "data_path": "src/calibpipe/atmosphere/models/test.ecsv",
+        },
+        help="Molecular atmospheric profile data",
+    ).tag(config=True)
+
+    rayleigh_extinction_data_path = Path(
+        help="Path to an ecsv file with rayleigh extinction profile data",
+        directory_ok=False,
+        allow_none=False,
+    ).tag(config=True)
+
+    def setup(self):
+        """Configure atmopsheric model container."""
+        super().setup()
+        self.am_container = AtmosphericModelContainer(**self.atmospheric_model)
+        if self.am_container.start is not None:
+            self.am_container.start = self.am_container.start.to_datetime()
+        if self.am_container.stop is not None:
+            self.am_container.stop = self.am_container.stop.to_datetime()
+        self.map_data_path = self.molecular_atmospheric_profile.pop("data_path", None)
+        self.map_meta_container = MolecularAtmosphericProfileMetaContainer(
+            **self.molecular_atmospheric_profile
+        )
+        self.map_meta_container.version = self.am_container.version
+
+    def start(self):
+        """Fetch atmospheric tables and upload them to the DB."""
+        map_table = QTable.read(self.map_data_path)
+        map_container = MolecularAtmosphericProfileContainer(**dict(map_table.items()))
+        map_container.version = self.am_container.version
+        macobac_table = QTable.read(self.macobac_data_path)
+        macobac_container = MacobacContainer()
+        macobac_container.co2_concentration = macobac_table["co2_concentration"][0]
+        macobac_container.estimation_date = (
+            macobac_table["estimation_date"][0].to_datetime().date()
+        )
+        macobac_container.version = self.am_container.version
+        re_table = QTable.read(self.rayleigh_extinction_data_path)
+        wl_cols = [
+            wl
+            for wl in re_table.colnames
+            if wl != "altitude_min" and wl != "altitude_max"
+        ]
+        wls = np.array([Quantity(wl).to_value(u.nm) for wl in wl_cols]) * u.nm
+        altitudes = re_table[["altitude_min", "altitude_max"]].to_pandas().values * u.km
+        aods = re_table[wl_cols].to_pandas().values * u.dimensionless_unscaled
+        re_container = RayleighExtinctionContainer(
+            wavelength=wls,
+            altitude=altitudes,
+            AOD=aods,
+        )
+        re_container.version = self.am_container.version
+        md_container = MolecularDensityContainer()
+        md_container.version = self.am_container.version
+        md_table = QTable.read(self.molecular_density_data_path)
+        md_container.density = md_table["density"][0]
+        md_container.season = md_table["season"][0]
+
+        containers = [
+            self.am_container,
+            self.map_meta_container,
+            map_container,
+            macobac_container,
+            re_container,
+            md_container,
+        ]
+        with CalibPipeDatabase(
+            **self.database_configuration,
+        ) as connection:
+            for container in containers:
+                table, insertion = TableHandler.get_database_table_insertion(
+                    container,
+                )
+                if (container == self.am_container) and (self.am_container.current):
+                    stmt = (
+                        table.update()
+                        .where(
+                            (table.c.current)
+                            & (table.c.season == self.am_container.season)
+                            & (
+                                table.c.name_Observatory
+                                == self.am_container.name_Observatory
+                            )
+                            & (
+                                table.c.version_Observatory
+                                == self.am_container.version_Observatory
+                            )
+                        )
+                        .values(current=False)
+                    )
+                    connection.execute(stmt)
+                TableHandler.insert_row_in_database(table, insertion, connection)
+
+    def finish(self):
+        """Do nothing."""
+        self.log.info("Shutting down.")
+
+
+def main():
+    """Run the tool."""
+    tool = UploadAtmosphericModel()
+    tool.run()

calibpipe/tools/basic_tool_with_db.py

@@ -0,0 +1,38 @@
+# CTA-related imports  # noqa: D100
+from ctapipe.core import Tool
+from ctapipe.core.traits import Bool, Dict, Integer, Unicode
+
+
+class BasicToolWithDB(Tool):
+    """Basic tool with database connection."""
+
+    name = Unicode("BasicToolWithDB")
+    description = "Basic tool with database connection."
+
+    database_configuration = Dict(
+        per_key_traits={
+            "user": Unicode(),
+            "password": Unicode(),
+            "database": Unicode(),
+            "host": Unicode(),
+            "port": Integer(allow_none=True),
+            "autocommit": Bool(),
+        },
+        default_value={
+            "user": "TEST_CALIBPIPE_DB_USER",
+            "password": "DUMMY_PSWRD",
+            "database": "TEST_CALIBPIPE_DB",
+            "host": "localhost",
+            "port": 5432,
+            "autocommit": True,
+        },
+        help="Database configuration",
+    ).tag(config=True)
+
+    def setup(self):
+        """Set up the database connection."""
+        if (
+            "database_configuration" in self.config
+            and "database_configuration" not in self.config[self.name]
+        ):
+            self.database_configuration = self.config["database_configuration"]