ctao-calibpipe 0.1.0rc7__py3-none-any.whl

calibpipe/tools/init_db.py

@@ -0,0 +1,37 @@
+# CTA-related imports  # noqa: D100
+from ctapipe.core.traits import Unicode
+
+from ..database.adapter.database_containers import ContainerMap
+from ..database.interfaces import TableHandler
+
+# Internal imports
+from .basic_tool_with_db import BasicToolWithDB
+
+
+class CalibPipeDatabaseInitialization(BasicToolWithDB):
+    """Tool to create empty data and metadata tables in the CalibPipe DB."""
+
+    name = Unicode("CalibPipeDatabaseInitialization")
+    description = "Populate an empty databased with empty tables."
+
+    def setup(self):
+        """Parse configuration, setup the database connection and fetch CalibPipe containers."""
+        super().setup()
+        self.containers = ContainerMap.get_cp_containers()
+
+    def start(self):
+        """Create tables in the database."""
+        TableHandler.prepare_db_tables(self.containers, self.database_configuration)
+
+    def finish(self):
+        """Log created tables."""
+        self.log.info(
+            "Data tables for %s was created and uploaded to CalibPipe DB",
+            [_.__name__ for _ in self.containers],
+        )
+
+
+def main():
+    """Run the app."""
+    tool = CalibPipeDatabaseInitialization()
+    tool.run()

calibpipe/tools/macobac_calculator.py

@@ -0,0 +1,82 @@
+"""Calculate the average CO2 concentration of the last 12 months (12-MACOBAC)."""
+
+# Python built-in imports
+from datetime import datetime, timezone
+
+# Third-party imports
+import astropy.units as u
+import numpy as np
+from astropy.table import QTable, Table
+from astropy.time import Time
+from astropy.units.cds import ppm
+
+# CTA-related imports
+from ctapipe.core import Tool
+from ctapipe.core.traits import Path, Unicode
+
+from ..atmosphere.atmosphere_containers import MacobacContainer
+
+# Internal imports
+from ..atmosphere.meteo_data_handlers import CO2DataHandler
+
+
+class CalculateMACOBAC(Tool):
+    """Download Keeling curve data and calculate the average CO2 concentration of the past 12 months."""
+
+    name = Unicode("CalculateMACOBAC")
+    description = "Download Keeling curve data and calculate average CO2 concentration of the past 12 months."
+
+    output_file = Path(
+        "macobac.ecsv", help="Output ecsv file where macobac container will be written"
+    ).tag(config=True)
+
+    classes = [CO2DataHandler]
+
+    def setup(self):
+        """Create CO2DataHandler."""
+        u.add_enabled_units([ppm])
+        self.data_handler = CO2DataHandler(parent=self)
+        self.macobac12_table = None
+
+    def start(self):
+        """Request meteorological data from Scripps server and compute 12-MACOBAC."""
+        self.data_handler.request_data()
+        macobac_table = Table.read(
+            f"{self.data_handler.data_path}/macobac.csv",
+            comment='"',
+            skipinitialspace=True,
+            format="pandas.csv",
+        )
+        mask = macobac_table["CO2"].value != "-99.99"
+        co2_values = macobac_table[mask][::-1][0:12]["CO2"].data
+        macobac12 = np.mean(co2_values.data.astype(float)) * ppm
+        self.log.debug(
+            "CO2 average atmospheric concentration for the previous 12 months: %f",
+            macobac12,
+        )
+        macobac12_container = MacobacContainer(
+            co2_concentration=macobac12,
+            estimation_date=Time(
+                str(datetime.now(timezone.utc).date()), out_subfmt="date"
+            ),
+        )
+        self.macobac12_table = QTable(
+            names=macobac12_container.keys(),
+            rows=[macobac12_container.values()],
+        )
+
+    def finish(self):
+        """Store results and perform the cleanup."""
+        self.log.info("Storing the results and performing the cleanup.")
+        self.macobac12_table.write(
+            self.output_file,
+            format="ascii.ecsv",
+            serialize_method={"estimation_date": "formatted_value"},
+        )
+        self.data_handler.cleanup()
+
+
+def main():
+    """Run the app."""
+    tool = CalculateMACOBAC()
+    tool.run()

calibpipe/tools/molecular_atmospheric_model_producer.py

@@ -0,0 +1,197 @@
+# noqa: D100
+
+import astropy.units as u
+import numpy as np
+from astropy.table import Table
+
+# CTA-related imports
+from ctapipe.core.traits import (
+    Dict,
+    Path,
+    Unicode,
+)
+from molecularprofiles.molecularprofiles import MolecularProfile
+
+from ..atmosphere.atmosphere_containers import (
+    AtmosphericModelContainer,
+    MolecularAtmosphericProfileContainer,
+    MolecularDensityContainer,
+)
+
+# Internal imports
+from ..core.exceptions import (
+    CorruptedInputDataError,
+    MissingInputDataError,
+)
+from ..database.connections import CalibPipeDatabase
+from ..database.interfaces import TableHandler
+from ..utils.observatory import Observatory
+from .atmospheric_base_tool import AtmosphericBaseTool
+
+
+class CreateMolecularAtmosphericModel(AtmosphericBaseTool):
+    """
+    Create an atmospheric model to be used as input for tailored MC simulations.
+
+    The model consists of:
+    - An atmospheric profile.
+    - A Rayleigh extinction table.
+
+    The output data is provided in ECSV data format.
+
+    Raises
+    ------
+    CorruptedInputDataError
+        If the MACOBAC12 table does not contain exactly one row.
+    MissingInputDataError
+        If the requested atmospheric DAS data is not available.
+    """
+
+    name = Unicode("CreateAtmosphericModel")
+    description = "Create an atmospheric profile"
+    aliases = Dict(
+        {"macobac12-table-path": "CreateMolecularAtmosphericModel.macobac12_table_path"}
+    )
+
+    macobac12_table_path = Path(
+        allow_none=False,
+        directory_ok=False,
+        default_value="macobac.ecsv",
+        help="Path to the MACOBAC12 table.",
+    ).tag(config=True)
+
+    def setup(self):
+        """
+        Update configuration and set up the MeteoDataHandler.
+
+        Raises
+        ------
+        CorruptedInputDataError
+            If the MACOBAC12 table does not contain exactly one row.
+        """
+        super().setup()
+
+        self.macobac12_table = Table.read(self.macobac12_table_path)
+        if len(self.macobac12_table) != 1:
+            raise CorruptedInputDataError(
+                "The MACOBAC12 table should contain exactly one row."
+            )
+        self.contemporary_atmospheric_profile = None
+        self.contemporary_rayleigh_extinction_profile = None
+
+    def start(self):
+        """
+        Produce molecular atmopsheric model.
+
+        This method performs the following steps:
+        1. Retrieves necessary atmospheric data from the specified
+            data assimilation system (usually ECMWF).
+        2. Combines the retrieved data with the provided 12-MACOBAC
+            and stored reference atmospheric model
+            to create a contemporary atmospheric profile and a Rayleigh extinction table.
+
+        Raises
+        ------
+        MissingInputDataError
+            If there is an error retrieving the necessary atmospheric data.
+        """
+        observatory = Observatory.from_db(
+            self.database_configuration,
+            site=self.observatory["name"].upper(),
+            version=self.observatory["version"],
+        )
+
+        latitude, longitude = observatory.coordinates
+        dusk, dawn = observatory.get_astronomical_night(self._timestamp)
+        self.data_handler.create_request(
+            start=dusk, stop=dawn, latitude=latitude, longitude=longitude
+        )
+        data_status = self.data_handler.request_data()
+        if data_status == 0:
+            co2_concentration = self.macobac12_table["co2_concentration"][0]
+            molecular_profile = MolecularProfile(
+                f"{self.data_handler.data_path}/merged_file.ecsv",
+                stat_columns=self.DEFAULT_METEO_COLUMNS,
+            )
+            molecular_profile.get_data()
+            # compute the molecular density profile
+            # in order to select a reference atmospheric model
+            # for upper atmosphere
+            site = observatory.name
+            with CalibPipeDatabase(
+                **self.database_configuration,
+            ) as connection:
+                atmospheric_model_table = TableHandler.read_table_from_database(
+                    AtmosphericModelContainer,
+                    connection,
+                    condition=f"(c.current == True) & (c.name_Observatory == '{site}')",
+                )
+                reference_density_table = TableHandler.read_table_from_database(
+                    MolecularDensityContainer,
+                    connection,
+                    condition=f"c.version.in_({list(atmospheric_model_table['version'].data)})",
+                )
+            contemporary_mdp = molecular_profile.create_molecular_density_profile()
+            number_density_at_15km = contemporary_mdp[
+                contemporary_mdp["altitude"] == 15000 * u.m
+            ]["number density"].quantity[0]
+            _, _, version = min(
+                reference_density_table,
+                key=lambda x: abs(
+                    x["density"]
+                    - number_density_at_15km  # FIXME: MDP table should have "number density" column instead of "density"
+                ),
+            )
+            with CalibPipeDatabase(
+                **self.database_configuration,
+            ) as connection:
+                reference_profile_table = TableHandler.read_table_from_database(
+                    MolecularAtmosphericProfileContainer,
+                    connection,
+                    condition=f'c.version == "{version}"',
+                )
+                reference_profile_table.remove_columns(["version"])
+                reference_profile = Table(
+                    [
+                        np.squeeze(reference_profile_table[col])
+                        for col in reference_profile_table.columns
+                    ],
+                    names=reference_profile_table.columns,
+                )
+
+            self.contemporary_atmospheric_profile = (
+                molecular_profile.create_atmospheric_profile(
+                    co2_concentration=co2_concentration,
+                    reference_atmosphere=reference_profile,
+                )
+            )
+            self.contemporary_rayleigh_extinction_profile = (
+                molecular_profile.create_rayleigh_extinction_profile(
+                    co2_concentration=co2_concentration,
+                    reference_atmosphere=reference_profile,
+                )
+            )
+        else:
+            raise MissingInputDataError(
+                "Contemporary meteorological data is not available. "
+                "Please check the configuration and/or try again later."
+            )
+
+    def finish(self):
+        """Save the results and perform cleanup."""
+        self.contemporary_atmospheric_profile.write(
+            f"{self.output_path}/contemporary_atmospheric_profile.{self.output_format}",
+            format=f"{self.output_format}",
+        )
+        self.contemporary_rayleigh_extinction_profile.write(
+            f"{self.output_path}/contemporary_rayleigh_extinction_profile.{self.output_format}",
+            format=f"{self.output_format}",
+        )
+        self.log.info("Shutting down.")
+        self.data_handler.cleanup()
+
+
+def main():
+    """Run the tool."""
+    tool = CreateMolecularAtmosphericModel()
+    tool.run()

calibpipe/tools/observatory_data_db_loader.py

@@ -0,0 +1,71 @@
+import sqlalchemy as sa  # noqa: D100
+
+# Internal imports
+from calibpipe.database.connections import CalibPipeDatabase
+from calibpipe.database.interfaces import TableHandler
+from calibpipe.utils.observatory import Observatory
+
+# CTA-related imports
+from ctapipe.core.traits import List, Unicode
+from traitlets.config import Config
+
+from .basic_tool_with_db import BasicToolWithDB
+
+
+class UploadObservatoryData(BasicToolWithDB):
+    """Upload observatory data to the calibpipe database."""
+
+    name = Unicode("Upload observatory data")
+    description = "Upload observatory data to the calibpipe database."
+
+    observatories = List(help="List of observatories configurations", minlen=1).tag(
+        config=True
+    )
+
+    classes = [
+        Observatory,
+    ]
+
+    def setup(self):
+        """Create Observatory objects from the configuration and register database configuration."""
+        super().setup()
+        self._observatories = [
+            Observatory(config=Config(key))
+            for key in self.config["UploadObservatoryData"]["observatories"]
+        ]
+
+    def start(self):
+        """Create the database tables and insert the data."""
+        containers = [
+            container
+            for observatory in self._observatories
+            for container in observatory.containers
+        ]
+        # Check if the tables exist, if not create them
+        with CalibPipeDatabase(
+            **self.database_configuration,
+        ) as connection:
+            for container in containers:
+                table, insertion = TableHandler.get_database_table_insertion(
+                    container,
+                )
+                if not sa.inspect(connection.engine).has_table(table.name):
+                    table.create(bind=connection.engine)
+        # Insert the data
+        with CalibPipeDatabase(
+            **self.database_configuration,
+        ) as connection:
+            for container in containers:
+                table, insertion = TableHandler.get_database_table_insertion(
+                    container,
+                )
+                TableHandler.insert_row_in_database(table, insertion, connection)
+
+    def finish(self):
+        """No finishing actions needed."""
+
+
+def main():
+    """Run the app."""
+    tool = UploadObservatoryData()
+    tool.run()

calibpipe/tools/reference_atmospheric_model_selector.py

@@ -0,0 +1,201 @@
+import astropy.units as u  # noqa: D100
+import numpy as np
+from astropy.table import Table
+from ctapipe.core.traits import (
+    Dict,
+    Unicode,
+)
+from molecularprofiles.molecularprofiles import MolecularProfile
+
+from ..atmosphere.atmosphere_containers import (
+    AtmosphericModelContainer,
+    MolecularAtmosphericProfileContainer,
+    MolecularDensityContainer,
+    RayleighExtinctionContainer,
+    SelectedAtmosphericModelContainer,
+)
+from ..core.exceptions import IntermittentError
+from ..database.connections import CalibPipeDatabase
+from ..database.interfaces import TableHandler
+from ..utils.observatory import Observatory, SeasonAlias
+from .atmospheric_base_tool import AtmosphericBaseTool
+
+
+class SelectMolecularAtmosphericModel(AtmosphericBaseTool):
+    """Select a reference molecular atmospheric model."""
+
+    name = Unicode("SelectMolecularAtmosphericModel")
+    description = (
+        "Select a reference Molecular Atmospheric Model "
+        "based on the molecular number density at 15km a.s.l. "
+        "computed from contemporary meteorological data."
+    )
+    aliases = Dict(
+        {
+            "timestamp": "SelectMolecularAtmosphericModel.timestamp",
+            "output_path": "SelectMolecularAtmosphericModel.output_path",
+        }
+    )
+
+    def setup(self):
+        """Parse configuration and setup the database connection and MeteoDataHandler."""
+        super().setup()
+        self.selected_model_container = None
+        self.failed_to_retrieve_meteo_data = True
+
+    def start(self):
+        """
+        Download meteorological data and select a reference atmospheric model.
+
+        This method performs the following operations:
+        1. Retrieves the observatory data from the database.
+        2. Calculates the astronomical night based on the observatory's coordinates and the provided timestamp.
+        3. Creates a data request for the calculated time frame and coordinates.
+        4. Attempts to fetch the meteorological data.
+        5a. If meteorological data is not available, select a reference model based on the date.
+        5b. If meteorological data is available, select a reference model based on the molecular number density at 15km a.s.l.
+        """
+        observatory = Observatory.from_db(
+            self.database_configuration,
+            site=self.observatory["name"].upper(),
+            version=self.observatory["version"],
+        )
+        site = observatory.name
+        with CalibPipeDatabase(
+            **self.database_configuration,
+        ) as connection:
+            atmospheric_model_table = TableHandler.read_table_from_database(
+                AtmosphericModelContainer,
+                connection,
+                condition=f"(c.current == True) & (c.name_Observatory == '{site}')",
+            )
+            reference_density_table = TableHandler.read_table_from_database(
+                MolecularDensityContainer,
+                connection,
+                condition=f"c.version.in_({list(atmospheric_model_table['version'].data)})",
+            )
+
+        latitude, longitude = observatory.coordinates
+        dusk, dawn = observatory.get_astronomical_night(self._timestamp)
+        self.data_handler.create_request(
+            start=dusk, stop=dawn, latitude=latitude, longitude=longitude
+        )
+        self.failed_to_retrieve_meteo_data = self.data_handler.request_data()
+        if self.failed_to_retrieve_meteo_data:
+            season = observatory.get_season_from_timestamp(self._timestamp)
+            version = -1
+        else:
+            molecular_profile = MolecularProfile(
+                f"{self.data_handler.data_path}/merged_file.ecsv",
+                stat_columns=self.DEFAULT_METEO_COLUMNS,
+            )
+            molecular_profile.get_data()
+            contemporary_mdp = molecular_profile.create_molecular_density_profile()
+            number_density_at_15km = contemporary_mdp[
+                contemporary_mdp["altitude"] == 15000 * u.m
+            ]["number density"].quantity[0]
+            season, _, version = min(
+                reference_density_table,
+                key=lambda x: abs(
+                    x["density"]
+                    - number_density_at_15km  # FIXME: MDP table should have "number density" column instead of "density"
+                ),
+            )
+
+        selected_season_table = atmospheric_model_table[
+            (atmospheric_model_table["season"] == season)
+        ]
+        if version == -1:
+            version = selected_season_table[(selected_season_table["current"])][
+                "version"
+            ][0]
+
+        self.selected_model_container = SelectedAtmosphericModelContainer(
+            date=self._timestamp.date(),
+            version=version,
+            season=SeasonAlias[season.upper()].value,
+            site=observatory.name,
+            provenance=self.meteo_data_handler,
+        )
+
+    def finish(self):
+        """Store the selected atmospheric model, and perform cleanup.
+
+        Raises
+        ------
+        IntermittentError
+            In case of missing meteorological data.
+        """
+        self.log.info(
+            "Selected atmospheric model container:\n%s", self.selected_model_container
+        )
+
+        with CalibPipeDatabase(
+            **self.database_configuration,
+        ) as connection:
+            table, insertion = TableHandler.get_database_table_insertion(
+                self.selected_model_container,
+            )
+            TableHandler.insert_row_in_database(table, insertion, connection)
+            selected_atmospheric_table = TableHandler.read_table_from_database(
+                MolecularAtmosphericProfileContainer,
+                connection,
+                condition=f"c.version == \"{self.selected_model_container['version']}\"",
+            )
+            selected_rayleigh_extinction_table = TableHandler.read_table_from_database(
+                RayleighExtinctionContainer,
+                connection,
+                condition=f"c.version == \"{self.selected_model_container['version']}\"",
+            )
+        selected_atmospheric_table.remove_column("version")
+        selected_atmospheric_profile = Table(
+            [
+                np.squeeze(selected_atmospheric_table[col])
+                for col in selected_atmospheric_table.columns
+            ],
+            names=selected_atmospheric_table.columns,
+        )
+        selected_atmospheric_profile.write(
+            f"{self.output_path}/selected_atmospheric_profile.{self.output_format}",
+            format=f"{self.output_format}",
+        )
+        selected_rayleigh_extinction_table.remove_column("version")
+
+        rayleigh_extinction_col_names = ["altitude_min", "altitude_max"]
+        rayleigh_extinction_col_names.extend(
+            [
+                f"{name:.1f}"
+                for name in selected_rayleigh_extinction_table["wavelength"].squeeze()
+            ]
+        )
+        data_array = np.hstack(
+            (
+                selected_rayleigh_extinction_table["altitude"].squeeze().to_value(u.km),
+                selected_rayleigh_extinction_table["AOD"].squeeze(),
+            )
+        )
+        selected_rayleigh_extinction_profile = Table(
+            data=data_array,
+            names=rayleigh_extinction_col_names,
+        )
+        selected_rayleigh_extinction_profile["altitude_min"] *= u.km
+        selected_rayleigh_extinction_profile["altitude_max"] *= u.km
+
+        selected_rayleigh_extinction_profile.write(
+            f"{self.output_path}/selected_rayleigh_extinction_profile.{self.output_format}",
+            format=f"{self.output_format}",
+        )
+
+        self.log.info("Shutting down.")
+        self.data_handler.cleanup()
+        if self.failed_to_retrieve_meteo_data:
+            raise IntermittentError(
+                f"Missing meteorological data from {self.meteo_data_handler}. "
+                "This is a known issue, the reference model was selected based on the date."
+            )
+
+
+def main():
+    """Run the app."""
+    tool = SelectMolecularAtmosphericModel()
+    tool.run()

calibpipe/utils/__init__.py

@@ -0,0 +1,10 @@
+from .observatory import Observatory, Season, SeasonAlias  # noqa: D104
+from .observatory_containers import ObservatoryContainer, SeasonContainer
+
+__all__ = [
+    "Observatory",
+    "Season",
+    "SeasonAlias",
+    "ObservatoryContainer",
+    "SeasonContainer",
+]