ctao-calibpipe 0.3.0rc2__py3-none-any.whl

calibpipe/tools/molecular_atmospheric_model_producer.py

@@ -0,0 +1,197 @@
+# noqa: D100
+
+import astropy.units as u
+import numpy as np
+from astropy.table import Table
+
+# CTA-related imports
+from ctapipe.core.traits import (
+    Dict,
+    Path,
+    Unicode,
+)
+from molecularprofiles.molecularprofiles import MolecularProfile
+
+from ..atmosphere.atmosphere_containers import (
+    AtmosphericModelContainer,
+    MolecularAtmosphericProfileContainer,
+    MolecularDensityContainer,
+)
+
+# Internal imports
+from ..core.exceptions import (
+    CorruptedInputDataError,
+    MissingInputDataError,
+)
+from ..database.connections import CalibPipeDatabase
+from ..database.interfaces import TableHandler
+from ..utils.observatory import Observatory
+from .atmospheric_base_tool import AtmosphericBaseTool
+
+
+class CreateMolecularAtmosphericModel(AtmosphericBaseTool):
+    """
+    Create an atmospheric model to be used as input for tailored MC simulations.
+
+    The model consists of:
+    - An atmospheric profile.
+    - A Rayleigh extinction table.
+
+    The output data is provided in ECSV data format.
+
+    Raises
+    ------
+    CorruptedInputDataError
+        If the MACOBAC12 table does not contain exactly one row.
+    MissingInputDataError
+        If the requested atmospheric DAS data is not available.
+    """
+
+    name = Unicode("CreateAtmosphericModel")
+    description = "Create an atmospheric profile"
+    aliases = Dict(
+        {"macobac12-table-path": "CreateMolecularAtmosphericModel.macobac12_table_path"}
+    )
+
+    macobac12_table_path = Path(
+        allow_none=False,
+        directory_ok=False,
+        default_value="macobac.ecsv",
+        help="Path to the MACOBAC12 table.",
+    ).tag(config=True)
+
+    def setup(self):
+        """
+        Update configuration and set up the MeteoDataHandler.
+
+        Raises
+        ------
+        CorruptedInputDataError
+            If the MACOBAC12 table does not contain exactly one row.
+        """
+        super().setup()
+
+        self.macobac12_table = Table.read(self.macobac12_table_path)
+        if len(self.macobac12_table) != 1:
+            raise CorruptedInputDataError(
+                "The MACOBAC12 table should contain exactly one row."
+            )
+        self.contemporary_atmospheric_profile = None
+        self.contemporary_rayleigh_extinction_profile = None
+
+    def start(self):
+        """
+        Produce molecular atmopsheric model.
+
+        This method performs the following steps:
+        1. Retrieves necessary atmospheric data from the specified
+            data assimilation system (usually ECMWF).
+        2. Combines the retrieved data with the provided 12-MACOBAC
+            and stored reference atmospheric model
+            to create a contemporary atmospheric profile and a Rayleigh extinction table.
+
+        Raises
+        ------
+        MissingInputDataError
+            If there is an error retrieving the necessary atmospheric data.
+        """
+        observatory = Observatory.from_db(
+            self.database_configuration,
+            site=self.observatory["name"].upper(),
+            version=self.observatory["version"],
+        )
+
+        latitude, longitude = observatory.coordinates
+        dusk, dawn = observatory.get_astronomical_night(self._timestamp)
+        self.data_handler.create_request(
+            start=dusk, stop=dawn, latitude=latitude, longitude=longitude
+        )
+        data_status = self.data_handler.request_data()
+        if data_status == 0:
+            co2_concentration = self.macobac12_table["co2_concentration"][0]
+            molecular_profile = MolecularProfile(
+                f"{self.data_handler.data_path}/merged_file.ecsv",
+                stat_columns=self.DEFAULT_METEO_COLUMNS,
+            )
+            molecular_profile.get_data()
+            # compute the molecular density profile
+            # in order to select a reference atmospheric model
+            # for upper atmosphere
+            site = observatory.name
+            with CalibPipeDatabase(
+                **self.database_configuration,
+            ) as connection:
+                atmospheric_model_table = TableHandler.read_table_from_database(
+                    AtmosphericModelContainer,
+                    connection,
+                    condition=f"(c.current == True) & (c.name_Observatory == '{site}')",
+                )
+                reference_density_table = TableHandler.read_table_from_database(
+                    MolecularDensityContainer,
+                    connection,
+                    condition=f"c.version.in_({list(atmospheric_model_table['version'].data)})",
+                )
+            contemporary_mdp = molecular_profile.create_molecular_density_profile()
+            number_density_at_15km = contemporary_mdp[
+                contemporary_mdp["altitude"] == 15000 * u.m
+            ]["number density"].quantity[0]
+            _, _, version = min(
+                reference_density_table,
+                key=lambda x: abs(
+                    x["density"]
+                    - number_density_at_15km  # FIXME: MDP table should have "number density" column instead of "density"
+                ),
+            )
+            with CalibPipeDatabase(
+                **self.database_configuration,
+            ) as connection:
+                reference_profile_table = TableHandler.read_table_from_database(
+                    MolecularAtmosphericProfileContainer,
+                    connection,
+                    condition=f'c.version == "{version}"',
+                )
+                reference_profile_table.remove_columns(["version"])
+                reference_profile = Table(
+                    [
+                        np.squeeze(reference_profile_table[col])
+                        for col in reference_profile_table.columns
+                    ],
+                    names=reference_profile_table.columns,
+                )
+
+            self.contemporary_atmospheric_profile = (
+                molecular_profile.create_atmospheric_profile(
+                    co2_concentration=co2_concentration,
+                    reference_atmosphere=reference_profile,
+                )
+            )
+            self.contemporary_rayleigh_extinction_profile = (
+                molecular_profile.create_rayleigh_extinction_profile(
+                    co2_concentration=co2_concentration,
+                    reference_atmosphere=reference_profile,
+                )
+            )
+        else:
+            raise MissingInputDataError(
+                "Contemporary meteorological data is not available. "
+                "Please check the configuration and/or try again later."
+            )
+
+    def finish(self):
+        """Save the results and perform cleanup."""
+        self.contemporary_atmospheric_profile.write(
+            f"{self.output_path}/contemporary_atmospheric_profile.{self.output_format}",
+            format=f"{self.output_format}",
+        )
+        self.contemporary_rayleigh_extinction_profile.write(
+            f"{self.output_path}/contemporary_rayleigh_extinction_profile.{self.output_format}",
+            format=f"{self.output_format}",
+        )
+        self.log.info("Shutting down.")
+        self.data_handler.cleanup()
+
+
+def main():
+    """Run the tool."""
+    tool = CreateMolecularAtmosphericModel()
+    tool.run()

calibpipe/tools/muon_throughput_calculator.py

@@ -0,0 +1,219 @@
+"""Tool for calculating of optical throughput and storing of the results in the DB."""
+
+import numpy as np
+from astropy.table import Table, join
+from astropy.time import Time
+from calibpipe.telescope.throughput.containers import OpticalThoughtputContainer
+from ctapipe.core import QualityQuery, Tool, traits
+from ctapipe.core.traits import (
+    Bool,
+    CInt,
+    ComponentName,
+    List,
+    Path,
+    Set,
+    classes_with_traits,
+)
+from ctapipe.instrument import SubarrayDescription
+from ctapipe.io import read_table, write_table
+from ctapipe.io.hdf5dataformat import (
+    DL1_TEL_CALIBRATION_GROUP,
+    DL1_TEL_GROUP,
+    DL1_TEL_TRIGGER_TABLE,
+)
+from ctapipe.monitoring.aggregator import StatisticsAggregator
+
+
+class MuonQualityQuery(QualityQuery):
+    """Quality criteria for muon parameters."""
+
+    quality_criteria = List(
+        default_value=[
+            ("min impact parameter", "muonefficiency_impact >= 0.0"),
+            ("max impact parameter", "muonefficiency_impact <= 10.0"),
+            (
+                "muon efficiency parameters are not at limit",
+                "~muonefficiency_parameters_at_limit",
+            ),
+            ("throughput calculation is valid", "muonefficiency_is_valid"),
+        ],
+        allow_none=True,
+        help=QualityQuery.quality_criteria.help,
+    ).tag(config=True)
+
+
+class CalculateThroughputWithMuons(Tool):
+    """Perform throughput calibration using muons for each telescope allowed in the EventSource."""
+
+    name = traits.Unicode("ThroughputCalibration")
+    description = __doc__
+
+    aliases = {
+        ("i", "input"): "CalculateThroughputWithMuons.input_url",
+    }
+
+    classes = [MuonQualityQuery] + classes_with_traits(StatisticsAggregator)
+
+    input_url = Path(
+        help="CTAO HDF5 files for DL1 calibration (muons).",
+        allow_none=False,
+        exists=True,
+        directory_ok=False,
+        file_ok=True,
+    ).tag(config=True)
+
+    allowed_tels = Set(
+        trait=CInt(),
+        default_value=None,
+        allow_none=True,
+        help=(
+            "List of allowed telescope IDs, others will be ignored. If None, all "
+            "telescopes in the input stream will be included. Requires the "
+            "telescope IDs to match between the groups of the monitoring file."
+        ),
+    ).tag(config=True)
+
+    aggregator_type = ComponentName(
+        StatisticsAggregator,
+        default_value="SigmaClippingAggregator",
+        help="The aggregation strategy to use for throughput calculation.",
+    ).tag(config=True)
+
+    append = Bool(
+        default_value=False,
+        help="If the throughput table already exists in the file, append to it.",
+    ).tag(config=True)
+
+    # Set the method name for throughput calculation
+    METHOD = "Muon Rings"
+
+    def setup(self):
+        """Read from the .h5 file necessary info and save it for further processing."""
+        # Load the subarray description from the input file
+        self.subarray = SubarrayDescription.from_hdf(self.input_url)
+
+        # Select a new subarray if the allowed_tels configuration is used
+        if self.allowed_tels is not None:
+            self.subarray = self.subarray.select_subarray(self.allowed_tels)
+
+        # Initialize the quality query for muon selection
+        self.quality_query = MuonQualityQuery(parent=self)
+
+        # Initialize the aggregator based on configuration
+        self.aggregator = StatisticsAggregator.from_name(
+            self.aggregator_type,
+            parent=self,
+        )
+
+        self.throughput_containers = {}
+
+    def _process_tel(self, tel_id):
+        """Process muon data for a single telescope ID."""
+        muon_table = read_table(
+            self.input_url,
+            f"{DL1_TEL_GROUP}/muon/tel_{tel_id:03d}",
+        )
+
+        filtered_table = muon_table[self.quality_query.get_table_mask(muon_table)]
+
+        # Read trigger table to get time information
+        trigger_table = read_table(
+            self.input_url,
+            DL1_TEL_TRIGGER_TABLE,
+        )
+
+        # Join timing information from trigger table
+        filtered_table = join(
+            filtered_table,
+            trigger_table[["obs_id", "event_id", "time"]],
+            keys=["obs_id", "event_id"],
+            join_type="left",  # keeps all muon events, even if no trigger match
+        )
+
+        # Ensure table is sorted by time
+        filtered_table.sort("time")
+
+        # Add time_mono column for aggregator (reusing original table)
+        filtered_table["time_mono"] = filtered_table["time"]
+
+        # Run aggregator with chunk processing - will raise ValueError if insufficient data
+        chunk_stats = self.aggregator(
+            table=filtered_table,
+            col_name="muonefficiency_optical_efficiency",
+        )
+
+        # Convert aggregator results to throughput containers
+        containers = []
+        for i in range(len(chunk_stats)):
+            # Create container with all parameters at once
+            container = OpticalThoughtputContainer(
+                obs_id=filtered_table["obs_id"],
+                method=self.METHOD,
+                mean=chunk_stats["mean"][i],
+                median=chunk_stats["median"][i],
+                std=chunk_stats["std"][i],
+                sem=np.squeeze(
+                    chunk_stats["std"][i] / (chunk_stats["n_events"][i] ** 0.5)
+                ),
+                time_start=Time(
+                    chunk_stats["time_start"][i], format="mjd", scale="tai"
+                ),
+                time_end=Time(chunk_stats["time_end"][i], format="mjd", scale="tai"),
+                n_events=np.squeeze(chunk_stats["n_events"][i]),
+            )
+            containers.append(container)
+        return containers
+
+    def start(self):
+        """
+        Apply the cuts on the muon data and process in chunks.
+
+        Only the events that passed quality cuts provided by configuration are considered.
+        Only events for which intensity fit converged, and parameters were not at the limit are considered.
+        """
+        for tel_id in self.subarray.tel_ids:
+            try:
+                containers = self._process_tel(tel_id)
+                self.throughput_containers[tel_id] = containers
+            except ValueError as e:
+                self.log.warning("Skipping telescope %s: %s", tel_id, e)
+                self.throughput_containers[tel_id] = {}
+                continue
+
+    def finish(self):
+        """Write the chunk-based results to the output file using write_table."""
+        for tel_id in self.subarray.tel_ids:
+            containers_list = self.throughput_containers[tel_id]
+
+            # Convert containers to table data using as_dict()
+            table_data = []
+            for container in containers_list:
+                table_data.append(container.as_dict())
+
+            if not table_data:
+                self.log.info("No throughput data to write for telescope %s", tel_id)
+                continue
+            # Create astropy Table from container data
+            throughput_table = Table(table_data)
+
+            # Write table to HDF5 file
+            if self.append:
+                write_table(
+                    throughput_table,
+                    self.input_url,
+                    f"{DL1_TEL_CALIBRATION_GROUP}/optical_throughput/tel_{tel_id:03d}",
+                    append=True,
+                )
+            else:
+                write_table(
+                    throughput_table,
+                    self.input_url,
+                    f"{DL1_TEL_CALIBRATION_GROUP}/optical_throughput/tel_{tel_id:03d}",
+                    overwrite=True,
+                )
+
+
+def main():
+    """Run the app."""
+    tool = CalculateThroughputWithMuons()
+    tool.run()

calibpipe/tools/observatory_data_db_loader.py

@@ -0,0 +1,71 @@
+import sqlalchemy as sa  # noqa: D100
+
+# Internal imports
+from calibpipe.database.connections import CalibPipeDatabase
+from calibpipe.database.interfaces import TableHandler
+from calibpipe.utils.observatory import Observatory
+
+# CTA-related imports
+from ctapipe.core.traits import List, Unicode
+from traitlets.config import Config
+
+from .basic_tool_with_db import BasicToolWithDB
+
+
+class UploadObservatoryData(BasicToolWithDB):
+    """Upload observatory data to the calibpipe database."""
+
+    name = Unicode("Upload observatory data")
+    description = "Upload observatory data to the calibpipe database."
+
+    observatories = List(help="List of observatories configurations", minlen=1).tag(
+        config=True
+    )
+
+    classes = [
+        Observatory,
+    ]
+
+    def setup(self):
+        """Create Observatory objects from the configuration and register database configuration."""
+        super().setup()
+        self._observatories = [
+            Observatory(config=Config(key))
+            for key in self.config["UploadObservatoryData"]["observatories"]
+        ]
+
+    def start(self):
+        """Create the database tables and insert the data."""
+        containers = [
+            container
+            for observatory in self._observatories
+            for container in observatory.containers
+        ]
+        # Check if the tables exist, if not create them
+        with CalibPipeDatabase(
+            **self.database_configuration,
+        ) as connection:
+            for container in containers:
+                table, insertion = TableHandler.get_database_table_insertion(
+                    container,
+                )
+                if not sa.inspect(connection.engine).has_table(table.name):
+                    table.create(bind=connection.engine)
+        # Insert the data
+        with CalibPipeDatabase(
+            **self.database_configuration,
+        ) as connection:
+            for container in containers:
+                table, insertion = TableHandler.get_database_table_insertion(
+                    container,
+                )
+                TableHandler.insert_row_in_database(table, insertion, connection)
+
+    def finish(self):
+        """No finishing actions needed."""
+
+
+def main():
+    """Run the app."""
+    tool = UploadObservatoryData()
+    tool.run()

calibpipe/tools/reference_atmospheric_model_selector.py

@@ -0,0 +1,201 @@
+import astropy.units as u  # noqa: D100
+import numpy as np
+from astropy.table import Table
+from ctapipe.core.traits import (
+    Dict,
+    Unicode,
+)
+from molecularprofiles.molecularprofiles import MolecularProfile
+
+from ..atmosphere.atmosphere_containers import (
+    AtmosphericModelContainer,
+    MolecularAtmosphericProfileContainer,
+    MolecularDensityContainer,
+    RayleighExtinctionContainer,
+    SelectedAtmosphericModelContainer,
+)
+from ..core.exceptions import IntermittentError
+from ..database.connections import CalibPipeDatabase
+from ..database.interfaces import TableHandler
+from ..utils.observatory import Observatory, SeasonAlias
+from .atmospheric_base_tool import AtmosphericBaseTool
+
+
+class SelectMolecularAtmosphericModel(AtmosphericBaseTool):
+    """Select a reference molecular atmospheric model."""
+
+    name = Unicode("SelectMolecularAtmosphericModel")
+    description = (
+        "Select a reference Molecular Atmospheric Model "
+        "based on the molecular number density at 15km a.s.l. "
+        "computed from contemporary meteorological data."
+    )
+    aliases = Dict(
+        {
+            "timestamp": "SelectMolecularAtmosphericModel.timestamp",
+            "output_path": "SelectMolecularAtmosphericModel.output_path",
+        }
+    )
+
+    def setup(self):
+        """Parse configuration and setup the database connection and MeteoDataHandler."""
+        super().setup()
+        self.selected_model_container = None
+        self.failed_to_retrieve_meteo_data = True
+
+    def start(self):
+        """
+        Download meteorological data and select a reference atmospheric model.
+
+        This method performs the following operations:
+        1. Retrieves the observatory data from the database.
+        2. Calculates the astronomical night based on the observatory's coordinates and the provided timestamp.
+        3. Creates a data request for the calculated time frame and coordinates.
+        4. Attempts to fetch the meteorological data.
+        5a. If meteorological data is not available, select a reference model based on the date.
+        5b. If meteorological data is available, select a reference model based on the molecular number density at 15km a.s.l.
+        """
+        observatory = Observatory.from_db(
+            self.database_configuration,
+            site=self.observatory["name"].upper(),
+            version=self.observatory["version"],
+        )
+        site = observatory.name
+        with CalibPipeDatabase(
+            **self.database_configuration,
+        ) as connection:
+            atmospheric_model_table = TableHandler.read_table_from_database(
+                AtmosphericModelContainer,
+                connection,
+                condition=f"(c.current == True) & (c.name_Observatory == '{site}')",
+            )
+            reference_density_table = TableHandler.read_table_from_database(
+                MolecularDensityContainer,
+                connection,
+                condition=f"c.version.in_({list(atmospheric_model_table['version'].data)})",
+            )
+
+        latitude, longitude = observatory.coordinates
+        dusk, dawn = observatory.get_astronomical_night(self._timestamp)
+        self.data_handler.create_request(
+            start=dusk, stop=dawn, latitude=latitude, longitude=longitude
+        )
+        self.failed_to_retrieve_meteo_data = self.data_handler.request_data()
+        if self.failed_to_retrieve_meteo_data:
+            season = observatory.get_season_from_timestamp(self._timestamp)
+            version = -1
+        else:
+            molecular_profile = MolecularProfile(
+                f"{self.data_handler.data_path}/merged_file.ecsv",
+                stat_columns=self.DEFAULT_METEO_COLUMNS,
+            )
+            molecular_profile.get_data()
+            contemporary_mdp = molecular_profile.create_molecular_density_profile()
+            number_density_at_15km = contemporary_mdp[
+                contemporary_mdp["altitude"] == 15000 * u.m
+            ]["number density"].quantity[0]
+            season, _, version = min(
+                reference_density_table,
+                key=lambda x: abs(
+                    x["density"]
+                    - number_density_at_15km  # FIXME: MDP table should have "number density" column instead of "density"
+                ),
+            )
+
+        selected_season_table = atmospheric_model_table[
+            (atmospheric_model_table["season"] == season)
+        ]
+        if version == -1:
+            version = selected_season_table[(selected_season_table["current"])][
+                "version"
+            ][0]
+
+        self.selected_model_container = SelectedAtmosphericModelContainer(
+            date=self._timestamp.date(),
+            version=version,
+            season=SeasonAlias[season.upper()].value,
+            site=observatory.name,
+            provenance=self.meteo_data_handler,
+        )
+
+    def finish(self):
+        """Store the selected atmospheric model, and perform cleanup.
+
+        Raises
+        ------
+        IntermittentError
+            In case of missing meteorological data.
+        """
+        self.log.info(
+            "Selected atmospheric model container:\n%s", self.selected_model_container
+        )
+
+        with CalibPipeDatabase(
+            **self.database_configuration,
+        ) as connection:
+            table, insertion = TableHandler.get_database_table_insertion(
+                self.selected_model_container,
+            )
+            TableHandler.insert_row_in_database(table, insertion, connection)
+            selected_atmospheric_table = TableHandler.read_table_from_database(
+                MolecularAtmosphericProfileContainer,
+                connection,
+                condition=f"c.version == \"{self.selected_model_container['version']}\"",
+            )
+            selected_rayleigh_extinction_table = TableHandler.read_table_from_database(
+                RayleighExtinctionContainer,
+                connection,
+                condition=f"c.version == \"{self.selected_model_container['version']}\"",
+            )
+        selected_atmospheric_table.remove_column("version")
+        selected_atmospheric_profile = Table(
+            [
+                np.squeeze(selected_atmospheric_table[col])
+                for col in selected_atmospheric_table.columns
+            ],
+            names=selected_atmospheric_table.columns,
+        )
+        selected_atmospheric_profile.write(
+            f"{self.output_path}/selected_atmospheric_profile.{self.output_format}",
+            format=f"{self.output_format}",
+        )
+        selected_rayleigh_extinction_table.remove_column("version")
+
+        rayleigh_extinction_col_names = ["altitude_min", "altitude_max"]
+        rayleigh_extinction_col_names.extend(
+            [
+                f"{name:.1f}"
+                for name in selected_rayleigh_extinction_table["wavelength"].squeeze()
+            ]
+        )
+        data_array = np.hstack(
+            (
+                selected_rayleigh_extinction_table["altitude"].squeeze().to_value(u.km),
+                selected_rayleigh_extinction_table["AOD"].squeeze(),
+            )
+        )
+        selected_rayleigh_extinction_profile = Table(
+            data=data_array,
+            names=rayleigh_extinction_col_names,
+        )
+        selected_rayleigh_extinction_profile["altitude_min"] *= u.km
+        selected_rayleigh_extinction_profile["altitude_max"] *= u.km
+
+        selected_rayleigh_extinction_profile.write(
+            f"{self.output_path}/selected_rayleigh_extinction_profile.{self.output_format}",
+            format=f"{self.output_format}",
+        )
+
+        self.log.info("Shutting down.")
+        self.data_handler.cleanup()
+        if self.failed_to_retrieve_meteo_data:
+            raise IntermittentError(
+                f"Missing meteorological data from {self.meteo_data_handler}. "
+                "This is a known issue, the reference model was selected based on the date."
+            )
+
+
+def main():
+    """Run the app."""
+    tool = SelectMolecularAtmosphericModel()
+    tool.run()