ctao-calibpipe 0.3.0rc2__py3-none-any.whl

calibpipe/tests/unittests/atmosphere/test_molecular_atmosphere.py

@@ -0,0 +1,327 @@
+# Set up logging
+from datetime import datetime, timezone
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+import yaml
+from astropy.table import QTable
+from astropy.time import Time
+from calibpipe.core.exceptions import IntermittentError, MissingInputDataError
+from calibpipe.tools.contemporary_mdp_producer import CreateMolecularDensityProfile
+from calibpipe.tools.macobac_calculator import CalculateMACOBAC
+from calibpipe.tools.molecular_atmospheric_model_producer import (
+    CreateMolecularAtmosphericModel,
+)
+from calibpipe.tools.reference_atmospheric_model_selector import (
+    SelectMolecularAtmosphericModel,
+)
+from ctapipe.core import run_tool
+from traitlets.config import Config
+
+
+@pytest.mark.verifies_usecase("UC-120-1.2")
+def test_calculate_macobac(tmp_path):
+    config = Config()
+    config.CalculateMACOBAC = Config()
+    config.CalculateMACOBAC.output_file = str(tmp_path / "macobac.ecsv")
+
+    with patch(
+        "calibpipe.tools.macobac_calculator.CO2DataHandler",
+        new_callable=MagicMock,
+    ) as mock_class:
+        mock_class.return_value.data_path = str(
+            Path(__file__).parent.parent.parent
+            / "data/atmosphere/molecular_atmosphere/"
+        )
+        tool = CalculateMACOBAC(config=config)
+        run_tool(tool)
+
+        output_file = Path(config.CalculateMACOBAC.output_file)
+        assert output_file.exists(), "Output file was not created."
+
+        result_table = QTable.read(output_file, format="ascii.ecsv")
+        expected_co2_concentration = 419.3
+        assert (
+            result_table["co2_concentration"][0].value
+            == pytest.approx(expected_co2_concentration, abs=0.1)
+        ), f"CO2 concentration does not match expected value ({result_table['co2_concentration'][0]} != {expected_co2_concentration})."
+        assert (
+            result_table["estimation_date"][0]
+            == Time(str(datetime.now(timezone.utc).date()), out_subfmt="date").iso
+        ), "Estimation date does not match expected value."
+
+
+@pytest.mark.db()
+@pytest.mark.verifies_usecase("UC-120-1.7")
+@pytest.mark.verifies_usecase("UC-120-1.8")
+def test_create_molecular_atmospheric_model(tmp_path):
+    config_path = (
+        Path(__file__).parent.parent.parent.parent.parent.parent
+        / "docs/source/user_guide/atmosphere/configuration/create_molecular_atmospheric_model.yaml"
+    )
+    with open(config_path) as file:
+        config = Config(yaml.load(file, Loader=yaml.SafeLoader))
+
+    config.CreateMolecularAtmosphericModel.output_path = str(tmp_path)
+    # Mock the necessary components
+    with patch(
+        # "calibpipe.tools.molecular_atmospheric_model_producer.MeteoDataHandler",
+        "calibpipe.tools.atmospheric_base_tool.MeteoDataHandler",
+        new_callable=MagicMock,
+    ) as mock_meteo_handler_class:
+        tool = CreateMolecularAtmosphericModel(config=config)
+        # Test missing input data
+        with pytest.raises(MissingInputDataError):
+            run_tool(
+                tool,
+                argv=[
+                    "-c",
+                    str(
+                        Path(__file__).parent.parent.parent.parent.parent.parent
+                        / "docs/source/user_guide/utils/configuration/db_config.yaml"
+                    ),
+                    "--macobac12-table-path",
+                    str(
+                        Path(__file__).parent.parent.parent
+                        / "data/atmosphere/molecular_atmosphere/macobac.ecsv"
+                    ),
+                ],
+                raises=True,
+            )
+
+        # Now set up the mock data handler
+        mock_meteo_handler = MagicMock()
+        mock_meteo_handler_class.from_name.return_value = mock_meteo_handler
+        mock_meteo_handler.data_path = str(
+            Path(__file__).parent.parent.parent
+            / "data/atmosphere/molecular_atmosphere/"
+        )
+        mock_meteo_handler.request_data.return_value = 0
+
+        run_tool(
+            tool,
+            argv=[
+                "-c",
+                str(
+                    Path(__file__).parent.parent.parent.parent.parent.parent
+                    / "docs/source/user_guide/utils/configuration/db_config.yaml"
+                ),
+                "--macobac12-table-path",
+                str(
+                    Path(__file__).parent.parent.parent
+                    / "data/atmosphere/molecular_atmosphere/macobac.ecsv"
+                ),
+            ],
+            raises=True,
+        )
+
+        # Check if the output files were created
+        output_profile = (
+            Path(config.CreateMolecularAtmosphericModel.output_path)
+            / "contemporary_atmospheric_profile.ascii.ecsv"
+        )
+        output_extinction = (
+            Path(config.CreateMolecularAtmosphericModel.output_path)
+            / "contemporary_rayleigh_extinction_profile.ascii.ecsv"
+        )
+        assert (
+            output_profile.exists()
+        ), "Contemporary atmospheric profile file was not created."
+        assert (
+            output_extinction.exists()
+        ), "Contemporary Rayleigh extinction profile file was not created."
+
+        # Read and validate the output files
+        profile_table = QTable.read(output_profile, format="ascii.ecsv")
+        extinction_table = QTable.read(output_extinction, format="ascii.ecsv")
+
+        expected_profile_columns = [
+            "altitude",
+            "atmospheric_density",
+            "atmospheric_thickness",
+            "refractive_index_m_1",
+            "temperature",
+            "pressure",
+            "partial_water_pressure",
+        ]
+
+        for column in expected_profile_columns:
+            assert (
+                column in profile_table.colnames
+            ), f"{column} column missing in profile table."
+
+        assert (
+            "altitude_max" in extinction_table.colnames
+        ), "Altitude_max column missing in extinction table."
+        assert (
+            "altitude_min" in extinction_table.colnames
+        ), "Altitude_min column missing in extinction table."
+
+
+@pytest.mark.db()
+@pytest.mark.verifies_usecase("UC-120-1.6")
+def test_create_molecular_density_profile(tmp_path):
+    config_path = (
+        Path(__file__).parent.parent.parent.parent.parent.parent
+        / "docs/source/user_guide/atmosphere/configuration/create_molecular_density_profile.yaml"
+    )
+    with open(config_path) as file:
+        config = Config(yaml.load(file, Loader=yaml.SafeLoader))
+
+    config.CreateMolecularDensityProfile.output_path = str(tmp_path)
+    # Mock the necessary components
+    with patch(
+        "calibpipe.tools.atmospheric_base_tool.MeteoDataHandler",
+        new_callable=MagicMock,
+    ) as mock_meteo_handler_class:
+        tool = CreateMolecularDensityProfile(config=config)
+        # Test missing input data
+        with pytest.raises(MissingInputDataError):
+            run_tool(
+                tool,
+                argv=[
+                    "-c",
+                    str(
+                        Path(__file__).parent.parent.parent.parent.parent.parent
+                        / "docs/source/user_guide/utils/configuration/db_config.yaml"
+                    ),
+                ],
+                raises=True,
+            )
+
+        # Now set up the mock data handler
+        mock_meteo_handler = MagicMock()
+        mock_meteo_handler_class.from_name.return_value = mock_meteo_handler
+        mock_meteo_handler.data_path = str(
+            Path(__file__).parent.parent.parent
+            / "data/atmosphere/molecular_atmosphere/"
+        )
+        mock_meteo_handler.request_data.return_value = 0
+
+        run_tool(
+            tool,
+            argv=[
+                "-c",
+                str(
+                    Path(__file__).parent.parent.parent.parent.parent.parent
+                    / "docs/source/user_guide/utils/configuration/db_config.yaml"
+                ),
+            ],
+            raises=True,
+        )
+
+        # Check if the output file was created
+        output_file = (
+            Path(config.CreateMolecularDensityProfile.output_path)
+            / "contemporary_molecular_density_profile.ascii.ecsv"
+        )
+        assert (
+            output_file.exists()
+        ), "Contemporary molecular density profile file was not created."
+
+        # Read and validate the output file
+        mdp_table = QTable.read(output_file, format="ascii.ecsv")
+
+        expected_columns = [
+            "altitude",
+            "number density",
+        ]
+
+        for column in expected_columns:
+            assert (
+                column in mdp_table.colnames
+            ), f"{column} column missing in molecular density profile table."
+
+
+@pytest.mark.db()
+@pytest.mark.verifies_usecase("UC-120-1.3")
+def test_select_molecular_atmospheric_model(tmp_path):
+    config_path = (
+        Path(__file__).parent.parent.parent.parent.parent.parent
+        / "docs/source/user_guide/atmosphere/configuration/select_reference_atmospheric_model.yaml"
+    )
+    with open(config_path) as file:
+        config = Config(yaml.load(file, Loader=yaml.SafeLoader))
+
+    config.SelectMolecularAtmosphericModel.output_path = str(tmp_path)
+    # Mock the necessary components
+    with patch(
+        "calibpipe.tools.atmospheric_base_tool.MeteoDataHandler",
+        new_callable=MagicMock,
+    ) as mock_meteo_handler_class:
+        tool = SelectMolecularAtmosphericModel(config=config)
+
+        # Mock database and data handler behavior
+        mock_meteo_handler = MagicMock()
+        mock_meteo_handler_class.from_name.return_value = mock_meteo_handler
+        mock_meteo_handler.data_path = str(
+            Path(__file__).parent.parent.parent
+            / "data/atmosphere/molecular_atmosphere/"
+        )
+
+        # Test missing input data
+        mock_meteo_handler.request_data.return_value = 1
+
+        with pytest.raises(IntermittentError):
+            run_tool(
+                tool,
+                argv=[
+                    "-c",
+                    str(
+                        Path(__file__).parent.parent.parent.parent.parent.parent
+                        / "docs/source/user_guide/utils/configuration/db_config.yaml"
+                    ),
+                ],
+                raises=True,
+            )
+
+        # Check if the output files were created
+        output_profile = (
+            Path(config.SelectMolecularAtmosphericModel.output_path)
+            / "selected_atmospheric_profile.ascii.ecsv"
+        )
+        output_extinction = (
+            Path(config.SelectMolecularAtmosphericModel.output_path)
+            / "selected_rayleigh_extinction_profile.ascii.ecsv"
+        )
+        assert (
+            output_profile.exists()
+        ), "Selected atmospheric profile file was not created."
+        assert (
+            output_extinction.exists()
+        ), "Selected Rayleigh extinction profile file was not created."
+
+        # Read and validate the output files
+        profile_table = QTable.read(output_profile, format="ascii.ecsv")
+        extinction_table = QTable.read(output_extinction, format="ascii.ecsv")
+
+        expected_profile_columns = [
+            "altitude",
+            "atmospheric_density",
+            "atmospheric_thickness",
+            "refractive_index_m_1",
+            "temperature",
+            "pressure",
+            "partial_water_pressure",
+        ]
+
+        for column in expected_profile_columns:
+            assert (
+                column in profile_table.colnames
+            ), f"{column} column missing in profile table."
+
+        expected_extinction_columns = ["altitude_min", "altitude_max"]
+        expected_extinction_columns.extend([f"{i}.0 nm" for i in range(200, 1000)])
+
+        for column in expected_extinction_columns:
+            assert (
+                column in extinction_table.colnames
+            ), f"{column} column missing in extinction table."
+            if "altitude" in column:
+                assert (
+                    extinction_table[column].unit == "km"
+                ), f"Column {column} unit does not match expected unit."
+            assert (
+                extinction_table[column].ndim == 1
+            ), f"Column {column} does not have the expected number of dimensions."

calibpipe/tests/unittests/database/test_table_handler.py

@@ -0,0 +1,163 @@
+# Import the necessary modules and classes for testing
+import datetime
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import astropy.units as u
+import pytest
+import yaml
+from calibpipe.core.common_metadata_containers import (
+    ContactReferenceMetadataContainer,
+    ProductReferenceMetadataContainer,
+    ReferenceMetadataContainer,
+)
+from calibpipe.database.adapter import Adapter
+from calibpipe.database.connections import CalibPipeDatabase
+from calibpipe.database.interfaces import TableHandler
+from calibpipe.telescope.throughput.containers import OpticalThoughtputContainer
+from calibpipe.utils.observatory import (
+    Observatory,
+)
+from traitlets.config import Config
+
+
+@pytest.fixture()
+def mock_connection():
+    """
+    Fixture to create a mock database connection.
+    """
+    connection = MagicMock(spec=CalibPipeDatabase)
+    return connection
+
+
+# Fixture to provide a database connection
+@pytest.fixture()
+def test_config():
+    # Setup and connect to the test database
+    config_path = Path(__file__).parent.joinpath(
+        "../../../../../docs/source/user_guide/utils/configuration/"
+    )
+    with open(config_path.joinpath("upload_observatory_data_db.yaml")) as yaml_file:
+        config_data = yaml.safe_load(yaml_file)
+    config_data = config_data["UploadObservatoryData"]
+
+    with open(config_path.joinpath("db_config.yaml")) as yaml_file:
+        config_data |= yaml.safe_load(yaml_file)
+    return config_data
+
+
+@pytest.fixture()
+def test_container(test_config):
+    return Observatory(config=Config(test_config["observatories"][0])).containers[0]
+
+
+# Test cases for TableHandler class and other functions in the module
+class TestTableHandler:
+    # Test get_database_table_insertion method
+    @pytest.mark.db()
+    def test_get_database_table_insertion(self, test_container):
+        # Prepare a mock container and call the method
+        table, kwargs = TableHandler.get_database_table_insertion(test_container)
+
+        # Assert that the table and kwargs are not None
+        assert table is not None
+        assert kwargs is not None
+
+    # Test read_table_from_database method
+    @pytest.mark.db()
+    def test_read_table_from_database(self, test_container, test_config):
+        TableHandler.prepare_db_tables(
+            [
+                test_container,
+            ],
+            test_config["database_configuration"],
+        )
+        condition = "c.elevation == 3000"
+        with CalibPipeDatabase(**test_config["database_configuration"]) as connection:
+            qtable = TableHandler.read_table_from_database(
+                type(test_container), connection, condition
+            )
+
+        # Assert that qtable is not None and has the expected columns
+        assert qtable is not None
+        assert "elevation" in qtable.colnames
+        assert qtable["elevation"].unit == u.m
+
+
+def test_upload_data(mock_connection):
+    """
+    Test the upload_data function to ensure it correctly uploads data and metadata.
+    """
+    # Use OpticalThoughtputContainer as the data container
+    data = OpticalThoughtputContainer(
+        mean=0.95,
+        median=0.90,
+        std=0.02,
+        sem=0.001,
+        method="muon analysis",
+        time_start=datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc),
+        time_end=datetime.datetime(2025, 12, 31, tzinfo=datetime.timezone.utc),
+        obs_id=12345,
+    )
+
+    # Mock metadata containers
+    reference_metadata = ReferenceMetadataContainer(
+        version_atmospheric_model="1.0",
+        version=1,
+        ID_optical_throughput=None,  # Will be set during the upload
+    )
+    product_metadata = ProductReferenceMetadataContainer(
+        description="Test product",
+        creation_time="2025-04-08T12:00:00Z",
+        product_id="12345",
+    )
+    contact_metadata = ContactReferenceMetadataContainer(
+        organization="Test Organization",
+        name="Test User",
+        email="test@example.com",
+    )
+
+    # Combine metadata into a list
+    metadata = [reference_metadata, product_metadata, contact_metadata]
+
+    # Mock database behavior
+    mock_connection.execute.return_value.fetchone.return_value = MagicMock(
+        _asdict=lambda: {"ID": 1}
+    )
+
+    # Patch the insert_row_in_database method
+    with patch(
+        "calibpipe.database.interfaces.table_handler.TableHandler.insert_row_in_database",
+    ) as mock_insert:
+        # Call the upload_data function
+        TableHandler.upload_data(data, metadata, mock_connection)
+
+        # Assertions to verify correct behavior
+        # Verify that the main data was inserted
+        data_table, data_kwargs = Adapter.to_postgres(data)
+
+        calls = mock_insert.call_args_list
+
+        # Check that a call was made with the expected table and connection
+        assert any(
+            call_args[0][0] == data_table
+            and call_args[0][2] == mock_connection
+            and call_args[0][1]["mean"] == 0.95
+            for call_args in calls
+        ), "Expected call to insert_row_in_database not found."
+
+        ref_table, ref_kwargs = Adapter.to_postgres(reference_metadata)
+
+        assert any(
+            call_args[0][0] == ref_table
+            and call_args[0][2] == mock_connection
+            and call_args[0][1]["version_atmospheric_model"] == "1.0"
+            for call_args in calls
+        ), "Expected call to insert_row_in_database not found."
+
+        for container in metadata[1:]:
+            meta_table, meta_kwargs = Adapter.to_postgres(container)
+            assert any(
+                call_args[0][0] == meta_table and call_args[0][2] == mock_connection
+                for call_args in calls
+            ), "Expected call to insert_row_in_database not found."

calibpipe/tests/unittests/database/test_types.py

@@ -0,0 +1,38 @@
+"""Test sqlalchemy types."""
+
+from calibpipe.database.interfaces.types import (
+    BigInteger,
+    Boolean,
+    ColumnType,
+    Date,
+    DateTime,
+    Double,
+    Float,
+    Integer,
+    NDArray,
+    Numeric,
+    SmallInteger,
+    String,
+    Time,
+)
+from sqlalchemy.sql.type_api import TypeEngine
+
+
+def test_types():
+    """Test that types are instance of TypeEngine."""
+    for type_ in [
+        Boolean,
+        SmallInteger,
+        Integer,
+        BigInteger,
+        Float,
+        Double,
+        Numeric,
+        String,
+        Date,
+        DateTime,
+        Time,
+        ColumnType,
+        NDArray,
+    ]:
+        assert issubclass(type_, TypeEngine)