climate-ref 0.5.0__py3-none-any.whl

climate_ref/migrations/env.py

@@ -0,0 +1,139 @@
+from alembic import context, op
+from loguru import logger
+from sqlalchemy import Connection, inspect
+
+from climate_ref.config import Config
+from climate_ref.database import Database
+from climate_ref.models import Base, MetricValue
+from climate_ref_core.logging import capture_logging
+from climate_ref_core.pycmec.controlled_vocabulary import CV
+
+# Setup logging
+capture_logging()
+logger.debug("Running alembic env")
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+ref_config: Config = config.attributes.get("ref_config") or Config.default()
+
+target_metadata = Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+# Custom migration functions that are run on every migration
+
+
+def _add_metric_value_columns(connection: Connection) -> None:
+    """
+    Add any missing columns in the current CV to the database
+
+    This must be run in online mode
+
+    Parameters
+    ----------
+    connection
+        Open connection to the database
+    """
+    metric_value_table = "metric_value"
+
+    inspector = inspect(connection)
+
+    # Check if table already exists
+    # Skip if it doesn't
+    tables = inspector.get_table_names()
+    if metric_value_table not in tables:
+        logger.warning(f"No table named {metric_value_table!r} found")
+        return
+
+    # Extract the current columns in the DB
+    existing_columns = [c["name"] for c in inspector.get_columns(metric_value_table)]
+
+    cv_file = ref_config.paths.dimensions_cv
+    cv = CV.load_from_file(cv_file)
+
+    for dimension in cv.dimensions:
+        if dimension.name not in existing_columns:
+            logger.info(f"Adding missing metric value dimension: {dimension.name!r}")
+            op.add_column(metric_value_table, MetricValue.build_dimension_column(dimension))
+
+
+def include_object(object_, name: str, type_, reflected, compare_to) -> bool:
+    """
+    Object-level check to include or exclude objects from the migration
+
+    Excludes columns that are marked with `skip_autogenerate` in the info dict
+
+    Based on  https://alembic.sqlalchemy.org/en/latest/autogenerate.html#omitting-based-on-object
+    """
+    if object_.info.get("skip_autogenerate", False):
+        return False
+    else:
+        return True
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_name="sqlite",
+        dialect_opts={"paramstyle": "named"},
+        render_as_batch=True,
+        include_object=include_object,
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+    """
+    connectable = config.attributes.get("connection", None)
+
+    if connectable is None:
+        db = Database.from_config(ref_config, run_migrations=False)
+        connectable = db._engine
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            render_as_batch=True,
+            include_object=include_object,
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+            # Set up the Operations context
+            # This is needed to alter the tables
+            with op.Operations.context(context.get_context()):  # type: ignore
+                _add_metric_value_columns(connection)
+
+
+if context.is_offline_mode():
+    logger.warning("Running in offline mode")
+    run_migrations_offline()
+else:
+    run_migrations_online()

climate_ref/migrations/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

climate_ref/migrations/versions/2025-05-02T1418_341a4aa2551e_regenerate.py

@@ -0,0 +1,292 @@
+"""regenerate
+
+Revision ID: 341a4aa2551e
+Revises:
+Create Date: 2025-05-02 14:18:54.273708
+
+"""
+
+from collections.abc import Sequence
+from typing import Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "341a4aa2551e"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "dataset",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("slug", sa.String(), nullable=False),
+        sa.Column(
+            "dataset_type",
+            sa.Enum("CMIP6", "CMIP7", "obs4MIPs", "PMPClimatology", name="sourcedatasettype"),
+            nullable=False,
+        ),
+        sa.Column("created_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_dataset")),
+        sa.UniqueConstraint("slug", name=op.f("uq_dataset_slug")),
+    )
+    op.create_table(
+        "provider",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("slug", sa.String(), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("version", sa.String(), nullable=False),
+        sa.Column("created_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_provider")),
+        sa.UniqueConstraint("slug", name=op.f("uq_provider_slug")),
+    )
+    op.create_table(
+        "cmip6_dataset",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("activity_id", sa.String(), nullable=False),
+        sa.Column("branch_method", sa.String(), nullable=True),
+        sa.Column("branch_time_in_child", sa.Float(), nullable=True),
+        sa.Column("branch_time_in_parent", sa.Float(), nullable=True),
+        sa.Column("experiment", sa.String(), nullable=False),
+        sa.Column("experiment_id", sa.String(), nullable=False),
+        sa.Column("frequency", sa.String(), nullable=False),
+        sa.Column("grid", sa.String(), nullable=False),
+        sa.Column("grid_label", sa.String(), nullable=False),
+        sa.Column("institution_id", sa.String(), nullable=False),
+        sa.Column("long_name", sa.String(), nullable=True),
+        sa.Column("member_id", sa.String(), nullable=False),
+        sa.Column("nominal_resolution", sa.String(), nullable=False),
+        sa.Column("parent_activity_id", sa.String(), nullable=True),
+        sa.Column("parent_experiment_id", sa.String(), nullable=True),
+        sa.Column("parent_source_id", sa.String(), nullable=True),
+        sa.Column("parent_time_units", sa.String(), nullable=True),
+        sa.Column("parent_variant_label", sa.String(), nullable=True),
+        sa.Column("realm", sa.String(), nullable=False),
+        sa.Column("product", sa.String(), nullable=False),
+        sa.Column("source_id", sa.String(), nullable=False),
+        sa.Column("standard_name", sa.String(), nullable=False),
+        sa.Column("source_type", sa.String(), nullable=False),
+        sa.Column("sub_experiment", sa.String(), nullable=False),
+        sa.Column("sub_experiment_id", sa.String(), nullable=False),
+        sa.Column("table_id", sa.String(), nullable=False),
+        sa.Column("units", sa.String(), nullable=False),
+        sa.Column("variable_id", sa.String(), nullable=False),
+        sa.Column("variant_label", sa.String(), nullable=False),
+        sa.Column("vertical_levels", sa.Integer(), nullable=True),
+        sa.Column("version", sa.String(), nullable=False),
+        sa.Column("instance_id", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(["id"], ["dataset.id"], name=op.f("fk_cmip6_dataset_id_dataset")),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_cmip6_dataset")),
+    )
+    op.create_table(
+        "dataset_file",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("dataset_id", sa.Integer(), nullable=False),
+        sa.Column("end_time", sa.DateTime(), nullable=True),
+        sa.Column("start_time", sa.DateTime(), nullable=True),
+        sa.Column("path", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["dataset_id"],
+            ["dataset.id"],
+            name=op.f("fk_dataset_file_dataset_id_dataset"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_dataset_file")),
+    )
+    op.create_table(
+        "diagnostic",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("slug", sa.String(), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("provider_id", sa.Integer(), nullable=False),
+        sa.Column("enabled", sa.Boolean(), nullable=False),
+        sa.Column("created_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["provider_id"], ["provider.id"], name=op.f("fk_diagnostic_provider_id_provider")
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_diagnostic")),
+        sa.UniqueConstraint("provider_id", "slug", name="diagnostic_ident"),
+        sa.UniqueConstraint("slug", name=op.f("uq_diagnostic_slug")),
+    )
+    op.create_table(
+        "obs4mips_dataset",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("activity_id", sa.String(), nullable=False),
+        sa.Column("frequency", sa.String(), nullable=False),
+        sa.Column("grid", sa.String(), nullable=False),
+        sa.Column("grid_label", sa.String(), nullable=False),
+        sa.Column("institution_id", sa.String(), nullable=False),
+        sa.Column("long_name", sa.String(), nullable=False),
+        sa.Column("nominal_resolution", sa.String(), nullable=False),
+        sa.Column("realm", sa.String(), nullable=False),
+        sa.Column("product", sa.String(), nullable=False),
+        sa.Column("source_id", sa.String(), nullable=False),
+        sa.Column("source_type", sa.String(), nullable=False),
+        sa.Column("units", sa.String(), nullable=False),
+        sa.Column("variable_id", sa.String(), nullable=False),
+        sa.Column("variant_label", sa.String(), nullable=False),
+        sa.Column("vertical_levels", sa.Integer(), nullable=False),
+        sa.Column("source_version_number", sa.String(), nullable=False),
+        sa.Column("instance_id", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(["id"], ["dataset.id"], name=op.f("fk_obs4mips_dataset_id_dataset")),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_obs4mips_dataset")),
+    )
+    op.create_table(
+        "pmp_climatology_dataset",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("activity_id", sa.String(), nullable=False),
+        sa.Column("frequency", sa.String(), nullable=False),
+        sa.Column("grid", sa.String(), nullable=False),
+        sa.Column("grid_label", sa.String(), nullable=False),
+        sa.Column("institution_id", sa.String(), nullable=False),
+        sa.Column("long_name", sa.String(), nullable=False),
+        sa.Column("nominal_resolution", sa.String(), nullable=False),
+        sa.Column("realm", sa.String(), nullable=False),
+        sa.Column("product", sa.String(), nullable=False),
+        sa.Column("source_id", sa.String(), nullable=False),
+        sa.Column("source_type", sa.String(), nullable=False),
+        sa.Column("units", sa.String(), nullable=False),
+        sa.Column("variable_id", sa.String(), nullable=False),
+        sa.Column("variant_label", sa.String(), nullable=False),
+        sa.Column("vertical_levels", sa.Integer(), nullable=False),
+        sa.Column("source_version_number", sa.String(), nullable=False),
+        sa.Column("instance_id", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(["id"], ["dataset.id"], name=op.f("fk_pmp_climatology_dataset_id_dataset")),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_pmp_climatology_dataset")),
+    )
+    op.create_table(
+        "execution_group",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("diagnostic_id", sa.Integer(), nullable=False),
+        sa.Column("key", sa.String(), nullable=False),
+        sa.Column("dirty", sa.Boolean(), nullable=False),
+        sa.Column("selectors", sa.JSON(), nullable=False),
+        sa.Column("created_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["diagnostic_id"], ["diagnostic.id"], name=op.f("fk_execution_group_diagnostic_id_diagnostic")
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_execution_group")),
+        sa.UniqueConstraint("diagnostic_id", "key", name="execution_ident"),
+    )
+    with op.batch_alter_table("execution_group", schema=None) as batch_op:
+        batch_op.create_index(batch_op.f("ix_execution_group_key"), ["key"], unique=False)
+
+    op.create_table(
+        "execution",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("output_fragment", sa.String(), nullable=False),
+        sa.Column("execution_group_id", sa.Integer(), nullable=False),
+        sa.Column("dataset_hash", sa.String(), nullable=False),
+        sa.Column("successful", sa.Boolean(), nullable=True),
+        sa.Column("path", sa.String(), nullable=True),
+        sa.Column("retracted", sa.Boolean(), nullable=False),
+        sa.Column("created_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False),
+        sa.ForeignKeyConstraint(["execution_group_id"], ["execution_group.id"], name="fk_execution_id"),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_execution")),
+    )
+    with op.batch_alter_table("execution", schema=None) as batch_op:
+        batch_op.create_index(batch_op.f("ix_execution_dataset_hash"), ["dataset_hash"], unique=False)
+
+    op.create_table(
+        "execution_dataset",
+        sa.Column("execution_id", sa.Integer(), nullable=True),
+        sa.Column("dataset_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["dataset_id"], ["dataset.id"], name=op.f("fk_execution_dataset_dataset_id_dataset")
+        ),
+        sa.ForeignKeyConstraint(
+            ["execution_id"], ["execution.id"], name=op.f("fk_execution_dataset_execution_id_execution")
+        ),
+    )
+    op.create_table(
+        "execution_output",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("execution_id", sa.Integer(), nullable=False),
+        sa.Column("output_type", sa.Enum("Plot", "Data", "HTML", name="resultoutputtype"), nullable=False),
+        sa.Column("filename", sa.String(), nullable=True),
+        sa.Column("short_name", sa.String(), nullable=True),
+        sa.Column("long_name", sa.String(), nullable=True),
+        sa.Column("description", sa.String(), nullable=True),
+        sa.Column("created_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["execution_id"], ["execution.id"], name=op.f("fk_execution_output_execution_id_execution")
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_execution_output")),
+    )
+    with op.batch_alter_table("execution_output", schema=None) as batch_op:
+        batch_op.create_index(batch_op.f("ix_execution_output_execution_id"), ["execution_id"], unique=False)
+        batch_op.create_index(batch_op.f("ix_execution_output_output_type"), ["output_type"], unique=False)
+
+    op.create_table(
+        "metric_value",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("execution_id", sa.Integer(), nullable=False),
+        sa.Column("value", sa.Float(), nullable=False),
+        sa.Column("attributes", sa.JSON(), nullable=False),
+        sa.Column("created_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), server_default=sa.text("(CURRENT_TIMESTAMP)"), nullable=False),
+        sa.Column("model", sa.Text(), nullable=True),
+        sa.Column("source_id", sa.Text(), nullable=True),
+        sa.Column("variant_label", sa.Text(), nullable=True),
+        sa.Column("metric", sa.Text(), nullable=True),
+        sa.Column("region", sa.Text(), nullable=True),
+        sa.Column("statistic", sa.Text(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["execution_id"], ["execution.id"], name=op.f("fk_metric_value_execution_id_execution")
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_metric_value")),
+    )
+    with op.batch_alter_table("metric_value", schema=None) as batch_op:
+        batch_op.create_index(batch_op.f("ix_metric_value_metric"), ["metric"], unique=False)
+        batch_op.create_index(batch_op.f("ix_metric_value_model"), ["model"], unique=False)
+        batch_op.create_index(batch_op.f("ix_metric_value_region"), ["region"], unique=False)
+        batch_op.create_index(batch_op.f("ix_metric_value_source_id"), ["source_id"], unique=False)
+        batch_op.create_index(batch_op.f("ix_metric_value_statistic"), ["statistic"], unique=False)
+        batch_op.create_index(batch_op.f("ix_metric_value_variant_label"), ["variant_label"], unique=False)
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("metric_value", schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f("ix_metric_value_variant_label"))
+        batch_op.drop_index(batch_op.f("ix_metric_value_statistic"))
+        batch_op.drop_index(batch_op.f("ix_metric_value_source_id"))
+        batch_op.drop_index(batch_op.f("ix_metric_value_region"))
+        batch_op.drop_index(batch_op.f("ix_metric_value_model"))
+        batch_op.drop_index(batch_op.f("ix_metric_value_metric"))
+
+    op.drop_table("metric_value")
+    with op.batch_alter_table("execution_output", schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f("ix_execution_output_output_type"))
+        batch_op.drop_index(batch_op.f("ix_execution_output_execution_id"))
+
+    op.drop_table("execution_output")
+    op.drop_table("execution_dataset")
+    with op.batch_alter_table("execution", schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f("ix_execution_dataset_hash"))
+
+    op.drop_table("execution")
+    with op.batch_alter_table("execution_group", schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f("ix_execution_group_key"))
+
+    op.drop_table("execution_group")
+    op.drop_table("pmp_climatology_dataset")
+    op.drop_table("obs4mips_dataset")
+    op.drop_table("diagnostic")
+    op.drop_table("dataset_file")
+    op.drop_table("cmip6_dataset")
+    op.drop_table("provider")
+    op.drop_table("dataset")
+    # ### end Alembic commands ###

climate_ref/models/__init__.py

@@ -0,0 +1,33 @@
+"""
+Declaration of the models used by the REF.
+
+These models are used to represent the data that is stored in the database.
+"""
+
+from typing import TypeVar
+
+from climate_ref.models.base import Base
+from climate_ref.models.dataset import Dataset
+from climate_ref.models.diagnostic import Diagnostic
+from climate_ref.models.execution import (
+    Execution,
+    ExecutionGroup,
+    ExecutionOutput,
+)
+from climate_ref.models.metric_value import MetricValue
+from climate_ref.models.provider import Provider
+
+Table = TypeVar("Table", bound=Base)
+
+
+__all__ = [
+    "Base",
+    "Dataset",
+    "Diagnostic",
+    "Execution",
+    "ExecutionGroup",
+    "ExecutionOutput",
+    "MetricValue",
+    "Provider",
+    "Table",
+]

climate_ref/models/base.py

@@ -0,0 +1,42 @@
+import datetime
+from typing import Any
+
+from sqlalchemy import JSON, MetaData, func
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+
+
+class Base(DeclarativeBase):
+    """
+    Base class for all models
+    """
+
+    type_annotation_map = {  # noqa: RUF012
+        dict[str, Any]: JSON,
+    }
+    metadata = MetaData(
+        # Enforce a common naming convention for constraints
+        # https://alembic.sqlalchemy.org/en/latest/naming.html
+        naming_convention={
+            "ix": "ix_%(column_0_label)s",
+            "uq": "uq_%(table_name)s_%(column_0_name)s",
+            "ck": "ck_%(table_name)s_`%(constraint_name)s`",
+            "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
+            "pk": "pk_%(table_name)s",
+        }
+    )
+
+
+class CreatedUpdatedMixin:
+    """
+    Mixin for models that have a created_at and updated_at fields
+    """
+
+    created_at: Mapped[datetime.datetime] = mapped_column(server_default=func.now())
+    """
+    When the dataset was added to the database
+    """
+
+    updated_at: Mapped[datetime.datetime] = mapped_column(server_default=func.now(), onupdate=func.now())
+    """
+    When the dataset was updated.
+    """