climate-ref 0.6.6__py3-none-any.whl → 0.8.0__py3-none-any.whl

climate_ref/cli/providers.py

@@ -7,14 +7,12 @@ from typing import Annotated
 import pandas as pd
 import typer
 from loguru import logger
-from rich.console import Console
 
 from climate_ref.cli._utils import pretty_print_df
 from climate_ref.provider_registry import ProviderRegistry
 from climate_ref_core.providers import CondaDiagnosticProvider, DiagnosticProvider
 
 app = typer.Typer(help=__doc__)
-console = Console()
 
 
 @app.command(name="list")
@@ -24,6 +22,7 @@ def list_(ctx: typer.Context) -> None:
     """
     config = ctx.obj.config
     db = ctx.obj.database
+    console = ctx.obj.console
     provider_registry = ProviderRegistry.build_from_config(config, db)
 
     def get_env(provider: DiagnosticProvider) -> str:

climate_ref/config.py

@@ -14,11 +14,14 @@ which always take precedence over any other configuration values.
 # `esgpull` configuration management system with some of the extra complexity removed.
 # https://github.com/ESGF/esgf-download/blob/main/esgpull/config.py
 
+import datetime
 import importlib.resources
 import os
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Literal
 
+import platformdirs
+import requests
 import tomlkit
 from attr import Factory
 from attrs import define, field
@@ -334,6 +337,46 @@ def _load_config(config_file: str | Path, doc: dict[str, Any]) -> "Config":
     return _converter_defaults_relaxed.structure(doc, Config)
 
 
+DEFAULT_IGNORE_DATASETS_MAX_AGE = datetime.timedelta(hours=6)
+DEFAULT_IGNORE_DATASETS_URL = (
+    "https://raw.githubusercontent.com/Climate-REF/climate-ref/refs/heads/main/default_ignore_datasets.yaml"
+)
+
+
+def _get_default_ignore_datasets_file() -> Path:
+    """
+    Get the path to the ignore datasets file
+    """
+    cache_dir = platformdirs.user_cache_path("climate_ref")
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    ignore_datasets_file = cache_dir / "default_ignore_datasets.yaml"
+
+    download = True
+    if ignore_datasets_file.exists():
+        # Only update if the ignore datasets file is older than `DEFAULT_IGNORE_DATASETS_MAX_AGE`.
+        modification_time = datetime.datetime.fromtimestamp(ignore_datasets_file.stat().st_mtime)
+        age = datetime.datetime.now() - modification_time
+        if age < DEFAULT_IGNORE_DATASETS_MAX_AGE:
+            download = False
+
+    if download:
+        logger.info(
+            f"Downloading default ignore datasets file from {DEFAULT_IGNORE_DATASETS_URL} "
+            f"to {ignore_datasets_file}"
+        )
+        response = requests.get(DEFAULT_IGNORE_DATASETS_URL, timeout=120)
+        try:
+            response.raise_for_status()
+        except requests.RequestException as exc:
+            logger.warning(f"Failed to download default ignore datasets file: {exc}")
+            ignore_datasets_file.touch(exist_ok=True)
+        else:
+            with ignore_datasets_file.open(mode="wb") as file:
+                file.write(response.content)
+
+    return ignore_datasets_file
+
+
 @define(auto_attribs=True)
 class Config:
     """
@@ -364,10 +407,30 @@ class Config:
     - `complete`: Use the complete parser, which parses the dataset based on all available metadata.
     """
 
-    paths: PathConfig = Factory(PathConfig)  # noqa
-    db: DbConfig = Factory(DbConfig)  # noqa
-    executor: ExecutorConfig = Factory(ExecutorConfig)  # noqa
-    diagnostic_providers: list[DiagnosticProviderConfig] = Factory(default_providers)  # noqa
+    ignore_datasets_file: Path = field(factory=_get_default_ignore_datasets_file)
+    """
+    Path to the file containing the ignore datasets
+
+    This file is a YAML file that contains a list of facets to ignore per diagnostic.
+
+    The format is:
+    ```yaml
+    provider:
+      diagnostic:
+        source_type:
+          - facet: value
+          - another_facet: [another_value1, another_value2]
+    ```
+
+    If this is not specified, a default ignore datasets file will be used.
+    The default file is downloaded from the Climate-REF GitHub repository
+    if it does not exist or is older than 6 hours.
+    """
+
+    paths: PathConfig = Factory(PathConfig)
+    db: DbConfig = Factory(DbConfig)
+    executor: ExecutorConfig = Factory(ExecutorConfig)
+    diagnostic_providers: list[DiagnosticProviderConfig] = Factory(default_providers)  # noqa: RUF009, RUF100
     _raw: TOMLDocument | None = field(init=False, default=None, repr=False)
     _config_file: Path | None = field(init=False, default=None, repr=False)
 

climate_ref/database.py

@@ -8,6 +8,7 @@ The `Database` class is the main entry point for interacting with the database.
 It provides a session object that can be used to interact with the database and run queries.
 """
 
+import enum
 import importlib.resources
 import shutil
 from datetime import datetime
@@ -23,6 +24,7 @@ from loguru import logger
 from sqlalchemy.orm import Session
 
 from climate_ref.models import MetricValue, Table
+from climate_ref.models.execution import ExecutionOutput
 from climate_ref_core.pycmec.controlled_vocabulary import CV
 
 if TYPE_CHECKING:
@@ -135,6 +137,16 @@ def validate_database_url(database_url: str) -> str:
     return database_url
 
 
+class ModelState(enum.Enum):
+    """
+    State of a model instance
+    """
+
+    CREATED = "created"
+    UPDATED = "updated"
+    DELETED = "deleted"
+
+
 class Database:
     """
     Manage the database connection and migrations
@@ -234,11 +246,57 @@ class Database:
         # This will add new columns to the db if the CVs have changed
         MetricValue.register_cv_dimensions(cv)
 
+        # Register the CV dimensions with the ExecutionOutput model
+        # This enables dimension-based filtering of outputs
+        ExecutionOutput.register_cv_dimensions(cv)
+
         return db
 
+    def update_or_create(
+        self, model: type[Table], defaults: dict[str, Any] | None = None, **kwargs: Any
+    ) -> tuple[Table, ModelState | None]:
+        """
+        Update an existing instance or create a new one
+
+        This doesn't commit the transaction,
+        so you will need to call `session.commit()` after this method
+        or use a transaction context manager.
+
+        Parameters
+        ----------
+        model
+            The model to update or create
+        defaults
+            Default values to use when creating a new instance, or values to update on existing instance
+        kwargs
+            The filter parameters to use when querying for an instance
+
+        Returns
+        -------
+        :
+            A tuple containing the instance and a state enum indicating if the instance was created or updated
+        """
+        instance = self.session.query(model).filter_by(**kwargs).first()
+        state: ModelState | None = None
+        if instance:
+            # Update existing instance with defaults
+            if defaults:
+                for key, value in defaults.items():
+                    if getattr(instance, key) != value:
+                        logger.debug(f"Updating {model.__name__} {key} to {value}")
+                        setattr(instance, key, value)
+                        state = ModelState.UPDATED
+            return instance, state
+        else:
+            # Create new instance
+            params = {**kwargs, **(defaults or {})}
+            instance = model(**params)
+            self.session.add(instance)
+            return instance, ModelState.CREATED
+
     def get_or_create(
         self, model: type[Table], defaults: dict[str, Any] | None = None, **kwargs: Any
-    ) -> tuple[Table, bool]:
+    ) -> tuple[Table, ModelState | None]:
         """
         Get or create an instance of a model
 
@@ -258,13 +316,13 @@ class Database:
         Returns
         -------
         :
-            A tuple containing the instance and a boolean indicating if the instance was created
+            A tuple containing the instance and enum indicating if the instance was created
         """
         instance = self.session.query(model).filter_by(**kwargs).first()
         if instance:
-            return instance, False
+            return instance, None
         else:
             params = {**kwargs, **(defaults or {})}
             instance = model(**params)
             self.session.add(instance)
-            return instance, True
+            return instance, ModelState.CREATED

climate_ref/dataset_registry/obs4ref_reference.txt

@@ -5,15 +5,6 @@ obs4REF/ColumbiaU/WECANN-1-0/mon/hfls/gn/20250516/hfls_mon_WECANN-1-0_REF_gn_200
 obs4REF/ColumbiaU/WECANN-1-0/mon/hfss/gn/20250516/hfss_mon_WECANN-1-0_REF_gn_200701-201512.nc md5:b7a911e0fc164d07d3ab42a86d09b18b
 obs4REF/ECMWF/ERA-20C/mon/psl/gn/v20210727/psl_mon_ERA-20C_PCMDI_gn_190001-201012.nc md5:c100cf25d5681c375cd6c1ee60b678ba
 obs4REF/ECMWF/ERA-20C/mon/ts/gn/v20210727/ts_mon_ERA-20C_PCMDI_gn_190001-201012.nc md5:9ed8dfbb805ed4caa282ed70f873a3a0
-obs4REF/ECMWF/ERA-5/mon/ta/gn/v20250220/ta_mon_ERA-5_PCMDI_gn_200701-200712.nc md5:695633a2b401cfb66c8addbf58073dbc
-obs4REF/ECMWF/ERA-5/mon/ta/gn/v20250220/ta_mon_ERA-5_PCMDI_gn_200801-200812.nc md5:404f1e1f111859be06c00bcb8d740ff2
-obs4REF/ECMWF/ERA-5/mon/ta/gn/v20250220/ta_mon_ERA-5_PCMDI_gn_200901-200912.nc md5:a1bb8584d60cdd71154c01a692fa1fb4
-obs4REF/ECMWF/ERA-5/mon/ta/gn/v20250220/ta_mon_ERA-5_PCMDI_gn_201001-201012.nc md5:b78016a3c61d99dc0fd29563aa344ca1
-obs4REF/ECMWF/ERA-5/mon/ta/gn/v20250220/ta_mon_ERA-5_PCMDI_gn_201101-201112.nc md5:d64c231a7f798a255997ffe196613ea1
-obs4REF/ECMWF/ERA-5/mon/ta/gn/v20250220/ta_mon_ERA-5_PCMDI_gn_201201-201212.nc md5:7d90ce60b872dc4f044b9b0101114983
-obs4REF/ECMWF/ERA-5/mon/ta/gn/v20250220/ta_mon_ERA-5_PCMDI_gn_201301-201312.nc md5:2fc032707cb8a31ac60fa4abe9efe183
-obs4REF/ECMWF/ERA-5/mon/ta/gn/v20250220/ta_mon_ERA-5_PCMDI_gn_201401-201412.nc md5:6022d17e11df7818f5b0429d6e401d17
-obs4REF/ECMWF/ERA-5/mon/ta/gn/v20250220/ta_mon_ERA-5_PCMDI_gn_201501-201512.nc md5:c68fdabf6eeb4813befceace089c9494
 obs4REF/ECMWF/ERA-INT/mon/hfls/gn/v20210727/hfls_mon_ERA-INT_PCMDI_gn_197901-201903.nc md5:1ae4587143f05ee81432b3d9960aab63
 obs4REF/ECMWF/ERA-INT/mon/hfss/gn/v20210727/hfss_mon_ERA-INT_PCMDI_gn_197901-201903.nc md5:261f02b8cbce18486548882a11f9aa34
 obs4REF/ECMWF/ERA-INT/mon/hur/gn/v20210727/hur_mon_ERA-INT_PCMDI_gn_198901-201001.nc md5:56fcd2df8ed2879f18b5e8c78134a148

climate_ref/dataset_registry/sample_data.txt

@@ -68,16 +68,16 @@ CMIP6/CMIP/MPI-M/MPI-ESM1-2-LR/esm-piControl/r1i1p1f1/Amon/tas/gn/v20190815/tas_
 CMIP6/CMIP/MPI-M/MPI-ESM1-2-LR/esm-piControl/r1i1p1f1/Amon/tas/gn/v20190815/tas_Amon_MPI-ESM1-2-LR_esm-piControl_r1i1p1f1_gn_189001-190912.nc 5ce6e74fb80748e34a567b2895f029131c5980a292c744fbbf555c2235afe77f
 CMIP6/CMIP/MPI-M/MPI-ESM1-2-LR/esm-piControl/r1i1p1f1/Amon/tas/gn/v20190815/tas_Amon_MPI-ESM1-2-LR_esm-piControl_r1i1p1f1_gn_191001-191512.nc f4a83f01af6563a63f43e4497ba0ea6e90297fb076fbcc8d63ac0105e6450ab5
 CMIP6/CMIP/MPI-M/MPI-ESM1-2-LR/esm-piControl/r1i1p1f1/fx/areacella/gn/v20190815/areacella_fx_MPI-ESM1-2-LR_esm-piControl_r1i1p1f1_gn.nc b67f7d92ee13d5f0fabc5397e8ba5743f11cb062fd2f761e42ae5ac8438e69a4
-CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/cli/gn/v20190308/cli_Amon_CESM2_historical_r1i1p1f1_gn_200701-201412.nc 579cdab588f2bfdc501fb296af15b5ff578bc0b05c65a5ed15848cdf96f4c5bd
-CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/clivi/gn/v20190308/clivi_Amon_CESM2_historical_r1i1p1f1_gn_200701-201412.nc 84cffb47d106ead933f174da8dd1fdff55c7672b28204b012adbb73eb0b59d8f
-CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/clt/gn/v20190308/clt_Amon_CESM2_historical_r1i1p1f1_gn_200701-201412.nc ba35c7ce1b221626cbb6363648e2de10a4bb403e214d6933ce650ed895c3f29e
-CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/clwvi/gn/v20190308/clwvi_Amon_CESM2_historical_r1i1p1f1_gn_200701-201412.nc bd15aff61ce2c9e2a17d1bc417a6ca0103ae3799d45f6a688d1ac1e994f85155
-CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/pr/gn/v20190401/pr_Amon_CESM2_historical_r1i1p1f1_gn_200701-201412.nc b6aae96e7bc02b20dbba58c43f6976700e9147c7adb6f72c123cde99d250bc74
-CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/rlut/gn/v20190308/rlut_Amon_CESM2_historical_r1i1p1f1_gn_200701-201412.nc af072cf5e065b18207f140045f1a260d03ea85763319e9cf41ace71394d55478
-CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/rlutcs/gn/v20190308/rlutcs_Amon_CESM2_historical_r1i1p1f1_gn_200701-201412.nc 469b9fcab4d7fb09426f2c5c67f8acf50a0904c9d8c6ef857abe933fac31a211
-CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/rsut/gn/v20190308/rsut_Amon_CESM2_historical_r1i1p1f1_gn_200701-201412.nc 634e496041968a53223bc7ca709bd891222925253ebf14ce2d6b19f3bbd039e0
-CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/rsutcs/gn/v20190308/rsutcs_Amon_CESM2_historical_r1i1p1f1_gn_200701-201412.nc 9ec147f4c44acb8de4d06b25bc4a890a9090726ea75b2dc855d5b3aa96f174f0
-CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/ta/gn/v20190308/ta_Amon_CESM2_historical_r1i1p1f1_gn_200701-201412.nc 08865486d7f8e2f086a957b002257b65e03a1b332540da484423bdce652af873
+CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/cli/gn/v20190308/cli_Amon_CESM2_historical_r1i1p1f1_gn_199601-201412.nc a3cf533720f63ad0cf7ae6668649df4b43169dfee8783ffea7889c285c6df925
+CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/clivi/gn/v20190308/clivi_Amon_CESM2_historical_r1i1p1f1_gn_199601-201412.nc 564c22fa5c94fe257a0bf613d4674b69e505b3c2967e69d0cf529654256bb5fd
+CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/clt/gn/v20190308/clt_Amon_CESM2_historical_r1i1p1f1_gn_199601-201412.nc bb1e70874e7589f98ed6c9a02bc0a0612050ca373bc66670ae404e0a2d0a138b
+CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/clwvi/gn/v20190308/clwvi_Amon_CESM2_historical_r1i1p1f1_gn_199601-201412.nc 7194d9f605e11f4f54ca93e08c045f3889f8d8d63ecebe40a78cf07913647d7a
+CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/pr/gn/v20190401/pr_Amon_CESM2_historical_r1i1p1f1_gn_199601-201412.nc cb5c48b389bcce3af009b32a7100c5669da0f79bd2058ebc3711489ca7ccbfb7
+CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/rlut/gn/v20190308/rlut_Amon_CESM2_historical_r1i1p1f1_gn_199601-201412.nc 948760afc79c7f9401d5ed1b94bded8919aed9297dc672f45917b2f9e0228973
+CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/rlutcs/gn/v20190308/rlutcs_Amon_CESM2_historical_r1i1p1f1_gn_199601-201412.nc 855961882d420cc2ddecd573c6b64c027c822402d57c3d157832fef42de0247b
+CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/rsut/gn/v20190308/rsut_Amon_CESM2_historical_r1i1p1f1_gn_199601-201412.nc 7f3cea7e273ad8b593b00fd0bee865949f20a11bf468df4ac91ca80657e8e37d
+CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/rsutcs/gn/v20190308/rsutcs_Amon_CESM2_historical_r1i1p1f1_gn_199601-201412.nc 0f7cd2a564cbc6382e35d4bbbd67fed3b4c337f8a926dd56222e1fbc7b77380d
+CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/Amon/ta/gn/v20190308/ta_Amon_CESM2_historical_r1i1p1f1_gn_199601-201412.nc c3cab5bbcd4cbf3563271e4e8b634b6a849182ae391e5dde21865ec7b22061ba
 CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/ImonAnt/snc/gn/v20190308/snc_ImonAnt_CESM2_historical_r1i1p1f1_gn_200001-201412.nc 3c933b6aaf471b170d6498fed4e01fa73e45169cb8e0790ed70051b69107482f
 CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/ImonGre/snc/gn/v20190308/snc_ImonGre_CESM2_historical_r1i1p1f1_gn_200001-201412.nc 6b6d84052cd6283663a60f416cc8e80cccfa0f4a2c963b5256ad208f567bbfde
 CMIP6/CMIP/NCAR/CESM2/historical/r1i1p1f1/LImon/snc/gn/v20190308/snc_LImon_CESM2_historical_r1i1p1f1_gn_200001-201412.nc b0f91e0eb9eef40ffa0ff6b57b8016bf2d3c25e6b3fdc8d12a8aca4d27e692a3
@@ -216,15 +216,6 @@ obs4REF/obs4REF/ColumbiaU/WECANN-1-0/mon/hfls/gn/20250516/hfls_mon_WECANN-1-0_RE
 obs4REF/obs4REF/ColumbiaU/WECANN-1-0/mon/hfss/gn/20250516/hfss_mon_WECANN-1-0_REF_gn_200701-201512.nc 14bdeae9e0b4b7bfe849c97dbdd29eae87f27d9464e8b3795d815369b13ffd0c
 obs4REF/obs4REF/ECMWF/ERA-20C/mon/psl/gn/v20210727/psl_mon_ERA-20C_PCMDI_gn_190001-201012.nc 53262d8f9076f233399d149810a644464d3bb36ae0f131fd55f164bc623b78da
 obs4REF/obs4REF/ECMWF/ERA-20C/mon/ts/gn/v20210727/ts_mon_ERA-20C_PCMDI_gn_190001-201012.nc 95bf8da4b8a071464688b527e822724c33c2794d100052eb12eb2804219ddb94
-obs4REF/obs4REF/ECMWF/ERA-5/mon/ta/gn/v20250220/ta_mon_ERA-5_PCMDI_gn_200701-200712.nc 36bd5cbda06258fb6aafd9fb2ccb79b4d08574116a6ebe8ccc48b6462bdb6419
-obs4REF/obs4REF/ECMWF/ERA-5/mon/ta/gn/v20250220/ta_mon_ERA-5_PCMDI_gn_200801-200812.nc 9b7802f845ca67f6b4d4bd0a73e0bce1c5042ecf3e7b209a5e470fd084ead238
-obs4REF/obs4REF/ECMWF/ERA-5/mon/ta/gn/v20250220/ta_mon_ERA-5_PCMDI_gn_200901-200912.nc 208a988bc440699beda1738342e7571c28dd2c3b2d169e0770c1764996bd41a4
-obs4REF/obs4REF/ECMWF/ERA-5/mon/ta/gn/v20250220/ta_mon_ERA-5_PCMDI_gn_201001-201012.nc 3bfb4dec6966cea160af4ce872302af4d84ee2bd8bd3bba91468a424e17d9eae
-obs4REF/obs4REF/ECMWF/ERA-5/mon/ta/gn/v20250220/ta_mon_ERA-5_PCMDI_gn_201101-201112.nc da16b7d20e764e25af3c6b834376bed5041872a0b11fab59234eca5cf1124495
-obs4REF/obs4REF/ECMWF/ERA-5/mon/ta/gn/v20250220/ta_mon_ERA-5_PCMDI_gn_201201-201212.nc 08ae50141a576dfcbba0a9cf15a32653f48fa88d58406b60d21383e50dd309f0
-obs4REF/obs4REF/ECMWF/ERA-5/mon/ta/gn/v20250220/ta_mon_ERA-5_PCMDI_gn_201301-201312.nc 488e55c4f6c858301abb957a5fb7de866e93fa54b234dbce08df652fad634649
-obs4REF/obs4REF/ECMWF/ERA-5/mon/ta/gn/v20250220/ta_mon_ERA-5_PCMDI_gn_201401-201412.nc 9c5c4656b929d1c6dba5d83d5459db61d7d543182e58e29168eacdb7f151b125
-obs4REF/obs4REF/ECMWF/ERA-5/mon/ta/gn/v20250220/ta_mon_ERA-5_PCMDI_gn_201501-201512.nc 98e254f10b15c4d90dd258f66b8352f6e8b758f9bd64f435c90cb3bdd99c7086
 obs4REF/obs4REF/ECMWF/ERA-INT/mon/hfls/gn/v20210727/hfls_mon_ERA-INT_PCMDI_gn_197901-201903.nc 50d2b48789dcd642641b30ab52cc0f3ad161c057220cda52788080b2be2b927e
 obs4REF/obs4REF/ECMWF/ERA-INT/mon/hfss/gn/v20210727/hfss_mon_ERA-INT_PCMDI_gn_197901-201903.nc 72f15a671e88cb0ec239af9e8c1a608bdf2837c884efde9721213481bcfa02a0
 obs4REF/obs4REF/ECMWF/ERA-INT/mon/hur/gn/v20210727/hur_mon_ERA-INT_PCMDI_gn_198901-201001.nc 54c939a1a461930230a1ae1423856c1929d5dd6bab72cbdad1fe24c5da579908

climate_ref/datasets/__init__.py

@@ -25,15 +25,15 @@ def get_dataset_adapter(source_type: str, **kwargs: Any) -> "DatasetAdapter":
         DatasetAdapter instance
     """
     if source_type.lower() == SourceDatasetType.CMIP6.value:
-        from climate_ref.datasets.cmip6 import CMIP6DatasetAdapter
+        from climate_ref.datasets.cmip6 import CMIP6DatasetAdapter  # noqa: PLC0415
 
         return CMIP6DatasetAdapter(**kwargs)
     elif source_type.lower() == SourceDatasetType.obs4MIPs.value.lower():
-        from climate_ref.datasets.obs4mips import Obs4MIPsDatasetAdapter
+        from climate_ref.datasets.obs4mips import Obs4MIPsDatasetAdapter  # noqa: PLC0415
 
         return Obs4MIPsDatasetAdapter(**kwargs)
     elif source_type.lower() == SourceDatasetType.PMPClimatology.value.lower():
-        from climate_ref.datasets.pmp_climatology import PMPClimatologyDatasetAdapter
+        from climate_ref.datasets.pmp_climatology import PMPClimatologyDatasetAdapter  # noqa: PLC0415
 
         return PMPClimatologyDatasetAdapter(**kwargs)
     else:

climate_ref/datasets/base.py

@@ -2,16 +2,36 @@ from pathlib import Path
 from typing import Any, Protocol, cast
 
 import pandas as pd
+from attrs import define
 from loguru import logger
 from sqlalchemy.orm import joinedload
 
 from climate_ref.config import Config
-from climate_ref.database import Database
+from climate_ref.database import Database, ModelState
 from climate_ref.datasets.utils import validate_path
 from climate_ref.models.dataset import Dataset, DatasetFile
 from climate_ref_core.exceptions import RefException
 
 
+@define
+class DatasetRegistrationResult:
+    """
+    Result of registering a dataset, containing information about file changes
+    """
+
+    dataset: Dataset
+    dataset_state: ModelState | None
+    files_added: list[str]
+    files_updated: list[str]
+    files_removed: list[str]
+    files_unchanged: list[str]
+
+    @property
+    def total_changes(self) -> int:
+        """Total number of file changes (added + updated + removed)"""
+        return len(self.files_added) + len(self.files_updated) + len(self.files_removed)
+
+
 def _log_duplicate_metadata(
     data_catalog: pd.DataFrame, unique_metadata: pd.DataFrame, slug_column: str
 ) -> None:
@@ -26,7 +46,8 @@ def _log_duplicate_metadata(
         invalid_dataset_columns = invalid_dataset_nunique[invalid_dataset_nunique.gt(1)].index.tolist()
 
         # Include time_range in the list of invalid columns to make debugging easier
-        invalid_dataset_columns.append("time_range")
+        if "time_range" in data_catalog.columns and "time_range" not in invalid_dataset_columns:
+            invalid_dataset_columns.append("time_range")
 
         data_catalog_subset = data_catalog[data_catalog[slug_column] == instance_id]
 
@@ -169,9 +190,9 @@ class DatasetAdapter(Protocol):
 
         return data_catalog
 
-    def register_dataset(
+    def register_dataset(  # noqa: PLR0915
         self, config: Config, db: Database, data_catalog_dataset: pd.DataFrame
-    ) -> Dataset | None:
+    ) -> DatasetRegistrationResult:
         """
         Register a dataset in the database using the data catalog
 
@@ -187,7 +208,7 @@ class DatasetAdapter(Protocol):
         Returns
         -------
         :
-            Registered dataset if successful, else None
+            Registration result with dataset and file change information
         """
         DatasetModel = self.dataset_cls
 
@@ -197,24 +218,104 @@ class DatasetAdapter(Protocol):
             raise RefException(f"Found multiple datasets in the same directory: {unique_slugs}")
         slug = unique_slugs[0]
 
+        # Upsert the dataset (create a new dataset or update the metadata)
         dataset_metadata = data_catalog_dataset[list(self.dataset_specific_metadata)].iloc[0].to_dict()
-        dataset, created = db.get_or_create(DatasetModel, defaults=dataset_metadata, slug=slug)
-        if not created:
-            logger.warning(f"{dataset} already exists in the database. Skipping")
-            return None
+        dataset, dataset_state = db.update_or_create(DatasetModel, defaults=dataset_metadata, slug=slug)
+        if dataset_state == ModelState.CREATED:
+            logger.info(f"Created new dataset: {dataset}")
+        elif dataset_state == ModelState.UPDATED:
+            logger.info(f"Updating existing dataset: {dataset}")
         db.session.flush()
-        for dataset_file in data_catalog_dataset.to_dict(orient="records"):
-            path = validate_path(dataset_file.pop("path"))
-
-            db.session.add(
-                DatasetFile(
-                    path=str(path),
-                    dataset_id=dataset.id,
-                    start_time=dataset_file.pop("start_time"),
-                    end_time=dataset_file.pop("end_time"),
+
+        # Initialize result tracking
+        files_added = []
+        files_updated = []
+        files_removed = []
+        files_unchanged = []
+
+        # Get current files for this dataset
+        current_files = db.session.query(DatasetFile).filter_by(dataset_id=dataset.id).all()
+        current_file_paths = {f.path: f for f in current_files}
+
+        # Get new file data from data catalog
+        new_file_data = data_catalog_dataset.to_dict(orient="records")
+        new_file_lookup = {}
+        for dataset_file in new_file_data:
+            file_path = str(validate_path(dataset_file["path"]))
+            new_file_lookup[file_path] = {
+                "start_time": dataset_file["start_time"],
+                "end_time": dataset_file["end_time"],
+            }
+
+        new_file_paths = set(new_file_lookup.keys())
+        existing_file_paths = set(current_file_paths.keys())
+
+        # TODO: support removing files that are no longer present
+        # We want to keep a record of the dataset if it was used by a diagnostic in the past
+        files_to_remove = existing_file_paths - new_file_paths
+        if files_to_remove:
+            files_removed = list(files_to_remove)
+            logger.warning(f"Files to remove: {files_removed}")
+            raise NotImplementedError("Removing files is not yet supported")
+
+        # Update existing files if start/end times have changed
+        for file_path, existing_file in current_file_paths.items():
+            if file_path in new_file_lookup:
+                new_times = new_file_lookup[file_path]
+                if (
+                    existing_file.start_time != new_times["start_time"]
+                    or existing_file.end_time != new_times["end_time"]
+                ):
+                    logger.warning(f"Updating file times for {file_path}")
+                    existing_file.start_time = new_times["start_time"]
+                    existing_file.end_time = new_times["end_time"]
+                    files_updated.append(file_path)
+                else:
+                    files_unchanged.append(file_path)
+
+        # Add new files (batch operation)
+        files_to_add = new_file_paths - existing_file_paths
+        if files_to_add:
+            files_added = list(files_to_add)
+            new_dataset_files = []
+            for file_path in files_to_add:
+                file_times = new_file_lookup[file_path]
+                new_dataset_files.append(
+                    DatasetFile(
+                        path=file_path,
+                        dataset_id=dataset.id,
+                        start_time=file_times["start_time"],
+                        end_time=file_times["end_time"],
+                    )
                 )
-            )
-        return dataset
+            db.session.add_all(new_dataset_files)
+
+        # Determine final dataset state
+        # If dataset metadata changed, use that state
+        # If no metadata changed but files changed, consider it updated
+        # If nothing changed, keep the original state (None for existing, CREATED for new)
+        final_dataset_state = dataset_state
+        if dataset_state is None and (files_added or files_updated or files_removed):
+            final_dataset_state = ModelState.UPDATED
+
+        result = DatasetRegistrationResult(
+            dataset=dataset,
+            dataset_state=final_dataset_state,
+            files_added=files_added,
+            files_updated=files_updated,
+            files_removed=files_removed,
+            files_unchanged=files_unchanged,
+        )
+        change_message = f": ({final_dataset_state.name})" if final_dataset_state else ""
+        logger.debug(
+            f"Dataset registration complete for {dataset.slug}{change_message} "
+            f"{len(files_added)} files added, "
+            f"{len(files_updated)} files updated, "
+            f"{len(files_removed)} files removed, "
+            f"{len(files_unchanged)} files unchanged"
+        )
+
+        return result
 
     def _get_dataset_files(self, db: Database, limit: int | None = None) -> pd.DataFrame:
         dataset_type = self.dataset_cls.__mapper_args__["polymorphic_identity"]

climate_ref/datasets/cmip6.py

@@ -119,6 +119,8 @@ class CMIP6DatasetAdapter(DatasetAdapter):
     file_specific_metadata = ("start_time", "end_time", "path")
 
     version_metadata = "version"
+    # See https://wcrp-cmip.github.io/WGCM_Infrastructure_Panel/Papers/CMIP6_global_attributes_filenames_CVs_v6.2.7.pdf
+    # under "Directory structure template"
     dataset_id_metadata = (
         "activity_id",
         "institution_id",

climate_ref/datasets/obs4mips.py

@@ -7,7 +7,6 @@ from typing import Any
 import pandas as pd
 import xarray as xr
 from ecgtools import Builder
-from ecgtools.parsers.utilities import extract_attr_with_regex  # type: ignore
 from loguru import logger
 
 from climate_ref.datasets.base import DatasetAdapter
@@ -15,7 +14,7 @@ from climate_ref.datasets.cmip6 import _parse_datetime
 from climate_ref.models.dataset import Dataset, Obs4MIPsDataset
 
 
-def parse_obs4mips(file: str, **kwargs: Any) -> dict[str, Any]:
+def parse_obs4mips(file: str, **kwargs: Any) -> dict[str, Any]:  # noqa: PLR0912
     """
     Parser for obs4mips
 
@@ -41,6 +40,7 @@ def parse_obs4mips(file: str, **kwargs: Any) -> dict[str, Any]:
                 "source_type",
                 "variable_id",
                 "variant_label",
+                "source_version_number",
             }
         )
     )
@@ -48,6 +48,10 @@ def parse_obs4mips(file: str, **kwargs: Any) -> dict[str, Any]:
     try:
         time_coder = xr.coders.CFDatetimeCoder(use_cftime=True)
         with xr.open_dataset(file, chunks={}, decode_times=time_coder) as ds:
+            if ds.attrs.get("activity_id", "") != "obs4MIPs":
+                traceback_message = f"{file} is not an obs4MIPs dataset"
+                raise TypeError(traceback_message)
+
             has_none_value = any(ds.attrs.get(key) is None for key in keys)
             if has_none_value:
                 missing_fields = [key for key in keys if ds.attrs.get(key) is None]
@@ -55,10 +59,6 @@ def parse_obs4mips(file: str, **kwargs: Any) -> dict[str, Any]:
                 raise AttributeError(traceback_message)
             info = {key: ds.attrs.get(key) for key in keys}
 
-            if info["activity_id"] != "obs4MIPs":
-                traceback_message = f"{file} is not an obs4MIPs dataset"
-                raise TypeError(traceback_message)
-
             variable_id = info["variable_id"]
 
             if variable_id:
@@ -86,12 +86,12 @@ def parse_obs4mips(file: str, **kwargs: Any) -> dict[str, Any]:
             else:
                 info["time_range"] = f"{start_time}-{end_time}"
         info["path"] = str(file)
-        info["source_version_number"] = (
-            extract_attr_with_regex(
-                str(file), regex=r"v\d{4}\d{2}\d{2}|v\d{1}", strip_chars=None, ignore_case=True
-            )
-            or "v0"
-        )
+        # Parsing the version like for CMIP6 fails because some obs4REF paths
+        # do not include "v" in the version directory name.
+        # TODO: fix obs4REF paths
+        info["version"] = Path(file).parent.name
+        if not info["version"].startswith("v"):  # type: ignore[union-attr]
+            info["version"] = "v{version}".format(**info)
         return info
 
     except (TypeError, AttributeError) as err:
@@ -99,7 +99,7 @@ def parse_obs4mips(file: str, **kwargs: Any) -> dict[str, Any]:
             logger.warning(str(err.args[0]))
         else:
             logger.warning(str(err.args))
-        return {"INVALID_ASSET": file, "TRACEBACK": traceback_message}
+        return {"INVALID_ASSET": file, "TRACEBACK": str(err)}
     except Exception:
         logger.warning(traceback.format_exc())
         return {"INVALID_ASSET": file, "TRACEBACK": traceback.format_exc()}
@@ -129,18 +129,22 @@ class Obs4MIPsDatasetAdapter(DatasetAdapter):
         "variant_label",
         "long_name",
         "units",
+        "version",
         "vertical_levels",
         "source_version_number",
         slug_column,
     )
 
     file_specific_metadata = ("start_time", "end_time", "path")
-    version_metadata = "source_version_number"
+    version_metadata = "version"
+    # See ODS2.5 at https://doi.org/10.5281/zenodo.11500474 under "Directory structure template"
     dataset_id_metadata = (
         "activity_id",
         "institution_id",
         "source_id",
+        "frequency",
         "variable_id",
+        "nominal_resolution",
         "grid_label",
     )
 
@@ -186,7 +190,14 @@ class Obs4MIPsDatasetAdapter(DatasetAdapter):
             self.version_metadata,
         ]
         datasets["instance_id"] = datasets.apply(
-            lambda row: "obs4MIPs." + ".".join([row[item] for item in drs_items]), axis=1
+            lambda row: "obs4MIPs."
+            + ".".join(
+                [
+                    row[item].replace(" ", "") if item == "nominal_resolution" else row[item]
+                    for item in drs_items
+                ]
+            ),
+            axis=1,
         )
         datasets["finalised"] = True
         return datasets