climate-ref 0.5.1__tar.gz → 0.5.3__tar.gz

{climate_ref-0.5.1 → climate_ref-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: climate-ref
-Version: 0.5.1
+Version: 0.5.3
 Summary: Application which runs the CMIP Rapid Evaluation Framework
 Author-email: Jared Lewis <jared.lewis@climate-resource.com>, Mika Pflueger <mika.pflueger@climate-resource.com>, Bouwe Andela <b.andela@esciencecenter.nl>, Jiwoo Lee <lee1043@llnl.gov>, Min Xu <xum1@ornl.gov>, Nathan Collier <collierno@ornl.gov>, Dora Hegedus <dora.hegedus@stfc.ac.uk>
 License: Apache-2.0
@@ -30,15 +30,19 @@ Requires-Dist: sqlalchemy>=2.0.36
 Requires-Dist: tomlkit>=0.13.2
 Requires-Dist: tqdm>=4.67.1
 Requires-Dist: typer>=0.12.5
+Provides-Extra: aft-providers
+Requires-Dist: climate-ref-esmvaltool>=0.5.0; extra == 'aft-providers'
+Requires-Dist: climate-ref-ilamb>=0.5.0; extra == 'aft-providers'
+Requires-Dist: climate-ref-pmp>=0.5.0; extra == 'aft-providers'
 Provides-Extra: celery
 Requires-Dist: climate-ref-celery>=0.5.0; extra == 'celery'
-Provides-Extra: metrics
-Requires-Dist: climate-ref-esmvaltool>=0.5.0; extra == 'metrics'
-Requires-Dist: climate-ref-ilamb>=0.5.0; extra == 'metrics'
-Requires-Dist: climate-ref-pmp>=0.5.0; extra == 'metrics'
 Provides-Extra: postgres
 Requires-Dist: alembic-postgresql-enum>=1.7.0; extra == 'postgres'
 Requires-Dist: psycopg2-binary>=2.9.2; extra == 'postgres'
+Provides-Extra: providers
+Requires-Dist: climate-ref-esmvaltool>=0.5.0; extra == 'providers'
+Requires-Dist: climate-ref-ilamb>=0.5.0; extra == 'providers'
+Requires-Dist: climate-ref-pmp>=0.5.0; extra == 'providers'
 Description-Content-Type: text/markdown
 
 # Climate REF (Rapid Evaluation Framework)
@@ -64,15 +68,15 @@ pip install climate-ref
 If you want to use the diagnostic providers for the Assessment Fast Track, you can install them with:
 
 ```bash
-pip install climate-ref[metrics]
+pip install climate-ref[aft-providers]
 ```
 
 ## Quick Start
 
 ```bash
 # Ingest some observation datasets
-ref datasets fetch-data --registry obs4ref --output-dir datasets/obs4ref
-ref datasets fetch-data --registry sample-data --output-dir datasets/sample-data
+ref datasets fetch-data --registry obs4ref ---output-directory datasets/obs4ref
+ref datasets fetch-data --registry sample-data ---output-directory datasets/sample-data
 
 # Run metrics against your climate data
 ref solve

{climate_ref-0.5.1 → climate_ref-0.5.3}/README.md

@@ -21,15 +21,15 @@ pip install climate-ref
 If you want to use the diagnostic providers for the Assessment Fast Track, you can install them with:
 
 ```bash
-pip install climate-ref[metrics]
+pip install climate-ref[aft-providers]
 ```
 
 ## Quick Start
 
 ```bash
 # Ingest some observation datasets
-ref datasets fetch-data --registry obs4ref --output-dir datasets/obs4ref
-ref datasets fetch-data --registry sample-data --output-dir datasets/sample-data
+ref datasets fetch-data --registry obs4ref ---output-directory datasets/obs4ref
+ref datasets fetch-data --registry sample-data ---output-directory datasets/sample-data
 
 # Run metrics against your climate data
 ref solve

{climate_ref-0.5.1 → climate_ref-0.5.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "climate-ref"
-version = "0.5.1"
+version = "0.5.3"
 description = "Application which runs the CMIP Rapid Evaluation Framework"
 readme = "README.md"
 authors = [
@@ -49,7 +49,12 @@ postgres = [
 celery = [
     "climate-ref-celery>=0.5.0",
 ]
-metrics = [
+aft-providers = [
+    "climate-ref-esmvaltool>=0.5.0",
+    "climate-ref-pmp>=0.5.0",
+    "climate-ref-ilamb>=0.5.0",
+]
+providers = [
     "climate-ref-esmvaltool>=0.5.0",
     "climate-ref-pmp>=0.5.0",
     "climate-ref-ilamb>=0.5.0",

{climate_ref-0.5.1 → climate_ref-0.5.3}/src/climate_ref/cli/solve.py

@@ -10,6 +10,12 @@ def solve(
     ctx: typer.Context,
     dry_run: bool = typer.Option(False, help="Do not execute any diagnostics"),
     timeout: int = typer.Option(60, help="Timeout in seconds for the solve operation"),
+    one_per_provider: bool = typer.Option(
+        False, help="Limit to one execution per provider. This is useful for testing"
+    ),
+    one_per_diagnostic: bool = typer.Option(
+        False, help="Limit to one execution per diagnostic. This is useful for testing"
+    ),
 ) -> None:
     """
     Solve for executions that require recalculation
@@ -19,4 +25,11 @@ def solve(
     """
     config = ctx.obj.config
     db = ctx.obj.database
-    solve_required_executions(config=config, db=db, dry_run=dry_run, timeout=timeout)
+    solve_required_executions(
+        config=config,
+        db=db,
+        dry_run=dry_run,
+        timeout=timeout,
+        one_per_provider=one_per_provider,
+        one_per_diagnostic=one_per_diagnostic,
+    )

{climate_ref-0.5.1 → climate_ref-0.5.3}/src/climate_ref/config.py

@@ -74,11 +74,13 @@ class PathConfig:
     /// admonition | Warning
         type: warning
 
-    These paths must be common across all systems that the REF is being run
+    These paths must be common across all systems that the REF is being run.
+    Generally, this means that they should be mounted in the same location on all systems.
     ///
 
     If any of these paths are specified as relative paths,
     they will be resolved to absolute paths.
+    These absolute paths will be used for all operations in the REF.
     """
 
     log: Path = env_field(name="LOG_ROOT", converter=ensure_absolute_path)
@@ -157,12 +159,12 @@ class ExecutorConfig:
 
     executor: str = env_field(name="EXECUTOR", default="climate_ref.executor.LocalExecutor")
     """
-    Executor to use for running diagnostics
+    Executor class to use for running diagnostics
 
     This should be the fully qualified name of the executor class
     (e.g. `climate_ref.executor.LocalExecutor`).
-    The default is to use the local executor.
-    The environment variable `REF_EXECUTOR` takes precedence over this configuration value.
+    The default is to use the local executor which runs the executions locally, in-parallel
+    using a process pool.
 
     This class will be used for all executions of diagnostics.
     """
@@ -172,6 +174,7 @@ class ExecutorConfig:
     Additional configuration for the executor.
 
     See the documentation for the executor for the available configuration options.
+    These options will be passed to the executor class when it is created.
     """
 
     def build(self, config: "Config", database: "Database") -> Executor:
@@ -199,7 +202,30 @@ class ExecutorConfig:
 @define
 class DiagnosticProviderConfig:
     """
-    Configuration for the diagnostic providers
+    Defining the diagnostic providers used by the REF.
+
+    Each diagnostic provider is a package that contains the logic for running a specific
+    set of diagnostics.
+    This configuration determines which diagnostic providers are loaded and used when solving.
+
+    Multiple diagnostic providers can be specified as shown in the example below.
+
+    ```toml
+    [[diagnostic_providers]]
+    provider = "climate_ref_esmvaltool.provider"
+
+    [diagnostic_providers.config]
+
+    [[diagnostic_providers]]
+    provider = "climate_ref_ilamb.provider"
+
+    [diagnostic_providers.config]
+
+    [[diagnostic_providers]]
+    provider = "climate_ref_pmp.provider"
+
+    [diagnostic_providers.config]
+    ```
     """
 
     provider: str
@@ -224,24 +250,42 @@ class DbConfig:
     """
     Database configuration
 
-    We currently only plan to support SQLite and PostgreSQL databases,
-    although only SQLite is currently implemented and tested.
+    We support SQLite and PostgreSQL databases.
+    The default is to use SQLite, which is a file-based database that is stored in the
+    `REF_CONFIGURATION` directory.
+    This is a good option for testing and development, but not recommended for production use.
+
+    For production use, we recommend using PostgreSQL.
     """
 
     database_url: str = env_field(name="DATABASE_URL")
     """
     Database URL that describes the connection to the database.
 
-    Defaults to sqlite:///{config.paths.db}/climate_ref.db".
+    Defaults to `sqlite:///{config.paths.db}/climate_ref.db`.
     This configuration value will be overridden by the `REF_DATABASE_URL` environment variable.
 
-    ## Schemas
+    **Schemas**
 
+    The following schemas are supported:
+    ```
     postgresql://USER:PASSWORD@HOST:PORT/NAME
+
     sqlite:///RELATIVE_PATH or sqlite:////ABS_PATH or sqlite:///:memory:
+    ```
     """
     run_migrations: bool = field(default=True)
 
+    max_backups: int = env_field(name="MAX_BACKUPS", default=5)
+    """
+    Maximum number of database backups to keep.
+
+
+    When running migrations for on-disk SQLite databases, a backup of the database is created.
+    This setting controls how many of these backups are retained.
+    The oldest backups are automatically removed when this limit is exceeded.
+    """
+
     @database_url.default
     def _connection_url_factory(self) -> str:
         filename = env.path("REF_CONFIGURATION") / "db" / "climate_ref.db"
@@ -285,12 +329,10 @@ def _load_config(config_file: str | Path, doc: dict[str, Any]) -> "Config":
     return _converter_defaults_relaxed.structure(doc, Config)
 
 
-@define
+@define(auto_attribs=True)
 class Config:
     """
-    REF configuration
-
-    This class is used to store the configuration of the REF application.
+    Configuration that is used by the REF
     """
 
     log_level: str = field(default="INFO")

{climate_ref-0.5.1 → climate_ref-0.5.3}/src/climate_ref/database.py

@@ -9,6 +9,8 @@ It provides a session object that can be used to interact with the database and
 """
 
 import importlib.resources
+import shutil
+from datetime import datetime
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 from urllib import parse as urlparse
@@ -53,6 +55,44 @@ def _get_database_revision(connection: sqlalchemy.engine.Connection) -> str | No
     return current_rev
 
 
+def _create_backup(db_path: Path, max_backups: int) -> Path:
+    """
+    Create a backup of the database file
+
+    Parameters
+    ----------
+    db_path
+        Path to the database file
+    max_backups
+        Maximum number of backups to keep
+
+    Returns
+    -------
+    :
+        Path to the backup file
+    """
+    if not db_path.exists():
+        logger.warning(f"Database file {db_path} does not exist, skipping backup")
+        return db_path
+
+    backup_dir = db_path.parent / "backups"
+    backup_dir.mkdir(exist_ok=True)
+
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    backup_path = backup_dir / f"{db_path.stem}_{timestamp}{db_path.suffix}"
+
+    logger.info(f"Creating backup of database at {backup_path}")
+    shutil.copy2(db_path, backup_path)
+
+    # Clean up old backups
+    backups = sorted(backup_dir.glob(f"{db_path.stem}_*{db_path.suffix}"), reverse=True)
+    for old_backup in backups[max_backups:]:
+        logger.info(f"Removing old backup {old_backup}")
+        old_backup.unlink()
+
+    return backup_path
+
+
 def validate_database_url(database_url: str) -> str:
     """
     Validate a database URL
@@ -153,6 +193,12 @@ class Database:
                     "Please delete your database and start again."
                 )
 
+        # Create backup before running migrations
+        split_url = urlparse.urlsplit(self.url)
+        if split_url.scheme == "sqlite" and split_url.path != ":memory:":
+            db_path = Path(split_url.path[1:])
+            _create_backup(db_path, config.db.max_backups)
+
         alembic.command.upgrade(self.alembic_config(config), "heads")
 
     @staticmethod

climate_ref-0.5.3/src/climate_ref/dataset_registry/obs4ref_reference.txt

@@ -0,0 +1,36 @@
+obs4REF/ECMWF/ERA-20C/mon/psl/gn/v20210727/psl_mon_ERA-20C_PCMDI_gn_190001-201012.nc md5:c100cf25d5681c375cd6c1ee60b678ba
+obs4REF/ECMWF/ERA-20C/mon/ts/gn/v20210727/ts_mon_ERA-20C_PCMDI_gn_190001-201012.nc md5:9ed8dfbb805ed4caa282ed70f873a3a0
+obs4REF/NOAA-ESRL-PSD/20CR/mon/psl/gn/v20210727/psl_mon_20CR_PCMDI_gn_187101-201212.nc md5:570ce90b3afd1d0b31690ae5dbe32d31
+obs4REF/NOAA-ESRL-PSD/20CR/mon/ts/gn/v20210727/ts_mon_20CR_PCMDI_gn_187101-201212.nc md5:e4890cc19ccc5bac29c6b70f28265ff1
+obs4REF/NOAA-NCEI/CMAP-V1902/mon/pr/gn/v20210727/pr_mon_CMAP-V1902_PCMDI_gn_197901-201901.nc md5:9d943d2dd0645850b616820f246aedf3
+obs4REF/MOHC/HadISST-1-1/mon/ts/gn/v20210727/ts_mon_HadISST-1-1_PCMDI_gn_187001-201907.nc md5:99c8691e0f615dc4d79b4fb5e926cc76
+obs4REF/ESSO/TropFlux-1-0/mon/hfls/gn/v20210727/hfls_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc md5:2f05191d6727068e1500d8d4ed90098a
+obs4REF/ESSO/TropFlux-1-0/mon/hfns/gn/v20210727/hfns_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc md5:7a9019e51a41d9e4ab1fcfb072d8ca8d
+obs4REF/ESSO/TropFlux-1-0/mon/hfss/gn/v20210727/hfss_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc md5:1da9d8fe862c61bc49c36c18b6527213
+obs4REF/ESSO/TropFlux-1-0/mon/tas/gn/v20210727/tas_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc md5:a6057931b5f6bc000a44514a1a8c891f
+obs4REF/ESSO/TropFlux-1-0/mon/tauu/gn/v20210727/tauu_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc md5:7c73a3deed3403fa9d21caef3a4d988d
+obs4REF/ESSO/TropFlux-1-0/mon/tauv/gn/v20210727/tauv_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc md5:8abc7a724a7a297826e2f783a4ea14f9
+obs4REF/ESSO/TropFlux-1-0/mon/ts/gn/v20210727/ts_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc md5:8697d3d7862f6e3b72bb5a161aa75ee8
+obs4REF/ECMWF/ERA-INT/mon/hfls/gn/v20210727/hfls_mon_ERA-INT_PCMDI_gn_197901-201903.nc md5:1ae4587143f05ee81432b3d9960aab63
+obs4REF/ECMWF/ERA-INT/mon/hfss/gn/v20210727/hfss_mon_ERA-INT_PCMDI_gn_197901-201903.nc md5:261f02b8cbce18486548882a11f9aa34
+obs4REF/ECMWF/ERA-INT/mon/hur/gn/v20210727/hur_mon_ERA-INT_PCMDI_gn_198901-201001.nc md5:56fcd2df8ed2879f18b5e8c78134a148
+obs4REF/ECMWF/ERA-INT/mon/hus/gn/v20210727/hus_mon_ERA-INT_PCMDI_gn_198901-201001.nc md5:6f3344d5d936fb84084ac1caf6431016
+obs4REF/ECMWF/ERA-INT/mon/pr/gn/v20210727/pr_mon_ERA-INT_PCMDI_gn_197901-201903.nc md5:92e48c992cde6850d41c52e1778057c2
+obs4REF/ECMWF/ERA-INT/mon/psl/gn/v20210727/psl_mon_ERA-INT_PCMDI_gn_197901-201903.nc md5:7409d0caa4a675436e28085bed12d0c9
+obs4REF/ECMWF/ERA-INT/mon/rlds/gn/v20210727/rlds_mon_ERA-INT_PCMDI_gn_197901-201903.nc md5:dfba31b6b8e6a4924b7b6cfba648715f
+obs4REF/ECMWF/ERA-INT/mon/rlus/gn/v20210727/rlus_mon_ERA-INT_PCMDI_gn_197901-201903.nc md5:94acf2c3bd5f39301148f610722b4590
+obs4REF/ECMWF/ERA-INT/mon/rsds/gn/v20210727/rsds_mon_ERA-INT_PCMDI_gn_197901-201903.nc md5:0a56d7cd3fdf112a37c48433b8cd1e29
+obs4REF/ECMWF/ERA-INT/mon/rsus/gn/v20210727/rsus_mon_ERA-INT_PCMDI_gn_197901-201903.nc md5:ec8f6e9cb64d81a7c0b70fbd7692e133
+obs4REF/ECMWF/ERA-INT/mon/sfcWind/gn/v20210727/sfcWind_mon_ERA-INT_PCMDI_gn_197901-201903.nc md5:8a54a0edef0b9c22ea0b1dd8d5f2dd57
+obs4REF/ECMWF/ERA-INT/mon/ta/gn/v20210727/ta_mon_ERA-INT_PCMDI_gn_198901-201001.nc md5:458b5d536c7314c29f8867a5417a95a1
+obs4REF/ECMWF/ERA-INT/mon/tauu/gn/v20210727/tauu_mon_ERA-INT_PCMDI_gn_197901-201903.nc md5:af86da822ebd76ed3ee0a7e5d5a9912f
+obs4REF/ECMWF/ERA-INT/mon/tauv/gn/v20210727/tauv_mon_ERA-INT_PCMDI_gn_197901-201903.nc md5:87757bbcaff35aef1d685649563ba0a7
+obs4REF/ECMWF/ERA-INT/mon/ts/gn/v20210727/ts_mon_ERA-INT_PCMDI_gn_197901-201903.nc md5:f6a6f2bd8710df95c06dcb910501a771
+obs4REF/ECMWF/ERA-INT/mon/ua/gn/v20210727/ua_mon_ERA-INT_PCMDI_gn_198901-201001.nc md5:054306f777065cf8cf94d44e412510bd
+obs4REF/ECMWF/ERA-INT/mon/uas/gn/v20210727/uas_mon_ERA-INT_PCMDI_gn_197901-201903.nc md5:2b3144fc737c5e3d2aa930c53751ff2b
+obs4REF/ECMWF/ERA-INT/mon/va/gn/v20210727/va_mon_ERA-INT_PCMDI_gn_198901-201001.nc md5:f67ca168d6cd87bfdd4a911eb72dd022
+obs4REF/ECMWF/ERA-INT/mon/vas/gn/v20210727/vas_mon_ERA-INT_PCMDI_gn_197901-201903.nc md5:ac19b48b897cfe839585df4ff0fc4a7b
+obs4REF/ECMWF/ERA-INT/mon/zg/gn/v20210727/zg_mon_ERA-INT_PCMDI_gn_198901-201001.nc md5:d8fb93f31ff4a6370ccee93db82af86c
+obs4REF/NOAA-NCEI/GPCP-2-3/mon/pr/gn/v20210727/pr_mon_GPCP-2-3_PCMDI_gn_197901-201907.nc md5:0877f014868b83547448f96c3e7c83e9
+obs4REF/NASA-GSFC/TRMM-3B43v-7/mon/pr/gn/v20210727/pr_mon_TRMM-3B43v-7_PCMDI_gn_199801-201712.nc md5:b80c9989d358656c781be5ea5a44c64c
+obs4REF/CNES/AVISO-1-0/mon/zos/gn/v20210727/zos_mon_AVISO-1-0_PCMDI_gn_199301-201912.nc md5:91252303cb65548fee5ff42dd3024825

climate_ref-0.5.3/src/climate_ref/dataset_registry/sample_data.txt

@@ -0,0 +1,105 @@
+CMIP6/C4MIP/CSIRO/ACCESS-ESM1-5/esm-1pct-brch-1000PgC/r1i1p1f1/Amon/tas/gn/v20191206/tas_Amon_ACCESS-ESM1-5_esm-1pct-brch-1000PgC_r1i1p1f1_gn_016801-026812.nc ffef38229e3b69ea703274f174152cfc4886665bb346c1cc604d88f932137828
+CMIP6/C4MIP/CSIRO/ACCESS-ESM1-5/esm-1pct-brch-1000PgC/r1i1p1f1/fx/areacella/gn/v20191206/areacella_fx_ACCESS-ESM1-5_esm-1pct-brch-1000PgC_r1i1p1f1_gn.nc d3292e1af73f8ab0327a4f689f8b4fdd7d818e861d8b78351daf392de15e4409
+CMIP6/C4MIP/MPI-M/MPI-ESM1-2-LR/esm-1pctCO2/r1i1p1f1/Amon/fco2antt/gn/v20190815/fco2antt_Amon_MPI-ESM1-2-LR_esm-1pctCO2_r1i1p1f1_gn_185001-186912.nc c027154d3ef3962584d48e603638754c59f8bb106923cd8523a08db5be67f2a0
+CMIP6/C4MIP/MPI-M/MPI-ESM1-2-LR/esm-1pctCO2/r1i1p1f1/Amon/fco2antt/gn/v20190815/fco2antt_Amon_MPI-ESM1-2-LR_esm-1pctCO2_r1i1p1f1_gn_187001-188912.nc a550c2c36e620eafadba06a6e6f80ff231481b23eb225d9c25874e50122107a9
+CMIP6/C4MIP/MPI-M/MPI-ESM1-2-LR/esm-1pctCO2/r1i1p1f1/Amon/fco2antt/gn/v20190815/fco2antt_Amon_MPI-ESM1-2-LR_esm-1pctCO2_r1i1p1f1_gn_189001-190912.nc 05734f60346b084873772e9b166c136fa5daf1338e8a5a15721e2834389c0cc6
+CMIP6/C4MIP/MPI-M/MPI-ESM1-2-LR/esm-1pctCO2/r1i1p1f1/Amon/fco2antt/gn/v20190815/fco2antt_Amon_MPI-ESM1-2-LR_esm-1pctCO2_r1i1p1f1_gn_191001-191412.nc 82bf8fbcfde0418b4ec95b14af9b653f9b654f748ca1d1deb6450c4ea4ef277c
+CMIP6/C4MIP/MPI-M/MPI-ESM1-2-LR/esm-1pctCO2/r1i1p1f1/Amon/tas/gn/v20190815/tas_Amon_MPI-ESM1-2-LR_esm-1pctCO2_r1i1p1f1_gn_185001-186912.nc f0dcb10542fc6b685238028a333f6255d22b630d588555a7691c26e213267033
+CMIP6/C4MIP/MPI-M/MPI-ESM1-2-LR/esm-1pctCO2/r1i1p1f1/Amon/tas/gn/v20190815/tas_Amon_MPI-ESM1-2-LR_esm-1pctCO2_r1i1p1f1_gn_187001-188912.nc b9cd82588c1a5be0a6eb9b54cb3f6699c9a0f39ce0ce87f59ea8a6ac76c16561
+CMIP6/C4MIP/MPI-M/MPI-ESM1-2-LR/esm-1pctCO2/r1i1p1f1/Amon/tas/gn/v20190815/tas_Amon_MPI-ESM1-2-LR_esm-1pctCO2_r1i1p1f1_gn_189001-190912.nc 5000371d6f31bab096119b000edd5be66b3dbe1b53962ccb01b3105ad449579a
+CMIP6/C4MIP/MPI-M/MPI-ESM1-2-LR/esm-1pctCO2/r1i1p1f1/Amon/tas/gn/v20190815/tas_Amon_MPI-ESM1-2-LR_esm-1pctCO2_r1i1p1f1_gn_191001-191412.nc 9213e72eb6e1e77e7585313838ca245c8ab8f87829ad2e55ba9769df8dde0035
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/1pctCO2/r1i1p1f1/Amon/tas/gn/v20191115/tas_Amon_ACCESS-ESM1-5_1pctCO2_r1i1p1f1_gn_010101-025012.nc b995e36ed0a7d1748901d83359f8995e0d7bfeb1af05ee25e6a376815417ba85
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/1pctCO2/r1i1p1f1/fx/areacella/gn/v20191115/areacella_fx_ACCESS-ESM1-5_1pctCO2_r1i1p1f1_gn.nc f6b1dcf34a6de5c9c0c0743d1f3344923e476e3ad33d73d29da67163b6973d01
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/abrupt-4xCO2/r1i1p1f1/Amon/rlut/gn/v20191115/rlut_Amon_ACCESS-ESM1-5_abrupt-4xCO2_r1i1p1f1_gn_010101-012512.nc d02d4096c7cd1892e284a97d5009e109fce474c2c9e91e43f83ecc9519ae2ce5
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/abrupt-4xCO2/r1i1p1f1/Amon/rsdt/gn/v20191115/rsdt_Amon_ACCESS-ESM1-5_abrupt-4xCO2_r1i1p1f1_gn_010101-012512.nc 14cd0b3ce52c8e11c92cf24ffc70bda191a1da41b0004afdcfbd43d71730d10d
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/abrupt-4xCO2/r1i1p1f1/Amon/rsut/gn/v20191115/rsut_Amon_ACCESS-ESM1-5_abrupt-4xCO2_r1i1p1f1_gn_010101-012512.nc 85c6e81dc193b326646781e5b46d481fa720435b30adf40370a8dc0dd1c49714
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/abrupt-4xCO2/r1i1p1f1/Amon/tas/gn/v20191115/tas_Amon_ACCESS-ESM1-5_abrupt-4xCO2_r1i1p1f1_gn_010101-012512.nc 9469aca1a6d082ae052e0885207acd8b16859aad37d9e0a514c71c7b45419887
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/abrupt-4xCO2/r1i1p1f1/fx/areacella/gn/v20191115/areacella_fx_ACCESS-ESM1-5_abrupt-4xCO2_r1i1p1f1_gn.nc 78f79e91d0fa163e2744aa70c1ac89b8c878a2753ece016110563b6e3c4afd8b
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Amon/pr/gn/v20191115/pr_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_185001-201412.nc 6696166383b0427476de40658f1d31666a28d9ec2334ffb68be49339e24f49bc
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Amon/psl/gn/v20191115/psl_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc 42c9e8f0d432fe634abab6fe3499854e19ae5ccaf5d70dba6e930aed4c8fd635
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Amon/rlut/gn/v20191115/rlut_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc 019b61b1a05bfc06ef35a1b97e0544d242996c3c1d177ad9da6189e549af0e19
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Amon/rlutcs/gn/v20191115/rlutcs_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200501-201412.nc d639634716a12c0af933aa4cbd62edef8e2f1bbab454f3935ed0b1d91d384b63
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Amon/rsdt/gn/v20191115/rsdt_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc dd43eb17caf7082bcc18f31ecd756566466f7fc17ae86284985177ba0ac7d386
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Amon/rsut/gn/v20191115/rsut_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc b3148af8bee4c3562607aeaec9d81d94abad366c85421dd55ef222516fd3f7e0
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Amon/rsutcs/gn/v20191115/rsutcs_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200501-201412.nc 9f336ed549c02bf31f73f4679b850463ca8edc4db7afb04b31acd643bc3b51c8
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Amon/tas/gn/v20191115/tas_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_185001-201412.nc 4c74c74031fbc88cb0df70826e0aea63a582af5f5a714603a0ae70b3ff834097
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Amon/ts/gn/v20191115/ts_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc b3b8e7675287471fd27f3de1f3ecada278eb7ea384da24031d0f3a949e04f757
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Emon/cSoil/gn/v20191115/cSoil_Emon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc d78d9bc532d79a50a0e40645f91572d61a92881aea43552061baea0e77353777
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Lmon/gpp/gn/v20191115/gpp_Lmon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc 41a43edaa1ff7385232f8fbf67fcc551eb77e51ba6cd75b95b5f0ee71af19544
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Lmon/lai/gn/v20191115/lai_Lmon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc 4e6e1530da9030cfb7c207543369c636f339ee3c59c4621e22405494dc436755
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Lmon/mrro/gn/v20191115/mrro_Lmon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc 3b9c010af44ca894bb23a55903fdc57dcd5af8a4e22ea1e98b41b18274720fd9
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Lmon/mrsos/gn/v20191115/mrsos_Lmon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc ca21d4f5341114cbc217d7c30f89c4375b02ea0a2b723d9cb069db6a9c08b08f
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Lmon/nbp/gn/v20191115/nbp_Lmon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc 476b4ebacf38ddb78d44975b01d859e9ca3ed7955b92adc4bd098591867e56b0
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Ofx/areacello/gn/v20191115/areacello_Ofx_ACCESS-ESM1-5_historical_r1i1p1f1_gn.nc fe5adde7cfe6d80462ad649fe68c2aa983ba700d96cff5ac830b1d7e7b09ca6f
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Ofx/sftof/gn/v20191115/sftof_Ofx_ACCESS-ESM1-5_historical_r1i1p1f1_gn.nc 89106ad204fb1ac9f3b25f4b93d6fcf9e95db5f1f4466b9baafcb426bad24e4f
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Omon/msftmz/gn/v20191115/msftmz_Omon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_197901-197912.nc 9016163abd89155710291dc541a4146778f18a320435a2ee618f0da1424b0338
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Omon/msftmz/gn/v20191115/msftmz_Omon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_198001-198912.nc 97094c32cb3e8f019ad87145d348307c2a57ba9b0acf0cc5c59978213060e8d4
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Omon/msftmz/gn/v20191115/msftmz_Omon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_199001-199912.nc be682038248b0d79ea9e0402a2a173bd5c054dfdd8cc0932f6cb85d925670889
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Omon/msftmz/gn/v20191115/msftmz_Omon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-200912.nc 266c386ddfbe9556404afadef1e778a64b5f4c5dd746c54a1beee3e5fbc9954b
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Omon/msftmz/gn/v20191115/msftmz_Omon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_201001-201412.nc 21fa3a223baced12da8a62ad406ba8126851c3e36589184fba2bb1c8b269c6e3
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Omon/sos/gn/v20191115/sos_Omon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_197901-201412.nc a757237973708d28f2333542fc34ed9543a15256c7f38d0139fcfe66bdb8d95c
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Omon/tos/gn/v20191115/tos_Omon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc 701fa01ac2c50b622c3454bb3d69a54fed1c92b66f9194c09148b2349aa11342
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/SImon/siconc/gn/v20200817/siconc_SImon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_197901-201412.nc 384b45a0f88678ffaf0c9701a0fc0175fb0c319d7d94d0b3c4334939d653e51d
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/fx/areacella/gn/v20191115/areacella_fx_ACCESS-ESM1-5_historical_r1i1p1f1_gn.nc 6750d4a2432842cb9342ef99b8ecb3569e8ff3dcbbf020f7f9f43a6f7af42f06
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/fx/sftlf/gn/v20191115/sftlf_fx_ACCESS-ESM1-5_historical_r1i1p1f1_gn.nc 08d84ba3cf02a2481b76611dfa1abe25cbf76326003eebc4eb00c99b32fea19a
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r2i1p1f1/Amon/psl/gn/v20191128/psl_Amon_ACCESS-ESM1-5_historical_r2i1p1f1_gn_200001-201412.nc 0344cf5babc7e3f827d66e1c41a9aaa89b52b6345381bf478146df44c7ec813e
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r2i1p1f1/Amon/ts/gn/v20191128/ts_Amon_ACCESS-ESM1-5_historical_r2i1p1f1_gn_200001-201412.nc e5dde58ae1b21c765d00dea5e6362c771f6945dcb62def864313c9756ed4b8db
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r2i1p1f1/fx/areacella/gn/v20191128/areacella_fx_ACCESS-ESM1-5_historical_r2i1p1f1_gn.nc 9a1df5675b2168b3da8e5537f339196ebb8eb73f2dbcf1431f5c269c46191f40
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/piControl/r1i1p1f1/Amon/rlut/gn/v20210316/rlut_Amon_ACCESS-ESM1-5_piControl_r1i1p1f1_gn_010101-012512.nc a8bf35b4415b5d24971fb94f7a04b99ea3534157ea3c1995abf379bad82bbc83
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/piControl/r1i1p1f1/Amon/rsdt/gn/v20210316/rsdt_Amon_ACCESS-ESM1-5_piControl_r1i1p1f1_gn_010101-012512.nc 158cea3284df26fa95851e22bdee4064b909ecd87e12acd52aec242df7171e49
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/piControl/r1i1p1f1/Amon/rsut/gn/v20210316/rsut_Amon_ACCESS-ESM1-5_piControl_r1i1p1f1_gn_010101-012512.nc 5a8f026e4be636e51625a6b59e504de6ed2dfcfae1df8be60084fdf1d602fc1c
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/piControl/r1i1p1f1/Amon/tas/gn/v20210316/tas_Amon_ACCESS-ESM1-5_piControl_r1i1p1f1_gn_010101-018012.nc e8705dda0c26cad086526d8e77f10a91707187e371d90f5ebb7feb4187cec2b4
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/piControl/r1i1p1f1/fx/areacella/gn/v20210316/areacella_fx_ACCESS-ESM1-5_piControl_r1i1p1f1_gn.nc c3f274b147fbcb7e7c6116a933a40993c4f2186952b8e8fd4ad8056326688887
+CMIP6/CMIP/MPI-M/MPI-ESM1-2-LR/esm-piControl/r1i1p1f1/Amon/tas/gn/v20190815/tas_Amon_MPI-ESM1-2-LR_esm-piControl_r1i1p1f1_gn_185001-186912.nc 141a24094d42fe2bc0b765774905c192d77817620da273b7e7c789510d0891c9
+CMIP6/CMIP/MPI-M/MPI-ESM1-2-LR/esm-piControl/r1i1p1f1/Amon/tas/gn/v20190815/tas_Amon_MPI-ESM1-2-LR_esm-piControl_r1i1p1f1_gn_187001-188912.nc 5f0dc6bee4c4eb12a05c532fb0ca08aaacbb869b43306c8b82f4bdd6adc661a4
+CMIP6/CMIP/MPI-M/MPI-ESM1-2-LR/esm-piControl/r1i1p1f1/Amon/tas/gn/v20190815/tas_Amon_MPI-ESM1-2-LR_esm-piControl_r1i1p1f1_gn_189001-190912.nc ae9193e06aec8a7f63f2f7ec63fec9d25b2c1dc5de2bbe7065428c722090f45c
+CMIP6/CMIP/MPI-M/MPI-ESM1-2-LR/esm-piControl/r1i1p1f1/Amon/tas/gn/v20190815/tas_Amon_MPI-ESM1-2-LR_esm-piControl_r1i1p1f1_gn_191001-191512.nc 2113b7d4a781c7c28de55b0ffe99c42b49692d113f5109e7a4d0214211a9eb1c
+CMIP6/CMIP/MPI-M/MPI-ESM1-2-LR/esm-piControl/r1i1p1f1/fx/areacella/gn/v20190815/areacella_fx_MPI-ESM1-2-LR_esm-piControl_r1i1p1f1_gn.nc 2f5595635a566d5b4a889468f687aabf99ca5cdadb11f6a770d05d09775bfabb
+CMIP6/DAMIP/CSIRO/ACCESS-ESM1-5/hist-GHG/r1i1p1f1/Amon/psl/gn/v20200615/psl_Amon_ACCESS-ESM1-5_hist-GHG_r1i1p1f1_gn_200001-202012.nc 5fed6b48717fcf508e8b5936be304db0346362c2182c5d13a141afe7c982021a
+CMIP6/DAMIP/CSIRO/ACCESS-ESM1-5/hist-GHG/r1i1p1f1/Amon/ts/gn/v20200615/ts_Amon_ACCESS-ESM1-5_hist-GHG_r1i1p1f1_gn_200001-202012.nc aa98fa93ef29ea266a94d87876b4ae08917caf77e38c5df1eae1ec98a5f559cb
+CMIP6/DAMIP/CSIRO/ACCESS-ESM1-5/hist-GHG/r1i1p1f1/fx/areacella/gn/v20200615/areacella_fx_ACCESS-ESM1-5_hist-GHG_r1i1p1f1_gn.nc 7179f75ff1754e9666b10f8c5508040bbe9fdac0c8117a9231ba5977d4889f27
+CMIP6/DAMIP/CSIRO/ACCESS-ESM1-5/hist-GHG/r2i1p1f1/Amon/psl/gn/v20200615/psl_Amon_ACCESS-ESM1-5_hist-GHG_r2i1p1f1_gn_200001-202012.nc 2073711f1517b53f2670f939b3ad8c81f24ae8d022bd193157e173733eb19636
+CMIP6/DAMIP/CSIRO/ACCESS-ESM1-5/hist-GHG/r2i1p1f1/Amon/ts/gn/v20200615/ts_Amon_ACCESS-ESM1-5_hist-GHG_r2i1p1f1_gn_200001-202012.nc dd8b8043a058454c6218d3895ffd12d3d8cfc2ccef485ece30c913ecc2244d57
+CMIP6/DAMIP/CSIRO/ACCESS-ESM1-5/hist-GHG/r2i1p1f1/fx/areacella/gn/v20200615/areacella_fx_ACCESS-ESM1-5_hist-GHG_r2i1p1f1_gn.nc 6a25555c13d62859cb81f739f24398ae4f1dae60d495d8ada5f799411471985b
+CMIP6/ScenarioMIP/CSIRO/ACCESS-ESM1-5/ssp126/r1i1p1f1/Amon/pr/gn/v20210318/pr_Amon_ACCESS-ESM1-5_ssp126_r1i1p1f1_gn_201501-210012.nc 3650d30ce27d44a0564ee29fd5e4741a60fe7e91ea634f4f6896f3fc494c1cd7
+CMIP6/ScenarioMIP/CSIRO/ACCESS-ESM1-5/ssp126/r1i1p1f1/Amon/rsdt/gn/v20210318/rsdt_Amon_ACCESS-ESM1-5_ssp126_r1i1p1f1_gn_201501-202512.nc 9a04641b91da68be60e307cd1b3e6af72098ae09e5d16103910368314ebf6217
+CMIP6/ScenarioMIP/CSIRO/ACCESS-ESM1-5/ssp126/r1i1p1f1/Amon/rsut/gn/v20210318/rsut_Amon_ACCESS-ESM1-5_ssp126_r1i1p1f1_gn_201501-202512.nc 61aee51691b9e5b4a9d1189bdd59a8dd206ad5e8d554c5ea5e47827e0fd9216a
+CMIP6/ScenarioMIP/CSIRO/ACCESS-ESM1-5/ssp126/r1i1p1f1/Amon/tas/gn/v20210318/tas_Amon_ACCESS-ESM1-5_ssp126_r1i1p1f1_gn_201501-210012.nc b164b974798624b284ac83425fc02f42ae88d75071d76aa3b860253f527ce27f
+CMIP6/ScenarioMIP/CSIRO/ACCESS-ESM1-5/ssp126/r1i1p1f1/Omon/tos/gn/v20210318/tos_Omon_ACCESS-ESM1-5_ssp126_r1i1p1f1_gn_201501-202512.nc 8138c9e1d5ba641799b547b0867fa0068902c17b95277cd26bec1c7e91c7eb0c
+CMIP6/ScenarioMIP/CSIRO/ACCESS-ESM1-5/ssp126/r1i1p1f1/fx/areacella/gn/v20210318/areacella_fx_ACCESS-ESM1-5_ssp126_r1i1p1f1_gn.nc 0b3daf08cacf722fe51e01dd106f0df2fa86b11764befcf26a55d2e670db74df
+obs4REF/obs4REF/CNES/AVISO-1-0/mon/zos/gn/v20210727/zos_mon_AVISO-1-0_PCMDI_gn_199301-201912.nc 1b0a61ea75fbc12a0c8f5ed64c62ddd52c14b30a2873b4977584899c879b66c2
+obs4REF/obs4REF/ECMWF/ERA-20C/mon/psl/gn/v20210727/psl_mon_ERA-20C_PCMDI_gn_190001-201012.nc a307849fa2ef38954400ecef089438400a98e2ba38806c36529ac4b7b91c11a5
+obs4REF/obs4REF/ECMWF/ERA-20C/mon/ts/gn/v20210727/ts_mon_ERA-20C_PCMDI_gn_190001-201012.nc e75d22c2cd5077cc4ced12526f280248b3ab251b0995b830f2f6d716a165af3f
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/hfls/gn/v20210727/hfls_mon_ERA-INT_PCMDI_gn_197901-201903.nc c51b850dccfa4a95d4cbd9555a333aef6f6299191a5a6ccf5a8dec0b3ba389a4
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/hfss/gn/v20210727/hfss_mon_ERA-INT_PCMDI_gn_197901-201903.nc 11f12bb01d71a8d769099fff0f2ff41a2c01a967ce6ab5eb4ef4f691f669de80
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/hur/gn/v20210727/hur_mon_ERA-INT_PCMDI_gn_198901-201001.nc 06b9290fef6dbd9bd776a428429309aed88c32faf2ddae2c3076c817d22ca75b
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/hus/gn/v20210727/hus_mon_ERA-INT_PCMDI_gn_198901-201001.nc 97af286fe733dcf33144517034df92803b80057fecbda488a576a60515763d62
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/pr/gn/v20210727/pr_mon_ERA-INT_PCMDI_gn_197901-201903.nc 7f663506b2984a98b952efc0cae7c2165d5b201884480462672d219e1ac03c13
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/psl/gn/v20210727/psl_mon_ERA-INT_PCMDI_gn_197901-201903.nc 7b9e6aeafe4926fa2435fcf0f8cb0a4afcbfc9aa8db47187e21d5f280879e0ec
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/rlds/gn/v20210727/rlds_mon_ERA-INT_PCMDI_gn_197901-201903.nc d20e0f45fe49252982afbd281980764cac4d3d2aec93e6c18f0f88f30382397c
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/rlus/gn/v20210727/rlus_mon_ERA-INT_PCMDI_gn_197901-201903.nc ba7f248c2dd60f4ce52bb34390c133f2ff73f507476899c6430ba88fa0e92baf
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/rsds/gn/v20210727/rsds_mon_ERA-INT_PCMDI_gn_197901-201903.nc b37c38913868538c6ef1aed10c80c0492dc83cf897073c7bcfd820f716f69db1
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/rsus/gn/v20210727/rsus_mon_ERA-INT_PCMDI_gn_197901-201903.nc 4840ef8fdce03ed7397756d5cbc445949bfce2e64e4f469559eefc0acb0bfd53
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/sfcWind/gn/v20210727/sfcWind_mon_ERA-INT_PCMDI_gn_197901-201903.nc 0e96f90f858cdcd394a6c75c37bd5cf16f8629c46a7ff7e160141c216812eb4f
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/ta/gn/v20210727/ta_mon_ERA-INT_PCMDI_gn_198901-201001.nc d8024c2a148d9c096bb89a6ab7cf88a4857d6368bd87b3b2e8ec2d9e6f99259c
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/tauu/gn/v20210727/tauu_mon_ERA-INT_PCMDI_gn_197901-201903.nc a89e8f5bc93ceca4f73eaab3320836ec29396125ae66df2ba8f55053dd7127fe
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/tauv/gn/v20210727/tauv_mon_ERA-INT_PCMDI_gn_197901-201903.nc 6879b06537c0bb899171f26594e80a4e83b6c5ff70e4304811bf5f334f841a91
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/ts/gn/v20210727/ts_mon_ERA-INT_PCMDI_gn_197901-201903.nc f37fae9b17febbe259044c8652798a94ed50549603dbd2bd0d481c8838bc8f7d
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/ua/gn/v20210727/ua_mon_ERA-INT_PCMDI_gn_198901-201001.nc 988cc905d04e6aa0278d915dd1873242277fca5901c82b3e674e4775bfe71da7
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/uas/gn/v20210727/uas_mon_ERA-INT_PCMDI_gn_197901-201903.nc c104226c7c8cb3ba1c0a75e848eb1c4ad33f4337c283b96daf9a565ead018283
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/va/gn/v20210727/va_mon_ERA-INT_PCMDI_gn_198901-201001.nc 1549f008a6ee6a1d2c8b76f9094328acd296a220cc36e39eb40e4ec345db7785
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/vas/gn/v20210727/vas_mon_ERA-INT_PCMDI_gn_197901-201903.nc 366d6726c8e8f998a4c51eaf9e3905f915cd269171b491afaedf0581d28be2a5
+obs4REF/obs4REF/ECMWF/ERA-INT/mon/zg/gn/v20210727/zg_mon_ERA-INT_PCMDI_gn_198901-201001.nc 60083418a6597e41d9d9b4f80f3731db2c93d065b4ed9113383470a3090943fc
+obs4REF/obs4REF/ESSO/TropFlux-1-0/mon/hfls/gn/v20210727/hfls_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc c05111c9b0680c52b0f78a0f0603c204130d941527f079ba6c431c35b81d1b09
+obs4REF/obs4REF/ESSO/TropFlux-1-0/mon/hfns/gn/v20210727/hfns_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc 0f47f55514cb6ac29d8903775cd5af197ccb828d3eb9403eb94c71bdc5a08fc4
+obs4REF/obs4REF/ESSO/TropFlux-1-0/mon/hfss/gn/v20210727/hfss_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc 1fc71a0befc05e29295025b4f2a1d3030000991ce54ffd5e4e57b513af74f9f5
+obs4REF/obs4REF/ESSO/TropFlux-1-0/mon/tas/gn/v20210727/tas_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc 8ac0c77ebc9672401bebfc599b7535696eead0c75f4f40ffa735be802b5d6a62
+obs4REF/obs4REF/ESSO/TropFlux-1-0/mon/tauu/gn/v20210727/tauu_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc b073ed8f915064cf69daeb51d5c31b1167ca8f1ac1f5d4f324ecdfd5847827c6
+obs4REF/obs4REF/ESSO/TropFlux-1-0/mon/tauv/gn/v20210727/tauv_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc f9b27f56e08dbccec19aa33ac70f1f04bd51eac4d1fa7554c237676b7a3411e2
+obs4REF/obs4REF/ESSO/TropFlux-1-0/mon/ts/gn/v20210727/ts_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc 8c8857d8dca575d7b2e9cfd7130cfe6172d96e41614ed3666cbfbdb4af9fbdda
+obs4REF/obs4REF/MOHC/HadISST-1-1/mon/ts/gn/v20210727/ts_mon_HadISST-1-1_PCMDI_gn_187001-201907.nc f5cc77d002cd90dc63c63a4db62863fcbf72763d02ae7ba743ff691fe3b10fe3
+obs4REF/obs4REF/NASA-GSFC/TRMM-3B43v-7/mon/pr/gn/v20210727/pr_mon_TRMM-3B43v-7_PCMDI_gn_199801-201712.nc 5df3a5cb29bb08f317dda64bd3659d81c382d69c29d1325f461cbcb0272eb5bc
+obs4REF/obs4REF/NOAA-ESRL-PSD/20CR/mon/psl/gn/v20210727/psl_mon_20CR_PCMDI_gn_187101-201212.nc 47c894191e9a9f62d4c455721ff6bf1a5ab1571082b5a08a2c52ff1240b77ac3
+obs4REF/obs4REF/NOAA-ESRL-PSD/20CR/mon/ts/gn/v20210727/ts_mon_20CR_PCMDI_gn_187101-201212.nc c9a4a75c3df7c24e5662756a7ca8674af8275244ea9fe213384d1a1e41e1bbad
+obs4REF/obs4REF/NOAA-NCEI/CMAP-V1902/mon/pr/gn/v20210727/pr_mon_CMAP-V1902_PCMDI_gn_197901-201901.nc 0a73fa7fef1f631321f2cb70a6aea12c9cc2a2ed00e6cb5d8c03618fe6c9f2ae
+obs4REF/obs4REF/NOAA-NCEI/GPCP-2-3/mon/pr/gn/v20210727/pr_mon_GPCP-2-3_PCMDI_gn_197901-201907.nc dcca502f8748a73b0be5c6e402e14086f1d45e5c85be3cbbe63ba6c07ab0eb21

{climate_ref-0.5.1 → climate_ref-0.5.3}/src/climate_ref/datasets/cmip6.py

@@ -1,13 +1,16 @@
 from __future__ import annotations
 
+import traceback
 import warnings
 from datetime import datetime
 from pathlib import Path
 from typing import Any
 
-import ecgtools.parsers
 import pandas as pd
+import xarray as xr
 from ecgtools import Builder
+from ecgtools.parsers.utilities import extract_attr_with_regex  # type: ignore
+from loguru import logger
 
 from climate_ref.datasets.base import DatasetAdapter
 from climate_ref.models.dataset import CMIP6Dataset
@@ -65,6 +68,88 @@ def _clean_branch_time(branch_time: pd.Series[str]) -> pd.Series[float]:
     return pd.to_numeric(branch_time.astype(str).str.replace("D", ""), errors="coerce")
 
 
+def parse_cmip6(file: str) -> dict[str, Any]:
+    """
+    Parser for CMIP6
+
+    This function parses the CMIP6 dataset and returns a dictionary with the metadata.
+    This was copied from the ecgtools package, but we want to log the exception when it fails.
+    """
+    keys = sorted(
+        {
+            "activity_id",
+            "branch_method",
+            "branch_time_in_child",
+            "branch_time_in_parent",
+            "experiment",
+            "experiment_id",
+            "frequency",
+            "grid",
+            "grid_label",
+            "institution_id",
+            "nominal_resolution",
+            "parent_activity_id",
+            "parent_experiment_id",
+            "parent_source_id",
+            "parent_time_units",
+            "parent_variant_label",
+            "realm",
+            "product",
+            "source_id",
+            "source_type",
+            "sub_experiment",
+            "sub_experiment_id",
+            "table_id",
+            "variable_id",
+            "variant_label",
+        }
+    )
+
+    try:
+        with xr.open_dataset(file, chunks={}, use_cftime=True) as ds:
+            info = {key: ds.attrs.get(key) for key in keys}
+            info["member_id"] = info["variant_label"]
+
+            variable_id = info["variable_id"]
+            if variable_id:  # pragma: no branch
+                attrs = ds[variable_id].attrs
+                for attr in ["standard_name", "long_name", "units"]:
+                    info[attr] = attrs.get(attr)
+
+            # Set the default of # of vertical levels to 1
+            vertical_levels = 1
+            start_time, end_time = None, None
+            init_year = None
+            try:
+                vertical_levels = ds[ds.cf["vertical"].name].size
+            except (KeyError, AttributeError, ValueError):
+                ...
+
+            try:
+                start_time, end_time = str(ds.cf["T"][0].data), str(ds.cf["T"][-1].data)
+            except (KeyError, AttributeError, ValueError):
+                ...
+            if info.get("sub_experiment_id"):  # pragma: no branch
+                init_year = extract_attr_with_regex(info["sub_experiment_id"], r"\d{4}")
+                if init_year:  # pragma: no cover
+                    init_year = int(init_year)
+            info["vertical_levels"] = vertical_levels
+            info["init_year"] = init_year
+            info["start_time"] = start_time
+            info["end_time"] = end_time
+            if not (start_time and end_time):
+                info["time_range"] = None
+            else:
+                info["time_range"] = f"{start_time}-{end_time}"
+        info["path"] = str(file)
+        info["version"] = extract_attr_with_regex(str(file), regex=r"v\d{4}\d{2}\d{2}|v\d{1}") or "v0"
+        return info
+
+    except Exception:
+        logger.exception(f"Failed to parse {file}")
+        return {"INVALID_ASSET": file, "TRACEBACK": traceback.format_exc()}
+
+
 class CMIP6DatasetAdapter(DatasetAdapter):
     """
     Adapter for CMIP6 datasets
@@ -171,7 +256,7 @@ class CMIP6DatasetAdapter(DatasetAdapter):
                 depth=10,
                 include_patterns=["*.nc"],
                 joblib_parallel_kwargs={"n_jobs": self.n_jobs},
-            ).build(parsing_func=ecgtools.parsers.parse_cmip6)
+            ).build(parsing_func=parse_cmip6)  # type: ignore
 
         datasets: pd.DataFrame = builder.df.drop(["init_year"], axis=1)
 

{climate_ref-0.5.1 → climate_ref-0.5.3}/src/climate_ref/executor/result_handling.py

@@ -133,24 +133,25 @@ def handle_execution_result(
             # execution.mark_failed()
 
         # Perform a bulk insert of scalar values
-        # TODO: The section below will likely fail until we have agreed on a controlled vocabulary
         # The current implementation will swallow the exception, but display a log message
         try:
-            # Perform this in a nested transaction to (hopefully) gracefully rollback if something
-            # goes wrong
-            with database.session.begin_nested():
-                database.session.execute(
-                    insert(ScalarMetricValue),
-                    [
-                        {
-                            "execution_id": execution.id,
-                            "value": result.value,
-                            "attributes": result.attributes,
-                            **result.dimensions,
-                        }
-                        for result in cmec_metric_bundle.iter_results()
-                    ],
-                )
+            scalar_values = [
+                {
+                    "execution_id": execution.id,
+                    "value": result.value,
+                    "attributes": result.attributes,
+                    **result.dimensions,
+                }
+                for result in cmec_metric_bundle.iter_results()
+            ]
+            if scalar_values:
+                # Perform this in a nested transaction to rollback if something goes wrong
+                # We will lose the metric values for a given execution, but not the whole execution
+                with database.session.begin_nested():
+                    database.session.execute(
+                        insert(ScalarMetricValue),
+                        scalar_values,
+                    )
         except Exception:
             # TODO: Remove once we have settled on a controlled vocabulary
             logger.exception("Something went wrong when ingesting diagnostic values")