climate-ref 0.6.4__py3-none-any.whl → 0.6.5__py3-none-any.whl

climate_ref/config.py

@@ -17,7 +17,7 @@ which always take precedence over any other configuration values.
 import importlib.resources
 import os
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Literal
 
 import tomlkit
 from attr import Factory
@@ -354,6 +354,16 @@ class Config:
     [loguru documentation](https://loguru.readthedocs.io/en/stable/api/logger.html#module-loguru._logger).
     """
 
+    cmip6_parser: Literal["drs", "complete"] = env_field("CMIP6_PARSER", default="complete")
+    """
+    Parser to use for CMIP6 datasets
+
+    This can be either `drs` or `complete`.
+
+    - `drs`: Use the DRS parser, which parses the dataset based on the DRS naming conventions.
+    - `complete`: Use the complete parser, which parses the dataset based on all available metadata.
+    """
+
     paths: PathConfig = Factory(PathConfig)  # noqa
     db: DbConfig = Factory(DbConfig)  # noqa
     executor: ExecutorConfig = Factory(ExecutorConfig)  # noqa

climate_ref/datasets/base.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Protocol, cast
+from typing import Any, Protocol, cast
 
 import pandas as pd
 from loguru import logger
@@ -35,6 +35,31 @@ def _log_duplicate_metadata(
         )
 
 
+class DatasetParsingFunction(Protocol):
+    """
+    Protocol for a function that parses metadata from a file or directory
+    """
+
+    def __call__(self, file: str, **kwargs: Any) -> dict[str, Any]:
+        """
+        Parse a file or directory and return metadata for the dataset
+
+        Parameters
+        ----------
+        file
+            File or directory to parse
+
+        kwargs
+            Additional keyword arguments to pass to the parsing function.
+
+        Returns
+        -------
+        :
+            Data catalog containing the metadata for the dataset
+        """
+        ...
+
+
 class DatasetAdapter(Protocol):
     """
     An adapter to provide a common interface for different dataset types
@@ -173,7 +198,7 @@ class DatasetAdapter(Protocol):
         slug = unique_slugs[0]
 
         dataset_metadata = data_catalog_dataset[list(self.dataset_specific_metadata)].iloc[0].to_dict()
-        dataset, created = db.get_or_create(DatasetModel, slug=slug, **dataset_metadata)
+        dataset, created = db.get_or_create(DatasetModel, defaults=dataset_metadata, slug=slug)
         if not created:
             logger.warning(f"{dataset} already exists in the database. Skipping")
             return None
@@ -212,6 +237,7 @@ class DatasetAdapter(Protocol):
                 {
                     **{k: getattr(file, k) for k in self.file_specific_metadata},
                     **{k: getattr(file.dataset, k) for k in self.dataset_specific_metadata},
+                    "finalised": file.dataset.finalised,
                 }
                 for file in result
             ],

climate_ref/datasets/cmip6.py

@@ -1,18 +1,17 @@
 from __future__ import annotations
 
-import traceback
 import warnings
 from datetime import datetime
 from pathlib import Path
 from typing import Any
 
 import pandas as pd
-import xarray as xr
 from ecgtools import Builder
-from ecgtools.parsers.utilities import extract_attr_with_regex  # type: ignore
 from loguru import logger
 
-from climate_ref.datasets.base import DatasetAdapter
+from climate_ref.config import Config
+from climate_ref.datasets.base import DatasetAdapter, DatasetParsingFunction
+from climate_ref.datasets.cmip6_parsers import parse_cmip6_complete, parse_cmip6_drs
 from climate_ref.models.dataset import CMIP6Dataset
 
 
@@ -22,16 +21,19 @@ def _parse_datetime(dt_str: pd.Series[str]) -> pd.Series[datetime | Any]:
     """
 
     def _inner(date_string: str | None) -> datetime | None:
-        if not date_string:
+        if not date_string or pd.isnull(date_string):
             return None
 
         # Try to parse the date string with and without milliseconds
-        try:
-            dt = datetime.strptime(date_string, "%Y-%m-%d %H:%M:%S")
-        except ValueError:
-            dt = datetime.strptime(date_string, "%Y-%m-%d %H:%M:%S.%f")
+        for fmt in ("%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S.%f"):
+            try:
+                return datetime.strptime(date_string, fmt)
+            except ValueError:
+                continue
 
-        return dt
+        # If all parsing attempts fail, log an error and return None
+        logger.error(f"Failed to parse date string: {date_string}")
+        return None
 
     return pd.Series(
         [_inner(dt) for dt in dt_str],
@@ -44,15 +46,16 @@ def _apply_fixes(data_catalog: pd.DataFrame) -> pd.DataFrame:
     def _fix_parent_variant_label(group: pd.DataFrame) -> pd.DataFrame:
         if group["parent_variant_label"].nunique() == 1:
             return group
-        group["parent_variant_label"] = group["variant_label"].iloc[0]
+        group["parent_variant_label"] = group["parent_variant_label"].iloc[0]
 
         return group
 
-    data_catalog = (
-        data_catalog.groupby("instance_id")
-        .apply(_fix_parent_variant_label, include_groups=False)
-        .reset_index(level="instance_id")
-    )
+    if "parent_variant_label" in data_catalog:
+        data_catalog = (
+            data_catalog.groupby("instance_id")
+            .apply(_fix_parent_variant_label, include_groups=False)
+            .reset_index(level="instance_id")
+        )
 
     if "branch_time_in_child" in data_catalog:
         data_catalog["branch_time_in_child"] = _clean_branch_time(data_catalog["branch_time_in_child"])
@@ -68,88 +71,6 @@ def _clean_branch_time(branch_time: pd.Series[str]) -> pd.Series[float]:
     return pd.to_numeric(branch_time.astype(str).str.replace("D", ""), errors="coerce")
 
 
-def parse_cmip6(file: str) -> dict[str, Any]:
-    """
-    Parser for CMIP6
-
-    This function parses the CMIP6 dataset and returns a dictionary with the metadata.
-    This was copied from the ecgtools package, but we want to log the exception when it fails.
-    """
-    keys = sorted(
-        {
-            "activity_id",
-            "branch_method",
-            "branch_time_in_child",
-            "branch_time_in_parent",
-            "experiment",
-            "experiment_id",
-            "frequency",
-            "grid",
-            "grid_label",
-            "institution_id",
-            "nominal_resolution",
-            "parent_activity_id",
-            "parent_experiment_id",
-            "parent_source_id",
-            "parent_time_units",
-            "parent_variant_label",
-            "realm",
-            "product",
-            "source_id",
-            "source_type",
-            "sub_experiment",
-            "sub_experiment_id",
-            "table_id",
-            "variable_id",
-            "variant_label",
-        }
-    )
-
-    try:
-        with xr.open_dataset(file, chunks={}, use_cftime=True) as ds:
-            info = {key: ds.attrs.get(key) for key in keys}
-            info["member_id"] = info["variant_label"]
-
-            variable_id = info["variable_id"]
-            if variable_id:  # pragma: no branch
-                attrs = ds[variable_id].attrs
-                for attr in ["standard_name", "long_name", "units"]:
-                    info[attr] = attrs.get(attr)
-
-            # Set the default of # of vertical levels to 1
-            vertical_levels = 1
-            start_time, end_time = None, None
-            init_year = None
-            try:
-                vertical_levels = ds[ds.cf["vertical"].name].size
-            except (KeyError, AttributeError, ValueError):
-                ...
-
-            try:
-                start_time, end_time = str(ds.cf["T"][0].data), str(ds.cf["T"][-1].data)
-            except (KeyError, AttributeError, ValueError):
-                ...
-            if info.get("sub_experiment_id"):  # pragma: no branch
-                init_year = extract_attr_with_regex(info["sub_experiment_id"], r"\d{4}")
-                if init_year:  # pragma: no cover
-                    init_year = int(init_year)
-            info["vertical_levels"] = vertical_levels
-            info["init_year"] = init_year
-            info["start_time"] = start_time
-            info["end_time"] = end_time
-            if not (start_time and end_time):
-                info["time_range"] = None
-            else:
-                info["time_range"] = f"{start_time}-{end_time}"
-        info["path"] = str(file)
-        info["version"] = extract_attr_with_regex(str(file), regex=r"v\d{4}\d{2}\d{2}|v\d{1}") or "v0"
-        return info
-
-    except Exception:
-        logger.exception(f"Failed to parse {file}")
-        return {"INVALID_ASSET": file, "TRACEBACK": traceback.format_exc()}
-
-
 class CMIP6DatasetAdapter(DatasetAdapter):
     """
     Adapter for CMIP6 datasets
@@ -191,6 +112,7 @@ class CMIP6DatasetAdapter(DatasetAdapter):
         "standard_name",
         "long_name",
         "units",
+        "finalised",
         slug_column,
     )
 
@@ -208,8 +130,30 @@ class CMIP6DatasetAdapter(DatasetAdapter):
         "grid_label",
     )
 
-    def __init__(self, n_jobs: int = 1):
+    def __init__(self, n_jobs: int = 1, config: Config | None = None):
         self.n_jobs = n_jobs
+        self.config = config or Config.default()
+
+    def get_parsing_function(self) -> DatasetParsingFunction:
+        """
+        Get the parsing function for CMIP6 datasets based on configuration
+
+        The parsing function used is determined by the `cmip6_parser` configuration value:
+        - "drs": Use the DRS parser (default)
+        - "complete": Use the complete parser that extracts all available metadata
+
+        Returns
+        -------
+        :
+            The appropriate parsing function based on configuration
+        """
+        parser_type = self.config.cmip6_parser
+        if parser_type == "complete":
+            logger.info("Using complete CMIP6 parser")
+            return parse_cmip6_complete
+        else:
+            logger.info(f"Using DRS CMIP6 parser (config value: {parser_type})")
+            return parse_cmip6_drs
 
     def find_local_datasets(self, file_or_directory: Path) -> pd.DataFrame:
         """
@@ -228,6 +172,8 @@ class CMIP6DatasetAdapter(DatasetAdapter):
         :
             Data catalog containing the metadata for the dataset
         """
+        parsing_function = self.get_parsing_function()
+
         with warnings.catch_warnings():
             # Ignore the DeprecationWarning from xarray
             warnings.simplefilter("ignore", DeprecationWarning)
@@ -237,7 +183,7 @@ class CMIP6DatasetAdapter(DatasetAdapter):
                 depth=10,
                 include_patterns=["*.nc"],
                 joblib_parallel_kwargs={"n_jobs": self.n_jobs},
-            ).build(parsing_func=parse_cmip6)  # type: ignore
+            ).build(parsing_func=parsing_function)
 
         datasets: pd.DataFrame = builder.df.drop(["init_year"], axis=1)
 
@@ -254,6 +200,14 @@ class CMIP6DatasetAdapter(DatasetAdapter):
             lambda row: "CMIP6." + ".".join([row[item] for item in drs_items]), axis=1
         )
 
+        # Add in any missing metadata columns
+        missing_columns = set(self.dataset_specific_metadata + self.file_specific_metadata) - set(
+            datasets.columns
+        )
+        if missing_columns:
+            for column in missing_columns:
+                datasets[column] = pd.NA
+
         # Temporary fix for some datasets
         # TODO: Replace with a standalone package that contains metadata fixes for CMIP6 datasets
         datasets = _apply_fixes(datasets)

climate_ref/datasets/cmip6_parsers.py

@@ -0,0 +1,189 @@
+"""
+CMIP6 parser functions for extracting metadata from netCDF files
+
+Additional non-official DRS's may be added in the future.
+"""
+
+import traceback
+from typing import Any
+
+import xarray as xr
+from ecgtools.parsers.cmip import parse_cmip6_using_directories  # type: ignore
+from ecgtools.parsers.utilities import extract_attr_with_regex  # type: ignore
+from loguru import logger
+
+
+def _parse_daterange(date_range: str) -> tuple[str | None, str | None]:
+    """
+    Parse a date range string into start and end dates
+
+    The output from this is an estimated date range until the file is completely parsed.
+
+    Parameters
+    ----------
+    date_range
+        Date range string in the format "YYYYMM-YYYYMM"
+
+    Returns
+    -------
+    :
+        Tuple containing start and end dates as strings in the format "YYYY-MM-DD"
+    """
+    try:
+        start, end = date_range.split("-")
+        if len(start) != 6 or len(end) != 6:  # noqa: PLR2004
+            raise ValueError("Date range must be in the format 'YYYYMM-YYYYMM'")
+
+        start = f"{start[:4]}-{start[4:6]}-01"
+        # Up to the 30th of the month, assuming a 30-day month
+        # These values will be corrected later when the file is parsed
+        end = f"{end[:4]}-{end[4:6]}-30"
+
+        return start, end
+    except ValueError:
+        logger.error(f"Invalid date range format: {date_range}")
+        return None, None
+
+
+def parse_cmip6_complete(file: str, **kwargs: Any) -> dict[str, Any]:
+    """
+    Complete parser for CMIP6 files
+
+    This parser loads each file and extracts all available metadata.
+
+    For some filesystems this may be slow, as it involves a lot of I/O operations.
+
+    Parameters
+    ----------
+    file
+        File to parse
+    kwargs
+        Additional keyword arguments (not used, but required for compatibility)
+
+    Returns
+    -------
+    :
+        Dictionary with extracted metadata
+    """
+    keys = sorted(
+        {
+            "activity_id",
+            "branch_method",
+            "branch_time_in_child",
+            "branch_time_in_parent",
+            "experiment",
+            "experiment_id",
+            "frequency",
+            "grid",
+            "grid_label",
+            "institution_id",
+            "nominal_resolution",
+            "parent_activity_id",
+            "parent_experiment_id",
+            "parent_source_id",
+            "parent_time_units",
+            "parent_variant_label",
+            "realm",
+            "product",
+            "source_id",
+            "source_type",
+            "sub_experiment",
+            "sub_experiment_id",
+            "table_id",
+            "variable_id",
+            "variant_label",
+        }
+    )
+
+    try:
+        with xr.open_dataset(file, chunks={}, use_cftime=True) as ds:
+            info = {key: ds.attrs.get(key) for key in keys}
+            info["member_id"] = info["variant_label"]
+
+            variable_id = info["variable_id"]
+            if variable_id:  # pragma: no branch
+                attrs = ds[variable_id].attrs
+                for attr in ["standard_name", "long_name", "units"]:
+                    info[attr] = attrs.get(attr)
+
+            # Set the default of # of vertical levels to 1
+            vertical_levels = 1
+            start_time, end_time = None, None
+            init_year = None
+            try:
+                vertical_levels = ds[ds.cf["vertical"].name].size
+            except (KeyError, AttributeError, ValueError):
+                ...
+
+            try:
+                start_time, end_time = str(ds.cf["T"][0].data), str(ds.cf["T"][-1].data)
+            except (KeyError, AttributeError, ValueError):
+                ...
+            if info.get("sub_experiment_id"):  # pragma: no branch
+                init_year = extract_attr_with_regex(info["sub_experiment_id"], r"\d{4}")
+                if init_year:  # pragma: no cover
+                    init_year = int(init_year)
+            info["vertical_levels"] = vertical_levels
+            info["init_year"] = init_year
+            info["start_time"] = start_time
+            info["end_time"] = end_time
+            if not (start_time and end_time):
+                info["time_range"] = None
+            else:
+                info["time_range"] = f"{start_time}-{end_time}"
+        info["path"] = str(file)
+        info["version"] = extract_attr_with_regex(str(file), regex=r"v\d{4}\d{2}\d{2}|v\d{1}") or "v0"
+
+        # Mark the dataset as finalised
+        # This is used to indicate that the dataset has been fully parsed and is ready for use
+        info["finalised"] = True
+
+        return info
+
+    except Exception:
+        logger.exception(f"Failed to parse {file}")
+        return {"INVALID_ASSET": file, "TRACEBACK": traceback.format_exc()}
+
+
+def parse_cmip6_drs(file: str, **kwargs: Any) -> dict[str, Any]:
+    """
+    DRS parser for CMIP6 files
+
+    This parser extracts metadata according to the CMIP6 Data Reference Syntax (DRS).
+    This includes the essential metadata required to identify the dataset and is included in the filename.
+
+    Parameters
+    ----------
+    file
+        File to parse
+    kwargs
+        Additional keyword arguments (not used, but required for compatibility)
+
+    Returns
+    -------
+    :
+        Dictionary with extracted metadata
+    """
+    info: dict[str, Any] = parse_cmip6_using_directories(file)
+
+    if "INVALID_ASSET" in info:
+        logger.warning(f"Failed to parse {file}: {info['INVALID_ASSET']}")
+        return info
+
+    # The member_id is technically incorrect
+    # but for simplicity we are going to ignore sub-experiments for the DRS parser
+    info["variant_label"] = info["member_id"]
+
+    # Rename the `dcpp_init_year` key to `init_year` if it exists
+    if "dcpp_init_year" in info:
+        info["init_year"] = info.pop("dcpp_init_year")
+
+    if info.get("time_range"):
+        # Parse the time range if it exists
+        start_time, end_time = _parse_daterange(info["time_range"])
+        info["start_time"] = start_time
+        info["end_time"] = end_time
+
+    info["finalised"] = False
+
+    return info

climate_ref/datasets/obs4mips.py

@@ -15,8 +15,17 @@ from climate_ref.datasets.cmip6 import _parse_datetime
 from climate_ref.models.dataset import Dataset, Obs4MIPsDataset
 
 
-def parse_obs4mips(file: str) -> dict[str, Any | None]:
-    """Parser for obs4mips"""
+def parse_obs4mips(file: str, **kwargs: Any) -> dict[str, Any]:
+    """
+    Parser for obs4mips
+
+    Parameters
+    ----------
+    file
+        File to parse
+    kwargs
+        Additional keyword arguments (not used, but required for protocol compatibility)
+    """
     keys = sorted(
         list(
             {
@@ -106,6 +115,7 @@ class Obs4MIPsDatasetAdapter(DatasetAdapter):
 
     dataset_specific_metadata = (
         "activity_id",
+        "finalised",
         "frequency",
         "grid",
         "grid_label",
@@ -159,7 +169,7 @@ class Obs4MIPsDatasetAdapter(DatasetAdapter):
             depth=10,
             include_patterns=["*.nc"],
             joblib_parallel_kwargs={"n_jobs": self.n_jobs},
-        ).build(parsing_func=parse_obs4mips)  # type: ignore[arg-type]
+        ).build(parsing_func=parse_obs4mips)
 
         datasets = builder.df
         if datasets.empty:
@@ -178,4 +188,5 @@ class Obs4MIPsDatasetAdapter(DatasetAdapter):
         datasets["instance_id"] = datasets.apply(
             lambda row: "obs4MIPs." + ".".join([row[item] for item in drs_items]), axis=1
         )
+        datasets["finalised"] = True
         return datasets

climate_ref/executor/hpc.py

@@ -21,7 +21,7 @@ from loguru import logger
 from parsl import python_app
 from parsl.config import Config as ParslConfig
 from parsl.executors import HighThroughputExecutor
-from parsl.launchers import SrunLauncher
+from parsl.launchers import SimpleLauncher, SrunLauncher
 from parsl.providers import SlurmProvider
 from tqdm import tqdm
 
@@ -34,6 +34,7 @@ from climate_ref_core.exceptions import DiagnosticError, ExecutionError
 from climate_ref_core.executor import execute_locally
 
 from .local import ExecutionFuture, process_result
+from .pbs_scheduler import SmartPBSProvider
 
 
 @python_app
@@ -96,8 +97,9 @@ class HPCExecutor:
         self.account = str(executor_config.get("account", os.environ.get("USER")))
         self.username = executor_config.get("username", os.environ.get("USER"))
         self.partition = str(executor_config.get("partition")) if executor_config.get("partition") else None
+        self.queue = str(executor_config.get("queue")) if executor_config.get("queue") else None
         self.qos = str(executor_config.get("qos")) if executor_config.get("qos") else None
-        self.req_nodes = int(executor_config.get("req_nodes", 1))
+        self.req_nodes = int(executor_config.get("req_nodes", 1)) if self.scheduler == "slurm" else 1
         self.walltime = str(executor_config.get("walltime", "00:10:00"))
         self.log_dir = str(executor_config.get("log_dir", "runinfo"))
 
@@ -181,21 +183,47 @@ class HPCExecutor:
     def _initialize_parsl(self) -> None:
         executor_config = self.config.executor.config
 
-        provider = SlurmProvider(
-            account=self.account,
-            partition=self.partition,
-            qos=self.qos,
-            nodes_per_block=self.req_nodes,
-            max_blocks=int(executor_config.get("max_blocks", 1)),
-            scheduler_options=executor_config.get("scheduler_options", "#SBATCH -C cpu"),
-            worker_init=executor_config.get("worker_init", "source .venv/bin/activate"),
-            launcher=SrunLauncher(
-                debug=True,
-                overrides=executor_config.get("overrides", ""),
-            ),
-            walltime=self.walltime,
-            cmd_timeout=int(executor_config.get("cmd_timeout", 120)),
-        )
+        provider: SlurmProvider | SmartPBSProvider
+        if self.scheduler == "slurm":
+            provider = SlurmProvider(
+                account=self.account,
+                partition=self.partition,
+                qos=self.qos,
+                nodes_per_block=self.req_nodes,
+                max_blocks=int(executor_config.get("max_blocks", 1)),
+                scheduler_options=executor_config.get("scheduler_options", "#SBATCH -C cpu"),
+                worker_init=executor_config.get("worker_init", "source .venv/bin/activate"),
+                launcher=SrunLauncher(
+                    debug=True,
+                    overrides=executor_config.get("overrides", ""),
+                ),
+                walltime=self.walltime,
+                cmd_timeout=int(executor_config.get("cmd_timeout", 120)),
+            )
+
+        elif self.scheduler == "pbs":
+            provider = SmartPBSProvider(
+                account=self.account,
+                queue=self.queue,
+                worker_init=executor_config.get("worker_init", "source .venv/bin/activate"),
+                nodes_per_block=_to_int(executor_config.get("nodes_per_block", 1)),
+                cpus_per_node=_to_int(executor_config.get("cpus_per_node", None)),
+                ncpus=_to_int(executor_config.get("ncpus", None)),
+                mem=executor_config.get("mem", "4GB"),
+                jobfs=executor_config.get("jobfs", "10GB"),
+                storage=executor_config.get("storage", ""),
+                init_blocks=executor_config.get("init_blocks", 1),
+                min_blocks=executor_config.get("min_blocks", 0),
+                max_blocks=executor_config.get("max_blocks", 1),
+                parallelism=executor_config.get("parallelism", 1),
+                scheduler_options=executor_config.get("scheduler_options", ""),
+                launcher=SimpleLauncher(),
+                walltime=self.walltime,
+                cmd_timeout=int(executor_config.get("cmd_timeout", 120)),
+            )
+        else:
+            raise ValueError(f"Unsupported scheduler: {self.scheduler}")
+
         executor = HighThroughputExecutor(
             label="ref_hpc_executor",
             cores_per_worker=self.cores_per_worker if self.cores_per_worker else 1,
@@ -206,8 +234,11 @@ class HPCExecutor:
         )
 
         hpc_config = ParslConfig(
-            run_dir=self.log_dir, executors=[executor], retries=int(executor_config.get("retries", 2))
+            run_dir=self.log_dir,
+            executors=[executor],
+            retries=int(executor_config.get("retries", 2)),
         )
+
         parsl.load(hpc_config)
 
     def run(

climate_ref/executor/pbs_scheduler.py

@@ -0,0 +1,152 @@
+import re
+import shutil
+import subprocess
+import textwrap
+from typing import Any
+
+from parsl.launchers import SimpleLauncher
+from parsl.providers import PBSProProvider
+
+
+class SmartPBSProvider(PBSProProvider):
+    """
+    A PBSProProvider subclass that adapts to systems where `-l select` is not supported.
+
+    Falls back to individual resource requests (ncpus, mem, jobfs, storage) if needed.
+    """
+
+    def __init__(  # noqa: PLR0913
+        self,
+        account: str | None = None,
+        queue: str | None = None,
+        scheduler_options: str = "",
+        worker_init: str = "",
+        nodes_per_block: int | None = 1,
+        cpus_per_node: int | None = 1,
+        ncpus: int | None = None,
+        mem: str = "4GB",
+        jobfs: str = "10GB",
+        storage: str = "",
+        init_blocks: int = 1,
+        min_blocks: int = 0,
+        max_blocks: int = 1,
+        parallelism: int = 1,
+        launcher: SimpleLauncher = SimpleLauncher(),
+        walltime: str = "00:20:00",
+        cmd_timeout: int = 120,
+    ) -> None:
+        self.ncpus = ncpus
+        self.mem = mem
+        self.jobfs = jobfs
+        self.storage = storage
+        self._select_supported = self._detect_select_support()
+
+        # Prepare fallback resource dictionary
+        self._fallback_resources = {"mem": mem, "jobfs": jobfs, "storage": storage}
+
+        # Parse and strip select if present in scheduler_options
+        if not self._select_supported and "-l select=" in scheduler_options:
+            scheduler_options = self._parse_select_from_scheduler_options(scheduler_options)
+
+        # Determine fallback ncpus
+        if "ncpus" not in self._fallback_resources:
+            self._fallback_resources["ncpus"] = str(ncpus if ncpus is not None else (cpus_per_node or 1))
+
+        # Map ncpus to cpus_per_node if needed (select mode only)
+        if self._select_supported:
+            if not ncpus and cpus_per_node:
+                cpus_per_node = ncpus
+            elif ncpus and cpus_per_node and int(ncpus) != int(cpus_per_node):
+                print(f"Warning: ncpus={ncpus} and cpus_per_node={cpus_per_node} differ.")
+                print(f"Using cpus_per_node={cpus_per_node}.")
+        else:
+            cpus_per_node = int(self._fallback_resources["ncpus"])
+
+        super().__init__(
+            account=account,
+            queue=queue,
+            scheduler_options=scheduler_options,
+            select_options="",  # Not used; we handle resources ourselves
+            worker_init=worker_init,
+            nodes_per_block=nodes_per_block,
+            cpus_per_node=cpus_per_node,
+            init_blocks=init_blocks,
+            min_blocks=min_blocks,
+            max_blocks=max_blocks,
+            parallelism=parallelism,
+            launcher=launcher,
+            walltime=walltime,
+            cmd_timeout=cmd_timeout,
+        )  # type: ignore
+
+        if not self._select_supported:
+            self.template_string = self._fallback_template()
+
+    def _detect_select_support(self) -> bool:
+        """Detect whether `-l select` is supported by the underlying PBS system."""
+        qsub_path = shutil.which("qsub")
+        if qsub_path is None:
+            raise RuntimeError("qsub command not found. Ensure PBS is installed and in PATH.")
+
+        try:
+            result = subprocess.run(  # noqa: S603
+                [qsub_path, "-l", "wd,select=1:ncpus=1", "--version"],
+                capture_output=True,
+                timeout=5,
+                check=False,
+            )
+            stderr = result.stderr.decode().lower()
+            return "unknown" not in stderr and result.returncode == 0
+        except Exception:
+            return False
+
+    def _parse_select_from_scheduler_options(self, scheduler_options: str) -> str:
+        """
+        Parse `-l select=...` from scheduler_options and update fallback resources.
+
+        Removes the select line from scheduler_options.
+        """
+        select_pattern = r"-l\s+select=([^\s]+)"
+        match = re.search(select_pattern, scheduler_options)
+        if match:
+            select_string = match.group(1)
+            scheduler_options = re.sub(select_pattern, "", scheduler_options).strip()
+
+            parts = select_string.split(":")[1:]  # skip the initial `select=1`
+            for part in parts:
+                if "=" in part:
+                    key, val = part.split("=")
+                    self._fallback_resources[key.strip()] = val.strip()
+        return scheduler_options
+
+    def _fallback_template(self) -> str:
+        """Submit script template used if `select` is not supported."""
+        return textwrap.dedent("""\
+            #!/bin/bash
+            #PBS -N ${jobname}
+            #PBS -l ncpus=${ncpus}
+            #PBS -l mem=${mem}
+            #PBS -l jobfs=${jobfs}
+            #PBS -l walltime=${walltime}
+            #PBS -l storage=${storage}
+            #PBS -o ${job_stdout_path}
+            #PBS -e ${job_stderr_path}
+            ${scheduler_options}
+
+            ${worker_init}
+
+            export JOBNAME="${jobname}"
+            ${user_script}
+
+        """)
+
+    def _write_submit_script(
+        self, template: str, script_filename: str, job_name: str, configs: dict[str, Any]
+    ) -> str:
+        """Inject fallback values into the submit script if `select` is not supported."""
+        if not self._select_supported:
+            configs.setdefault("ncpus", self._fallback_resources.get("ncpus", "1"))
+            configs.setdefault("mem", self._fallback_resources.get("mem", "4GB"))
+            configs.setdefault("jobfs", self._fallback_resources.get("jobfs", "10GB"))
+            configs.setdefault("storage", self._fallback_resources.get("storage", "gdata1"))
+        return super()._write_submit_script(template, script_filename, job_name, configs)  # type: ignore

climate_ref/migrations/versions/2025-07-20T1521_94beace57a9c_cmip6_finalised.py

@@ -0,0 +1,57 @@
+"""cmip6-finalised
+
+Revision ID: 94beace57a9c
+Revises: 795c1e6cf496
+Create Date: 2025-07-20 15:21:17.132458
+
+"""
+
+from collections.abc import Sequence
+from typing import Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "94beace57a9c"
+down_revision: Union[str, None] = "795c1e6cf496"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("cmip6_dataset", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("finalised", sa.Boolean(), nullable=False))
+        batch_op.alter_column("experiment", existing_type=sa.VARCHAR(), nullable=True)
+        batch_op.alter_column("frequency", existing_type=sa.VARCHAR(), nullable=True)
+        batch_op.alter_column("grid", existing_type=sa.VARCHAR(), nullable=True)
+        batch_op.alter_column("nominal_resolution", existing_type=sa.VARCHAR(), nullable=True)
+        batch_op.alter_column("realm", existing_type=sa.VARCHAR(), nullable=True)
+        batch_op.alter_column("product", existing_type=sa.VARCHAR(), nullable=True)
+        batch_op.alter_column("standard_name", existing_type=sa.VARCHAR(), nullable=True)
+        batch_op.alter_column("source_type", existing_type=sa.VARCHAR(), nullable=True)
+        batch_op.alter_column("sub_experiment", existing_type=sa.VARCHAR(), nullable=True)
+        batch_op.alter_column("sub_experiment_id", existing_type=sa.VARCHAR(), nullable=True)
+        batch_op.alter_column("units", existing_type=sa.VARCHAR(), nullable=True)
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("cmip6_dataset", schema=None) as batch_op:
+        batch_op.alter_column("units", existing_type=sa.VARCHAR(), nullable=False)
+        batch_op.alter_column("sub_experiment_id", existing_type=sa.VARCHAR(), nullable=False)
+        batch_op.alter_column("sub_experiment", existing_type=sa.VARCHAR(), nullable=False)
+        batch_op.alter_column("source_type", existing_type=sa.VARCHAR(), nullable=False)
+        batch_op.alter_column("standard_name", existing_type=sa.VARCHAR(), nullable=False)
+        batch_op.alter_column("product", existing_type=sa.VARCHAR(), nullable=False)
+        batch_op.alter_column("realm", existing_type=sa.VARCHAR(), nullable=False)
+        batch_op.alter_column("nominal_resolution", existing_type=sa.VARCHAR(), nullable=False)
+        batch_op.alter_column("grid", existing_type=sa.VARCHAR(), nullable=False)
+        batch_op.alter_column("frequency", existing_type=sa.VARCHAR(), nullable=False)
+        batch_op.alter_column("experiment", existing_type=sa.VARCHAR(), nullable=False)
+        batch_op.drop_column("finalised")
+
+    # ### end Alembic commands ###

climate_ref/migrations/versions/2025-08-05T0327_a1b2c3d4e5f6_finalised_on_base_dataset.py

@@ -0,0 +1,57 @@
+"""finalised-on-base-dataset
+
+Move finalised from cmip6_dataset to base dataset table and default all existing rows to True.
+
+Revision ID: a1b2c3d4e5f6
+Revises: 94beace57a9c
+Create Date: 2025-08-05 03:27:00
+
+"""
+
+from collections.abc import Sequence
+from typing import Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "ba5e"
+down_revision: Union[str, None] = "94beace57a9c"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Add finalised to base dataset with default True, non-null
+    with op.batch_alter_table("dataset", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("finalised", sa.Boolean(), nullable=True, server_default=sa.text("true"))
+        )
+
+    # Backfill: ensure all existing rows are True
+    op.execute("UPDATE dataset SET finalised = TRUE WHERE finalised IS NULL")
+
+    # Enforce NOT NULL after backfill
+    with op.batch_alter_table("dataset", schema=None) as batch_op:
+        batch_op.alter_column("finalised", nullable=False)
+
+    # Drop column from cmip6_dataset if it exists
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    cmip6_cols = {col["name"] for col in inspector.get_columns("cmip6_dataset")}
+    if "finalised" in cmip6_cols:
+        with op.batch_alter_table("cmip6_dataset", schema=None) as batch_op:
+            batch_op.drop_column("finalised")
+
+
+def downgrade() -> None:
+    # Re-create cmip6_dataset.finalised as non-nullable boolean default False
+    # Note: Original migration 94beace57a9c added cmip6_dataset.finalised NOT NULL, with no default.
+    with op.batch_alter_table("cmip6_dataset", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("finalised", sa.Boolean(), nullable=False, server_default=sa.text("false"))
+        )
+
+    # Drop base dataset finalised
+    with op.batch_alter_table("dataset", schema=None) as batch_op:
+        batch_op.drop_column("finalised")

climate_ref/models/dataset.py

@@ -45,6 +45,16 @@ class Dataset(Base):
     Updating a dataset will trigger a new diagnostic calculation.
     """
 
+    # Universal finalisation flag for all dataset types
+    # Only CMIP6 currently uses unfinalised datasets in practice; other types should be finalised on creation.
+    finalised: Mapped[bool] = mapped_column(default=True, nullable=False)
+    """
+    Whether the complete set of metadata for the dataset has been finalised.
+
+    For CMIP6, ingestion may initially create unfinalised datasets (False) until all metadata is extracted.
+    For other dataset types (e.g., obs4MIPs, PMP climatology), this should be True upon creation.
+    """
+
     def __repr__(self) -> str:
         return f"<Dataset slug={self.slug} dataset_type={self.dataset_type} >"
 
@@ -90,9 +100,7 @@ class CMIP6Dataset(Dataset):
     """
     Represents a CMIP6 dataset
 
-    Fields that are not marked as required in
-    https://wcrp-cmip.github.io/WGCM_Infrastructure_Panel/Papers/CMIP6_global_attributes_filenames_CVs_v6.2.7.pdf
-    are optional.
+    Fields that are not in the DRS are marked optional.
     """
 
     __tablename__ = "cmip6_dataset"
@@ -102,29 +110,29 @@ class CMIP6Dataset(Dataset):
     branch_method: Mapped[str] = mapped_column(nullable=True)
     branch_time_in_child: Mapped[float] = mapped_column(nullable=True)
     branch_time_in_parent: Mapped[float] = mapped_column(nullable=True)
-    experiment: Mapped[str] = mapped_column()
+    experiment: Mapped[str] = mapped_column(nullable=True)
     experiment_id: Mapped[str] = mapped_column()
-    frequency: Mapped[str] = mapped_column()
-    grid: Mapped[str] = mapped_column()
+    frequency: Mapped[str] = mapped_column(nullable=True)
+    grid: Mapped[str] = mapped_column(nullable=True)
     grid_label: Mapped[str] = mapped_column()
     institution_id: Mapped[str] = mapped_column()
     long_name: Mapped[str] = mapped_column(nullable=True)
     member_id: Mapped[str] = mapped_column()
-    nominal_resolution: Mapped[str] = mapped_column()
+    nominal_resolution: Mapped[str] = mapped_column(nullable=True)
     parent_activity_id: Mapped[str] = mapped_column(nullable=True)
     parent_experiment_id: Mapped[str] = mapped_column(nullable=True)
     parent_source_id: Mapped[str] = mapped_column(nullable=True)
     parent_time_units: Mapped[str] = mapped_column(nullable=True)
     parent_variant_label: Mapped[str] = mapped_column(nullable=True)
-    realm: Mapped[str] = mapped_column()
-    product: Mapped[str] = mapped_column()
+    realm: Mapped[str] = mapped_column(nullable=True)
+    product: Mapped[str] = mapped_column(nullable=True)
     source_id: Mapped[str] = mapped_column()
-    standard_name: Mapped[str] = mapped_column()
-    source_type: Mapped[str] = mapped_column()
-    sub_experiment: Mapped[str] = mapped_column()
-    sub_experiment_id: Mapped[str] = mapped_column()
+    standard_name: Mapped[str] = mapped_column(nullable=True)
+    source_type: Mapped[str] = mapped_column(nullable=True)
+    sub_experiment: Mapped[str] = mapped_column(nullable=True)
+    sub_experiment_id: Mapped[str] = mapped_column(nullable=True)
     table_id: Mapped[str] = mapped_column()
-    units: Mapped[str] = mapped_column()
+    units: Mapped[str] = mapped_column(nullable=True)
     variable_id: Mapped[str] = mapped_column()
     variant_label: Mapped[str] = mapped_column()
     vertical_levels: Mapped[int] = mapped_column(nullable=True)
@@ -132,7 +140,7 @@ class CMIP6Dataset(Dataset):
 
     instance_id: Mapped[str] = mapped_column()
     """
-    Unique identifier for the dataset.
+    Unique identifier for the dataset (including the version).
     """
 
     __mapper_args__: ClassVar[Any] = {"polymorphic_identity": SourceDatasetType.CMIP6}  # type: ignore

{climate_ref-0.6.4.dist-info → climate_ref-0.6.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: climate-ref
-Version: 0.6.4
+Version: 0.6.5
 Summary: Application which runs the CMIP Rapid Evaluation Framework
 Author-email: Jared Lewis <jared.lewis@climate-resource.com>, Mika Pflueger <mika.pflueger@climate-resource.com>, Bouwe Andela <b.andela@esciencecenter.nl>, Jiwoo Lee <lee1043@llnl.gov>, Min Xu <xum1@ornl.gov>, Nathan Collier <collierno@ornl.gov>, Dora Hegedus <dora.hegedus@stfc.ac.uk>
 License-Expression: Apache-2.0

{climate_ref-0.6.4.dist-info → climate_ref-0.6.5.dist-info}/RECORD

@@ -1,7 +1,7 @@
 climate_ref/__init__.py,sha256=M45QGfl0KCPK48A8MjI08weNvZHMYH__GblraQMxsoM,808
 climate_ref/_config_helpers.py,sha256=-atI5FX7SukhLE_jz_rL-EHQ7s0YYqKu3dSFYWxSyMU,6632
 climate_ref/alembic.ini,sha256=WRvbwSIFuZ7hWNMnR2-yHPJAwYUnwhvRYBzkJhtpGdg,3535
-climate_ref/config.py,sha256=4jlqYnR5es9fmKI3sfT9snnutH342o9XAMf6ipNS2lI,17083
+climate_ref/config.py,sha256=WW6R7RLwEDuI11XYLYO57FwvmQz1psq9bNM3WVL3e_s,17481
 climate_ref/constants.py,sha256=9RaNLgUSuQva7ki4eRW3TjOKeVP6T81QNiu0veB1zVk,111
 climate_ref/database.py,sha256=b_6XHdr78Mo7KeLqQJ5DjLsySHPdQE83P8dRpdMfzfM,8661
 climate_ref/provider_registry.py,sha256=dyfj4vU6unKHNXtT03HafQtAi3LilL37uvu3paCnmNY,4159
@@ -19,14 +19,16 @@ climate_ref/cli/solve.py,sha256=ZTXrwDFDXNrX5GLMJTN9tFnpV3zlcZbEu2aF3JDJVxI,2367
 climate_ref/dataset_registry/obs4ref_reference.txt,sha256=2zJMbsAsQ49KaWziX3CqrlILq9yN7S2ygmfV3V5rsnw,8395
 climate_ref/dataset_registry/sample_data.txt,sha256=3JAHy14pRbLlo9-oNxUXLgZ_QOFJXUieEftBbapSY8E,20124
 climate_ref/datasets/__init__.py,sha256=PV3u5ZmhyfcHbKqySgwVA8m4-naZgxzydLXSBqdTGLM,1171
-climate_ref/datasets/base.py,sha256=yoip8UCcTCUPn2xVlsJ1If9zXw_476dDYViH5iMgcIE,10352
-climate_ref/datasets/cmip6.py,sha256=3MVJ1kPdw6f6V3G4gdHIiqDGUyMqPs-_wttkw2YKAH0,8425
-climate_ref/datasets/obs4mips.py,sha256=CmMm4kopfb0yFsMSgUlHUm8clGJImBaodSkh6lAv_Ug,5926
+climate_ref/datasets/base.py,sha256=uZ55u625ckRNjsn-AqJg4_xO5uvHchqYvwBZIt4iHtY,11017
+climate_ref/datasets/cmip6.py,sha256=KO761ConHvX40n9X0xLrxjhzN7wmighNWL2JyYygRAA,7049
+climate_ref/datasets/cmip6_parsers.py,sha256=wH4WKQAR2_aniXwsW7nch6nIpXk2pSpPxkT4unjV4hQ,6041
+climate_ref/datasets/obs4mips.py,sha256=q0_erQb4k5KBaGMvEGgUtVSDvXQjuftqDmvW4QZpWZI,6138
 climate_ref/datasets/pmp_climatology.py,sha256=goHDc_3B2Wdiy_hmpERNvWDdDYZACPOyFDt3Du6nGc0,534
 climate_ref/datasets/utils.py,sha256=iLJO7h4G3DWsRe9hIC4qkIyi5_zIW1ZMw-FDASLujtM,359
 climate_ref/executor/__init__.py,sha256=PYtJs3oBS_GiUHbt8BF-6wJibpF6_vREm1Cg9TxVbLI,648
-climate_ref/executor/hpc.py,sha256=4o90sCyoC4jlkem3BXNo4uwFZpIvOUGfrqYucB6EtU8,12251
+climate_ref/executor/hpc.py,sha256=ZhGtzM0skH_ojnkSc6UNYIetXoyBRCwfXJusuezBZGw,13876
 climate_ref/executor/local.py,sha256=65LUl41YtURFb87YTWZQHjDpIRlIKJ5Ny51c9DZjy0s,8582
+climate_ref/executor/pbs_scheduler.py,sha256=WoH1sTmDl7bdmYodpcxZjkUSvInYUcWR4x7buIgBxqk,5807
 climate_ref/executor/result_handling.py,sha256=i7ZMX5vvyPY5gW-WWd-JHLi1BLviB9FXhn4FE8C9d4w,7787
 climate_ref/executor/synchronous.py,sha256=o4TndsoKMu9AzJYLkusU9lRkgHCy6HcCP46tEs6o86U,1895
 climate_ref/migrations/README,sha256=xM5osYbyEbEFA2eh5kwary_oh-5VFWtDubA-vgWwvlE,935
@@ -35,16 +37,18 @@ climate_ref/migrations/script.py.mako,sha256=MEqL-2qATlST9TAOeYgscMn1uy6HUS9NFvD
 climate_ref/migrations/versions/2025-05-02T1418_341a4aa2551e_regenerate.py,sha256=S8Q4THCI4TPnlaQHgQJUCiNW5LAyQClaiTB-0dwhtXU,14050
 climate_ref/migrations/versions/2025-05-09T2032_03dbb4998e49_series_metric_value.py,sha256=s9nZ_l64pSF7sWN53rRPCQlqW_xHqR8tlWhU-ovmsME,2043
 climate_ref/migrations/versions/2025-07-03T1505_795c1e6cf496_drop_unique_requirement_on_slug.py,sha256=TfBHJkm3oPlz0P5Z1tiY6LBp2B1oDvdyL_OOYoV-OiI,984
+climate_ref/migrations/versions/2025-07-20T1521_94beace57a9c_cmip6_finalised.py,sha256=NSCMMV65v48B8_OoEf4X4bRthAlhzbDo0UlC6nqW3qs,2908
+climate_ref/migrations/versions/2025-08-05T0327_a1b2c3d4e5f6_finalised_on_base_dataset.py,sha256=G-SZKdU9dx9WyMh4JLwPKcud4gtFrxu-tULXG9vXGAU,2034
 climate_ref/models/__init__.py,sha256=rUDKRANeAEAHVOrzJVIZoZ99dDG5O4AGzHmOpC876Nc,801
 climate_ref/models/base.py,sha256=YMyovT2Z_tRv59zz6qC9YCCDodhO3x6OLnFdBtPJkho,1271
-climate_ref/models/dataset.py,sha256=Rpwrx0HqOJBHs4Sb4n6B0In__Uo0PqXSZKvZR-juGCg,7491
+climate_ref/models/dataset.py,sha256=in9FNLR4K_bpVSlWlk6A6IyFtkFy2v8ZFNcDXbwSMWI,8078
 climate_ref/models/diagnostic.py,sha256=0mKVvASEWNxx41R2Y-5VxplarZ4JAP6q0oaO14FKZuk,1751
 climate_ref/models/execution.py,sha256=lRCpaKLSR7rZbuoL94GW76tm9wLMsSDoIOA7bIa6xgY,9848
 climate_ref/models/metric_value.py,sha256=44OLcZz-qLx-p_9w7YWDKpD5S7Y9HyTKKsvSb77RBro,10190
 climate_ref/models/provider.py,sha256=RAE2qAAxwObu-72CdK4kt5ACMmKYEn07WJm7DU9hF28,990
-climate_ref-0.6.4.dist-info/METADATA,sha256=2SVSEweQ-ZaXnnpymVG9iBc8ImUBwQwnVF9Y2-0tJz8,4505
-climate_ref-0.6.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-climate_ref-0.6.4.dist-info/entry_points.txt,sha256=IaggEJlDIhoYWXdXJafacWbWtCcoEqUKceP1qD7_7vU,44
-climate_ref-0.6.4.dist-info/licenses/LICENCE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-climate_ref-0.6.4.dist-info/licenses/NOTICE,sha256=4qTlax9aX2-mswYJuVrLqJ9jK1IkN5kSBqfVvYLF3Ws,128
-climate_ref-0.6.4.dist-info/RECORD,,
+climate_ref-0.6.5.dist-info/METADATA,sha256=uKVSOC5iN1SGV3eoQ9uClB1UC_FpDbbM2ArYL0EHd0U,4505
+climate_ref-0.6.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+climate_ref-0.6.5.dist-info/entry_points.txt,sha256=IaggEJlDIhoYWXdXJafacWbWtCcoEqUKceP1qD7_7vU,44
+climate_ref-0.6.5.dist-info/licenses/LICENCE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+climate_ref-0.6.5.dist-info/licenses/NOTICE,sha256=4qTlax9aX2-mswYJuVrLqJ9jK1IkN5kSBqfVvYLF3Ws,128
+climate_ref-0.6.5.dist-info/RECORD,,