climate-ref-pmp 0.5.0__py3-none-any.whl

climate_ref_pmp/params/pmp_param_MoV-ts.py

@@ -0,0 +1,94 @@
+import datetime
+import os
+
+# =================================================
+# Background Information
+# -------------------------------------------------
+mip = "cmip6"
+exp = "historical"
+frequency = "mo"
+realm = "atm"
+
+# =================================================
+# Analysis Options
+# -------------------------------------------------
+variability_mode = "PDO"  # Available domains: NAM, NAO, SAM, PNA, PDO
+seasons = ["monthly"]  # Available seasons: DJF, MAM, JJA, SON, monthly, yearly
+
+landmask = True  # Maskout land region thus consider only ocean grid (default=False)
+
+ConvEOF = True  # Calculate conventioanl EOF for model
+CBF = True  # Calculate Common Basis Function (CBF) for model
+
+# =================================================
+# Miscellaneous
+# -------------------------------------------------
+update_json = False  # False
+debug = False  # False
+
+# =================================================
+# Observation
+# -------------------------------------------------
+reference_data_name = "HadISSTv1.1"
+reference_data_path = (
+    "/p/user_pub/PCMDIobs/obs4MIPs/MOHC/HadISST-1-1/"
+    + "mon/ts/gn/v20210727/ts_mon_HadISST-1-1_PCMDI_gn_187001-201907.nc"
+)
+
+# varOBS = "sst"
+# ObsUnitsAdjust = (False, 0, 0)  # degC
+varOBS = "ts"
+ModUnitsAdjust = (True, "subtract", 273.15)  # degK to degC
+
+osyear = 1900
+oeyear = 2005
+eofn_obs = 1
+
+# =================================================
+# Models
+# -------------------------------------------------
+modpath = os.path.join(
+    "/p/css03/cmip5_css02/data/cmip5/output1/CSIRO-BOM/ACCESS1-0/historical/mon/atmos/Amon/r1i1p1/ts/1/",
+    "ts_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc",
+)
+
+modpath_lf = os.path.join(
+    "/p/css03/cmip5_css02/data/cmip5/output1/CSIRO-BOM/ACCESS1-0/amip/fx/atmos/fx/r0i0p0/sftlf/1/",
+    "sftlf_fx_ACCESS1-0_amip_r0i0p0.nc",
+)
+
+modnames = ["ACCESS1-0"]
+
+realization = "r1i1p1f1"
+
+varModel = "ts"
+ModUnitsAdjust = (True, "subtract", 273.15)  # degK to degC
+
+msyear = 1900
+meyear = 2005
+eofn_mod = 1
+
+# =================================================
+# Output
+# -------------------------------------------------
+case_id = f"{datetime.datetime.now():v%Y%m%d}"
+pmprdir = "/p/user_pub/pmp/pmp_results/"
+
+results_dir = os.path.join(
+    pmprdir,
+    "%(output_type)",
+    "variability_modes",
+    "%(mip)",
+    "%(exp)",
+    "%(case_id)",
+    "%(variability_mode)",
+    "%(reference_data_name)",
+)
+
+# Output for obs
+plot_obs = True  # Create map graphics
+nc_out_obs = True  # Write output in NetCDF
+
+# Output for models
+nc_out = True
+plot = True

climate_ref_pmp/params/pmp_param_annualcycle_1-clims.py

@@ -0,0 +1,21 @@
+#
+#  OPTIONS ARE SET BY USER IN THIS FILE AS INDICATED BELOW BY:
+#
+#
+
+# VARIABLES TO USE
+# vars = ["rlut"]
+
+# START AND END DATES FOR CLIMATOLOGY
+start = "1981-01"
+end = "2005-12"
+
+# INPUT DATASET - CAN BE MODEL OR OBSERVATIONS
+infile = (
+    "obs4MIPs_PCMDI_monthly/NASA-LaRC/CERES-EBAF-4-1"
+    "/mon/rlut/gn/v20210727"
+    "/rlut_mon_CERES-EBAF-4-1_PCMDI_gn_200301-201812.nc"
+)
+
+# DIRECTORY WHERE TO PUT RESULTS
+outfile = "climo/rlut_mon_CERES-EBAF-4-1_BE_gn.nc"

climate_ref_pmp/params/pmp_param_annualcycle_2-metrics.py

@@ -0,0 +1,54 @@
+import os
+
+#
+#  OPTIONS ARE SET BY USER IN THIS FILE AS INDICATED BELOW BY:
+#
+#
+
+# RUN IDENTIFICATION
+# DEFINES A SUBDIRECTORY TO METRICS OUTPUT RESULTS SO MULTIPLE CASES CAN
+# BE COMPARED
+case_id = "basicTest"
+
+# LIST OF MODEL VERSIONS TO BE TESTED - WHICH ARE EXPECTED TO BE PART OF
+# CLIMATOLOGY FILENAME
+# test_data_set = ["ACCESS1-0", "CanCM4"]
+
+
+# VARIABLES TO USE
+# vars = ["rlut"]
+
+
+# Observations to use at the moment "default" or "alternate"
+reference_data_set = ["all"]
+# ext = '.nc'
+
+# INTERPOLATION OPTIONS
+target_grid = "2.5x2.5"  # OPTIONS: '2.5x2.5' or an actual cdms2 grid object
+regrid_tool = "regrid2"  # 'regrid2' # OPTIONS: 'regrid2','esmf'
+# -- OPTIONS: 'linear','conservative', only if tool is esmf
+regrid_method = "linear"
+regrid_tool_ocn = "esmf"  # OPTIONS: "regrid2","esmf"
+# -- OPTIONS: 'linear','conservative', only if tool is esmf
+regrid_method_ocn = "linear"
+
+# Templates for climatology files
+# %(param) will subsitute param with values in this file
+filename_template = "cmip5.historical.%(model_version).r1i1p1.mon.%(variable).198101-200512.AC.v20200426.nc"
+
+# filename template for landsea masks ('sftlf')
+sftlf_filename_template = "sftlf_%(model_version).nc"
+generate_sftlf = True  # if land surface type mask cannot be found, generate one
+
+# Region
+regions = {"rlut": ["Global"]}
+
+# ROOT PATH FOR MODELS CLIMATOLOGIES
+test_data_path = "demo_data_tmp/CMIP5_demo_clims/"
+# ROOT PATH FOR OBSERVATIONS
+# Note that atm/mo/%(variable)/ac will be added to this
+reference_data_path = ""
+custom_observations = "obs_dict.json"
+
+# DIRECTORY WHERE TO PUT RESULTS
+metrics_output_path = os.path.join("demo_output_tmp", "%(case_id)")

climate_ref_pmp/pmp_driver.py

@@ -0,0 +1,258 @@
+import importlib.metadata
+import importlib.resources
+import json
+import os
+import pathlib
+from typing import Any
+
+from loguru import logger
+from rich.pretty import pretty_repr
+
+from climate_ref_core.pycmec.metric import CMECMetric
+from climate_ref_core.pycmec.output import CMECOutput
+
+
+def _remove_nested_key(data: dict[str, Any], key: str) -> dict[str, Any]:
+    """
+    Remove a nested key from a dictionary
+
+    Parameters
+    ----------
+    data
+        Dictionary to remove the key from
+    key
+        Key to remove
+
+    Returns
+    -------
+        The dictionary with the key removed
+    """
+    if key in data:
+        data.pop(key)
+    for k, v in data.items():
+        if isinstance(v, dict):
+            data[k] = _remove_nested_key(v, key)
+    return data
+
+
+def process_json_result(
+    json_filename: pathlib.Path, png_files: list[pathlib.Path], data_files: list[pathlib.Path]
+) -> tuple[CMECOutput, CMECMetric]:
+    """
+    Process a PMP JSON result into the appropriate CMEC bundles
+
+    Parameters
+    ----------
+    json_filename
+        Filename of the JSON file that is written out by PMP
+    png_files
+        List of PNG files to be included in the output
+    data_files
+        List of data files to be included in the output
+
+    Returns
+    -------
+        tuple of CMEC output and diagnostic bundles
+    """
+    with open(json_filename) as fh:
+        json_result = json.load(fh)
+
+    cmec_output = CMECOutput.create_template()
+    cmec_output["provenance"] = {**cmec_output["provenance"], **json_result["provenance"]}
+
+    # Add the plots and data files
+    for fname in png_files:
+        cmec_output["plots"][fname.name] = {
+            "filename": str(fname),
+            "long_name": "Plot",
+            "description": "Plot produced by the diagnostic",
+        }
+    for fname in data_files:
+        cmec_output["data"][fname.name] = {
+            "filename": str(fname),
+            "long_name": "Output data",
+            "description": "Data produced by the diagnostic",
+        }
+
+    cmec_metric = CMECMetric.create_template()
+    cmec_metric["DIMENSIONS"] = {}
+    dimensions = json_result["DIMENSIONS"]
+
+    if "dimensions" in dimensions:  # pragma: no branch
+        # Merge the contents of inner "dimensions" into the parent "DIMENSIONS"
+        dimensions.update(dimensions["dimensions"])
+        del dimensions["dimensions"]
+
+    if "statistic" in dimensions["json_structure"]:  # pragma: no branch
+        dimensions["json_structure"].remove("statistic")
+        dimensions.pop("statistic")
+
+    # Remove the "attributes" key from the RESULTS
+    # This isn't standard CMEC output, but it is what PMP produces
+    results = json_result["RESULTS"]
+
+    cmec_metric["RESULTS"] = results
+    cmec_metric["DIMENSIONS"] = dimensions
+
+    if "provenance" in json_result:  # pragma: no branch
+        cmec_metric["provenance"] = json_result["provenance"]
+
+    logger.info(f"cmec_output: {pretty_repr(cmec_output)}")
+    logger.info(f"cmec_metric: {pretty_repr(cmec_metric)}")
+
+    return CMECOutput(**cmec_output), CMECMetric(**cmec_metric)
+
+
+def _get_resource(package: str, resource_name: str | pathlib.Path, use_resources: bool) -> str:
+    """
+    Get the path to a resource within the pcmdi_metric package without importing.
+
+    Parameters
+    ----------
+    package: str
+        Python package name if use_resources is True, otherwise the distribution name
+        (the pypi package name).
+    resource_name : str
+        The resource path relative to the package.
+    use_resources : bool
+        If True, use the importlib.resources API, otherwise use importlib.metadata.
+
+        importlib.resources is the preferred way to access resources because it handles
+        packages which have been editably installed,
+        but it implictly imports the package.
+
+        Whereas `importlib.metadata` uses the package metadata to resolve the location of a resource.
+
+    Returns
+    -------
+        The full path to the target resource.
+    """
+    if use_resources:
+        resource_path = str(importlib.resources.files(package) / str(resource_name))
+    else:
+        distribution = importlib.metadata.distribution(package)
+        resource_path = str(distribution.locate_file(pathlib.Path(package) / resource_name))
+    if not pathlib.Path(resource_path).exists():
+        raise FileNotFoundError(f"Resource {resource_name} not found in {package} package.")
+    return str(resource_path)
+
+
+def build_pmp_command(
+    driver_file: str,
+    parameter_file: str,
+    **kwargs: dict[str, str | int | float | list[str]],
+) -> list[str]:
+    """
+    Run a PMP driver script via a conda environment
+
+    This function runs a PMP driver script using a specific conda environment.
+    The driver script is responsible for running the PMP diagnostics and producing output.
+    The output consists of a JSON file that contains the executions of the PMP diagnostics,
+    and a set of PNG and data files that are produced by the diagnostics.
+
+    Parameters
+    ----------
+    driver_file
+        Filename of the PMP driver script to run
+    parameter_file
+        Filename of the parameter file to use
+    kwargs
+        Additional arguments to pass to the driver script
+    """
+    # Note this uses the driver script from the REF env *not* the PMP conda env
+    _driver_script = _get_resource("pcmdi_metrics", driver_file, use_resources=False)
+    _parameter_file = _get_resource("climate_ref_pmp.params", parameter_file, use_resources=True)
+
+    # Run the driver script inside the PMP conda environment
+    cmd = [
+        "python",
+        _driver_script,
+        "-p",
+        _parameter_file,
+    ]
+
+    # Loop through additional arguments if they exist
+    if kwargs:  # pragma: no cover
+        for key, value in kwargs.items():
+            if value:
+                cmd.extend([f"--{key}", str(value)])
+            else:
+                cmd.extend([f"--{key}"])
+
+    logger.info("-- PMP command to run --")
+    logger.info("[PMP] Command to run:", " ".join(map(str, cmd)))
+    logger.info("[PMP] Command generation for the driver completed.")
+
+    return cmd
+
+
+def build_glob_pattern(paths: list[str]) -> str:
+    """
+    Generate a glob pattern that matches files based on common path, prefix, and suffix.
+
+    Generate a glob pattern that matches all files in the given list of paths,
+    based on their common directory, filename prefix, and suffix.
+
+    Parameters
+    ----------
+    paths : list of str
+        A list of full file paths. The paths should point to actual files,
+        and should have enough similarity in their structure and naming
+        to extract common patterns.
+
+    Returns
+    -------
+    str
+        A glob pattern string that can be used with `glob.glob(pattern, recursive=True)`
+        to match all the provided files and others with the same structural pattern.
+
+    Examples
+    --------
+    >>> paths = [
+    ...     "/home/user/data/folder1/file1.txt",
+    ...     "/home/user/data/folder1/file2.txt",
+    ...     "/home/user/data/folder2/file3.txt",
+    ... ]
+    >>> pattern = build_glob_pattern(paths)
+    >>> print(pattern)
+    /home/user/data/**/file*.txt
+    """
+    if not paths:
+        raise ValueError("The path list is empty.")
+
+    # Find the common directory path
+    common_path = os.path.commonpath(paths)
+
+    # Extract filenames and parent directories
+    filenames = [os.path.basename(path) for path in paths]
+    dirnames = [os.path.dirname(path) for path in paths]
+    same_dir = all(d == dirnames[0] for d in dirnames)
+
+    # Helper to find common prefix
+    def common_prefix(strings: list[str]) -> str:
+        if not strings:
+            return ""
+        prefix = strings[0]
+        for s in strings[1:]:
+            while not s.startswith(prefix):
+                prefix = prefix[:-1]
+                if not prefix:
+                    break
+        return prefix
+
+    # Helper to find common suffix
+    def common_suffix(strings: list[str]) -> str:
+        reversed_strings = [s[::-1] for s in strings]
+        reversed_suffix = common_prefix(reversed_strings)
+        return reversed_suffix[::-1]
+
+    prefix = common_prefix(filenames)
+    suffix = common_suffix(filenames)
+
+    # Use simpler pattern if all files are in the same directory
+    if same_dir:
+        pattern = os.path.join(dirnames[0], f"{prefix}*{suffix}")
+    else:
+        pattern = os.path.join(common_path, "**", f"{prefix}*{suffix}")
+
+    return pattern