climate-ref-ilamb 0.5.4__py3-none-any.whl → 0.5.5__py3-none-any.whl

climate_ref_ilamb/__init__.py

@@ -10,9 +10,8 @@ import importlib.resources
 
 import yaml
 
-from climate_ref_core.dataset_registry import dataset_registry_manager
+from climate_ref_core.dataset_registry import DATASET_URL, dataset_registry_manager
 from climate_ref_core.providers import DiagnosticProvider
-from climate_ref_ilamb.datasets import ILAMB_DATA_VERSION
 from climate_ref_ilamb.standard import ILAMBStandard
 
 __version__ = importlib.metadata.version("climate-ref-ilamb")
@@ -22,27 +21,21 @@ provider = DiagnosticProvider("ILAMB", __version__)
 # Register some datasets
 dataset_registry_manager.register(
     "ilamb-test",
-    base_url="https://www.ilamb.org/ILAMB-Data/DATA",
+    base_url=DATASET_URL,
     package="climate_ref_ilamb.dataset_registry",
     resource="test.txt",
-    cache_name="ilamb3",
-    version=ILAMB_DATA_VERSION,
 )
 dataset_registry_manager.register(
     "ilamb",
-    base_url="https://www.ilamb.org/ILAMB-Data/DATA",
+    base_url=DATASET_URL,
     package="climate_ref_ilamb.dataset_registry",
     resource="ilamb.txt",
-    cache_name="ilamb3",
-    version=ILAMB_DATA_VERSION,
 )
 dataset_registry_manager.register(
     "iomb",
-    base_url="https://www.ilamb.org/ilamb3-data/",
+    base_url=DATASET_URL,
     package="climate_ref_ilamb.dataset_registry",
     resource="iomb.txt",
-    cache_name="ilamb3",
-    version=ILAMB_DATA_VERSION,
 )
 
 # Dynamically register ILAMB diagnostics

climate_ref_ilamb/configure/ilamb.yaml

@@ -2,25 +2,25 @@ registry: ilamb
 
 gpp-WECANN:
   sources:
-    gpp: gpp/WECANN/gpp.nc
+    gpp: ilamb/gpp/WECANN/gpp.nc
   relationships:
-    pr: pr/GPCPv2.3/pr.nc
-    tas: tas/CRU4.02/tas.nc
+    pr: ilamb/pr/GPCPv2.3/pr.nc
+    tas: ilamb/tas/CRU4.02/tas.nc
   variable_cmap: Greens
 
 gpp-FLUXNET2015:
   sources:
-    gpp: gpp/FLUXNET2015/gpp.nc
+    gpp: ilamb/gpp/FLUXNET2015/gpp.nc
   variable_cmap: Greens
 
 mrro-LORA:
   sources:
-    mrro: mrro/LORA/LORA.nc
+    mrro: ilamb/mrro/LORA/LORA.nc
   variable_cmap: Blues
 
 mrsos-WangMao:
   sources:
-    mrsol: mrsol/WangMao/mrsol_olc.nc
+    mrsol: ilamb/mrsol/WangMao/mrsol_olc.nc
   alternate_vars:
   - mrsos
   depth: 0.0
@@ -30,16 +30,16 @@ mrsos-WangMao:
 
 cSoil-HWSD2:
   sources:
-    cSoil: cSoil/HWSD2/hwsd2_cSoil.nc
+    cSoil: ilamb/cSoil/HWSD2/hwsd2_cSoil.nc
   variable_cmap: viridis
 
 lai-AVH15C1:
   sources:
-    lai: lai/AVH15C1/lai.nc
+    lai: ilamb/lai/AVH15C1/lai.nc
   variable_cmap: Greens
 
 nbp-Hoffman:
   analyses:
   - nbp
   sources:
-    nbp: nbp/HOFFMAN/nbp_1850-2010.nc
+    nbp: ilamb/nbp/HOFFMAN/nbp_1850-2010.nc

climate_ref_ilamb/configure/iomb.yaml

@@ -2,7 +2,7 @@ registry: iomb
 
 thetao-WOA2023-surface:
   sources:
-    thetao: WOA/thetao_mon_WOA_A5B4_gn_200501-201412.nc
+    thetao: ilamb/WOA/thetao_mon_WOA_A5B4_gn_200501-201412.nc
   variable_cmap: Reds
   depth: 0.0
   alternate_vars:
@@ -10,7 +10,7 @@ thetao-WOA2023-surface:
 
 so-WOA2023-surface:
   sources:
-    so: WOA/so_mon_WOA_A5B4_gn_200501-201412.nc
+    so: ilamb/WOA/so_mon_WOA_A5B4_gn_200501-201412.nc
   variable_cmap: YlGn
   depth: 0.0
   alternate_vars:
@@ -24,4 +24,4 @@ amoc-RAPID:
   transform:
   - msftmz_to_rapid
   sources:
-    amoc: RAPID/amoc_mon_RAPID_BE_NA_200404-202302.nc
+    amoc: ilamb/RAPID/amoc_mon_RAPID_BE_NA_200404-202302.nc

climate_ref_ilamb/dataset_registry/ilamb.txt

@@ -1,11 +1,11 @@
-cSoil/HWSD2/hwsd2_cSoil.nc sha1:9a6377e4c5ff457c08c194d2c376c46e003a4f84
-gpp/FLUXNET2015/gpp.nc sha1:16fd177e007caef2565687e2cd32884e20ef16e5
-gpp/WECANN/gpp.nc sha1:6e864a6ae201195cdf995a3a81720188af441e13
-lai/AVH15C1/lai.nc sha1:ccace4f84912d63acbb9ee09ee7b743412207a0d
-mrro/LORA/LORA.nc sha1:72bb16787877591d0c54a36d74697d0d208f985a
-mrsol/WangMao/mrsol_olc.nc sha1:24cbc9df69569bed3a39c20e499cfe4f911bd30e
-regions/GlobalLand.nc sha1:2f987d44fdba6ad0e72d14d6a2fecb7e8df2a9c5
-regions/Koppen_coarse.nc sha1:e464030db49f0295a6a22a81ca602b0f3c499b72
-pr/GPCPv2.3/pr.nc sha1:e1b942863ec76a75aa972b6d75e2e08646741259
-tas/CRU4.02/tas.nc sha1:2674da18a1a93483b50b1626e7a7ab741bf53d09
-nbp/HOFFMAN/nbp_1850-2010.nc sha1:8350af00614d6afc6b70ad314aa499a9ece80ec2
+ilamb/cSoil/HWSD2/hwsd2_cSoil.nc sha1:9a6377e4c5ff457c08c194d2c376c46e003a4f84
+ilamb/gpp/FLUXNET2015/gpp.nc sha1:16fd177e007caef2565687e2cd32884e20ef16e5
+ilamb/gpp/WECANN/gpp.nc sha1:6e864a6ae201195cdf995a3a81720188af441e13
+ilamb/lai/AVH15C1/lai.nc sha1:ccace4f84912d63acbb9ee09ee7b743412207a0d
+ilamb/mrro/LORA/LORA.nc sha1:72bb16787877591d0c54a36d74697d0d208f985a
+ilamb/mrsol/WangMao/mrsol_olc.nc sha1:24cbc9df69569bed3a39c20e499cfe4f911bd30e
+ilamb/regions/GlobalLand.nc sha1:2f987d44fdba6ad0e72d14d6a2fecb7e8df2a9c5
+ilamb/regions/Koppen_coarse.nc sha1:e464030db49f0295a6a22a81ca602b0f3c499b72
+ilamb/pr/GPCPv2.3/pr.nc sha1:e1b942863ec76a75aa972b6d75e2e08646741259
+ilamb/tas/CRU4.02/tas.nc sha1:2674da18a1a93483b50b1626e7a7ab741bf53d09
+ilamb/nbp/HOFFMAN/nbp_1850-2010.nc sha1:8350af00614d6afc6b70ad314aa499a9ece80ec2

climate_ref_ilamb/dataset_registry/iomb.txt

@@ -1,3 +1,3 @@
-WOA/so_mon_WOA_A5B4_gn_200501-201412.nc sha1:831c42c3b2ba443c255150289a2c725d7f3e5838
-WOA/thetao_mon_WOA_A5B4_gn_200501-201412.nc sha1:86d9056208291d76233e65b26c658c1fa54c3ea6
-RAPID/amoc_mon_RAPID_BE_NA_200404-202302.nc sha1:3efe773e5c2a3c832977791ff7fd9cb9f473fe65
+ilamb/WOA/so_mon_WOA_A5B4_gn_200501-201412.nc sha1:831c42c3b2ba443c255150289a2c725d7f3e5838
+ilamb/WOA/thetao_mon_WOA_A5B4_gn_200501-201412.nc sha1:86d9056208291d76233e65b26c658c1fa54c3ea6
+ilamb/RAPID/amoc_mon_RAPID_BE_NA_200404-202302.nc sha1:3efe773e5c2a3c832977791ff7fd9cb9f473fe65

climate_ref_ilamb/dataset_registry/test.txt

@@ -1,3 +1,3 @@
-test/Site/tas.nc sha1:3e757c9feadea581a92e6f4afb402b86168e113a
-test/Grid/gpp.nc sha1:2688085f95fa13617b20c723c66b0009bd661847
-test/Grid/pr.nc sha1:d667eb216939f5797d59b59cb4ade5ec31e67659
+ilamb/test/Site/tas.nc sha1:3e757c9feadea581a92e6f4afb402b86168e113a
+ilamb/test/Grid/gpp.nc sha1:2688085f95fa13617b20c723c66b0009bd661847
+ilamb/test/Grid/pr.nc sha1:d667eb216939f5797d59b59cb4ade5ec31e67659

climate_ref_ilamb/datasets.py

@@ -7,8 +7,6 @@ import pooch
 
 from climate_ref_core.datasets import DatasetCollection
 
-ILAMB_DATA_VERSION = "0.1"  # we don't really have data versions for the collection :/
-
 
 def registry_to_collection(registry: pooch.Pooch) -> DatasetCollection:
     """

climate_ref_ilamb/standard.py

@@ -23,63 +23,132 @@ from climate_ref_ilamb.datasets import (
 )
 
 
-def _build_cmec_bundle(name: str, df: pd.DataFrame) -> dict[str, Any]:
+def format_cmec_output_bundle(
+    dataset: pd.DataFrame,
+    dimensions: list[str],
+    metadata_columns: list[str],
+    value_column: str = "value",
+) -> dict[str, Any]:
+    """
+    Create a CMEC output bundle for the dataset.
+
+    Parameters
+    ----------
+    dataset
+        Processed dataset
+    dimensions
+        The dimensions of the dataset (e.g., ["source_id", "member_id", "region"])
+    metadata_columns
+        The columns to be used as metadata (e.g., ["Description", "LongName"])
+    value_column
+        The column containing the values
+
+    Returns
+    -------
+        A CMEC output bundle ready to be written to disk
+    """
+    # Validate that all required columns exist
+    required_columns = set(dimensions) | {value_column} | set(metadata_columns)
+    missing_columns = required_columns - set(dataset.columns)
+    if missing_columns:
+        raise ValueError(f"Missing required columns: {missing_columns}")
+
+    # Build the dimensions section
+    dimensions_dict: dict[str, dict[str, dict[str, str]]] = {}
+
+    # For each dimension, create a dictionary of unique values and their metadata
+    for dim in dimensions:
+        unique_values = dataset[dim].unique()
+        dim_dict: dict[str, dict[str, str]] = {}
+
+        for val in unique_values:
+            # Get the row for this dimension value
+
+            dim_dict[str(val)] = {}
+
+            if dim == dimensions[-1]:
+                # If this is the last dimension, add the value column to the metadata
+
+                dim_dict[str(val)] = dataset[dataset[dim] == val].iloc[0][metadata_columns].to_dict()
+
+        dimensions_dict[dim] = dim_dict
+
+    # Build the results section - create nested structure based on dimensions
+    def nest_results(df: pd.DataFrame, dims: list[str]) -> dict[str, Any] | float:
+        if not dims:
+            return float(df[value_column].iloc[0].item())
+
+        current_dim = dims[0]
+        remaining_dims = dims[1:]
+
+        return {
+            str(group_name): nest_results(group_df, remaining_dims)
+            for group_name, group_df in df.groupby(current_dim)
+        }
+
+    results = nest_results(dataset, list(dimensions))
+
+    return {"DIMENSIONS": {"json_structure": list(dimensions), **dimensions_dict}, "RESULTS": results}
+
+
+def _build_cmec_bundle(df: pd.DataFrame) -> dict[str, Any]:
     """
     Build a CMEC bundle from information in the dataframe.
 
-    TODO: Migrate to use pycmec when ready.
-    TODO: Add plots and html output.
     """
+    # TODO: Handle the reference data
+    # reference_df = df[df["source"] == "Reference"]
+    model_df = df[df["source"] != "Reference"]
+
+    # Source is formatted as "ACCESS-ESM1-5-r1i1p1f1-gn"
+    # This assumes that the member_id and grid_label are always the last two parts of the source string
+    # and don't contain '-'
+    extracted_source = model_df.source.str.extract(r"([\w-]+)-([\w\d]+)-([\w\d]+)")
+    model_df["source_id"] = extracted_source[0]
+    model_df["member_id"] = extracted_source[1]
+    model_df["grid_label"] = extracted_source[2]
+
+    # Strip out units from the name
+    # These are available in the attributes
+    extracted_source = model_df.name.str.extract(r"(.*)\s\[.*\]")
+    model_df["name"] = extracted_source[0]
+
+    model_df = model_df.rename(
+        columns={
+            "analysis": "metric",
+            "name": "statistic",
+        }
+    )
+
+    dimensions = ["experiment_id", "source_id", "member_id", "grid_label", "region", "metric", "statistic"]
+    attributes = ["type", "units"]
+
+    bundle = format_cmec_output_bundle(
+        model_df,
+        dimensions=dimensions,
+        metadata_columns=attributes,
+        value_column="value",
+    )
+
     ilamb_regions = ilr.Regions()
-    bundle = {
-        "DIMENSIONS": {
-            "json_structure": ["region", "model", "metric", "statistic"],
-            "region": {
-                r: {
-                    "LongName": "None" if r == "None" else ilamb_regions.get_name(r),
-                    "Description": "Reference data extents" if r == "None" else ilamb_regions.get_name(r),
-                    "Generator": "N/A" if r == "None" else ilamb_regions.get_source(r),
-                }
-                for r in df["region"].unique()
-            },
-            "model": {m: {"Description": m, "Source": m} for m in df["source"].unique() if m != "Reference"},
-            "metric": {
-                name: {
-                    "Name": name,
-                    "Abstract": "benchmark score",
-                    "URI": [
-                        "https://www.osti.gov/biblio/1330803",
-                        "https://doi.org/10.1029/2018MS001354",
-                    ],
-                    "Contact": "forrest AT climatemodeling.org",
-                }
-            },
-            "statistic": {s: {} for s in df["name"].unique()},
-        },
-        "RESULTS": {
-            r: {
-                m: {
-                    name: {
-                        s: float(
-                            df[(df["source"] == m) & (df["region"] == r) & (df["name"] == s)].iloc[0]["value"]
-                        )
-                        for s in df["name"].unique()
-                    }
-                }
-                for m in df["source"].unique()
-                if m != "Reference"
-            }
-            for r in df["region"].unique()
-        },
-    }
+    for region, region_info in bundle["DIMENSIONS"]["region"].items():
+        if region == "None":
+            region_info["LongName"] = "None"
+            region_info["Description"] = "Reference data extents"
+            region_info["Generator"] = "N/A"
+        else:
+            region_info["LongName"] = ilamb_regions.get_name(region)
+            region_info["Description"] = ilamb_regions.get_name(region)
+            region_info["Generator"] = ilamb_regions.get_source(region)
+
     return bundle
 
 
-def _form_bundles(key: str, df: pd.DataFrame) -> tuple[CMECMetric, CMECOutput]:
+def _form_bundles(df: pd.DataFrame) -> tuple[CMECMetric, CMECOutput]:
     """
     Create the output bundles (really a lift to make Ruff happy with the size of run()).
     """
-    metric_bundle = _build_cmec_bundle(key, df)
+    metric_bundle = _build_cmec_bundle(df)
     output_bundle = CMECOutput.create_template()
     return CMECMetric.model_validate(metric_bundle), CMECOutput.model_validate(output_bundle)
 
@@ -91,8 +160,8 @@ def _set_ilamb3_options(registry: pooch.Pooch, registry_file: str) -> None:
     ilamb3.conf.reset()
     ilamb_regions = ilr.Regions()
     if registry_file == "ilamb":
-        ilamb_regions.add_netcdf(registry.fetch("regions/GlobalLand.nc"))
-        ilamb_regions.add_netcdf(registry.fetch("regions/Koppen_coarse.nc"))
+        ilamb_regions.add_netcdf(registry.fetch("ilamb/regions/GlobalLand.nc"))
+        ilamb_regions.add_netcdf(registry.fetch("ilamb/regions/Koppen_coarse.nc"))
         ilamb3.conf.set(regions=["global", "tropical"])
 
 
@@ -160,7 +229,15 @@ class ILAMBStandard(Diagnostic):
                 group_by=("experiment_id",),
             ),
         )
-        self.facets = ("region", "model", "metric", "statistic")
+        self.facets = (
+            "experiment_id",
+            "source_id",
+            "member_id",
+            "grid_label",
+            "region",
+            "metric",
+            "statistic",
+        )
 
         # Setup ILAMB data and options
         self.registry_file = registry_file
@@ -197,10 +274,14 @@ class ILAMBStandard(Diagnostic):
         -------
             An execution result object
         """
+        selectors = definition.datasets[SourceDatasetType.CMIP6].selector_dict()
         _set_ilamb3_options(self.registry, self.registry_file)
 
         df = _load_csv_and_merge(definition.output_directory)
-        metric_bundle, output_bundle = _form_bundles(definition.key, df)
+        # Add the selectors to the dataframe
+        for key, value in selectors.items():
+            df[key] = value
+        metric_bundle, output_bundle = _form_bundles(df)
 
         return ExecutionResult.build_from_output_bundle(
             definition, cmec_output_bundle=output_bundle, cmec_metric_bundle=metric_bundle

{climate_ref_ilamb-0.5.4.dist-info → climate_ref_ilamb-0.5.5.dist-info}/METADATA

@@ -1,14 +1,15 @@
 Metadata-Version: 2.4
 Name: climate-ref-ilamb
-Version: 0.5.4
+Version: 0.5.5
 Summary: ILAMB diagnostic provider for the Rapid Evaluation Framework
 Author-email: Nathan Collier <nathaniel.collier@gmail.com>
 License: Apache-2.0
 License-File: LICENCE
 License-File: NOTICE
-Classifier: Development Status :: 4 - Beta
+Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
@@ -19,7 +20,6 @@ Classifier: Topic :: Scientific/Engineering
 Requires-Python: >=3.11
 Requires-Dist: climate-ref-core
 Requires-Dist: ilamb3>=2025.4.28
-Requires-Dist: types-pyyaml>=6.0.12.20241230
 Description-Content-Type: text/markdown
 
 # climate-ref-ilamb

climate_ref_ilamb-0.5.5.dist-info/RECORD

@@ -0,0 +1,14 @@
+climate_ref_ilamb/__init__.py,sha256=hMEkSjBY3yo-EbdMNOIvMSdGK14G2s5PERmWrBEtzFk,1414
+climate_ref_ilamb/datasets.py,sha256=MVCt1pxV5dIfYLm6huC0BZWP5stCamYNwXzc7kKW5AI,799
+climate_ref_ilamb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+climate_ref_ilamb/standard.py,sha256=xloR7gtinWVILjBFsoFP9N2DbovmdXR8GawFXS9L90w,9834
+climate_ref_ilamb/configure/ilamb.yaml,sha256=iq1u-VNBr622pxB4zUBRPAT_dNDBeW-hoWbivK6QHfo,799
+climate_ref_ilamb/configure/iomb.yaml,sha256=HeA5DDp6Pi2NAkQh5d3Vxsc2LVY5ucKLua1W4A0vOTQ,494
+climate_ref_ilamb/dataset_registry/ilamb.txt,sha256=gO-nyNjJQYyFdCQcRaDjA7DtC8YaLTrxfXixfGRq4Dw,817
+climate_ref_ilamb/dataset_registry/iomb.txt,sha256=-ud-u2sTmamGNVFRS1GuJUfi-7UcMUMo_22h5Ke8kUg,284
+climate_ref_ilamb/dataset_registry/test.txt,sha256=gBjUJ6W-crghYqKN0QOFmjyqpMxKK50dU3SYTuIA6jM,206
+climate_ref_ilamb-0.5.5.dist-info/METADATA,sha256=pOhMANm5mtkx_QaeEMsMsva1uyo3aTUi-iJ6Xa4IBks,2258
+climate_ref_ilamb-0.5.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+climate_ref_ilamb-0.5.5.dist-info/licenses/LICENCE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+climate_ref_ilamb-0.5.5.dist-info/licenses/NOTICE,sha256=4qTlax9aX2-mswYJuVrLqJ9jK1IkN5kSBqfVvYLF3Ws,128
+climate_ref_ilamb-0.5.5.dist-info/RECORD,,

climate_ref_ilamb-0.5.4.dist-info/RECORD

@@ -1,14 +0,0 @@
-climate_ref_ilamb/__init__.py,sha256=PW3FMUdCT7JCx3WgCx6caJ-fgttrYDyfqQjfsYzGiKc,1711
-climate_ref_ilamb/datasets.py,sha256=LlhdxDkesDZayqTWinrCcwLCJhYH6W-OU9rVXhyO-ds,887
-climate_ref_ilamb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-climate_ref_ilamb/standard.py,sha256=hv5KhnpCXkNlM1UOqWl6i2X6N29u90bHeO8n_jTSUOY,7298
-climate_ref_ilamb/configure/ilamb.yaml,sha256=bxovGQoKFLzhmV5pjMizm5GBK8bA9OoJtWB2L87hBwY,745
-climate_ref_ilamb/configure/iomb.yaml,sha256=rZa1xPr7H0MzumJmeE_ca6JwT0cRX5pPOh4crm85kw0,476
-climate_ref_ilamb/dataset_registry/ilamb.txt,sha256=FdjBSdR5cFMUWO89CumSMX2YYi6mhAghyn_cNLpOFaE,751
-climate_ref_ilamb/dataset_registry/iomb.txt,sha256=Si4c0Q8kXeyJLpzpmDGTkbQEI0GQ-lbHuXsmvP0QUuo,266
-climate_ref_ilamb/dataset_registry/test.txt,sha256=FiEQbUzeUi5Ww6akNXKoHmcuYkJae5G4n4rRisY7-pg,188
-climate_ref_ilamb-0.5.4.dist-info/METADATA,sha256=LDtC1dFmo5BZ7swTuUKleB5oMk-bZiQvlHr1mSVpzUo,2239
-climate_ref_ilamb-0.5.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-climate_ref_ilamb-0.5.4.dist-info/licenses/LICENCE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-climate_ref_ilamb-0.5.4.dist-info/licenses/NOTICE,sha256=4qTlax9aX2-mswYJuVrLqJ9jK1IkN5kSBqfVvYLF3Ws,128
-climate_ref_ilamb-0.5.4.dist-info/RECORD,,