CMIP7-data-request-api 1.1.2__py3-none-any.whl

data_request_api/dev/JA/workflow_example_GRtest.py

@@ -0,0 +1,275 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+
+"""
+from __future__ import division, print_function, unicode_literals, absolute_import
+
+import pprint
+from collections import defaultdict
+
+import six
+
+import json
+
+from data_request_api.stable.content.dreq_api import dreq_content as dc
+from data_request_api.stable.query.data_request import DataRequest
+from data_request_api.stable.utilities.logger import change_log_file, change_log_level
+
+# Set up log file (default to stdout) and log level
+change_log_file(default=True)
+change_log_level("debug")
+
+### Step 1: Get the content of the DR
+# Define content version to be used
+use_dreq_version = 'v1.0beta'
+# use_dreq_version = "first_export"
+# use_dreq_version = 'new_export_15Oct2024'
+# Download specified version of data request content (if not locally cached)
+# dc.retrieve(use_dreq_version)
+# Load content into python dict
+# content = dc.load(use_dreq_version)
+
+### Step 2: Load it into the software of the DR
+# DR = DataRequest.from_input(json_input=content, version=use_dreq_version)
+# path = f'../sandbox/MS/dreq_api/dreq_res/{use_dreq_version}'
+path = f'../MS/dreq_api/dreq_res/{use_dreq_version}'
+DR = DataRequest.from_separated_inputs(DR_input=f"{path}/DR_content.json",
+                                       VS_input=f"{path}/VS_content.json")
+
+
+GR_demo = False
+if GR_demo:
+    ### Step 3: Get information from the DR
+    # -> Print DR content
+    print(DR)
+    # -> Print an experiment group content
+    print(DR.get_experiments_groups()[0])
+    # -> Get all variables' id associated with an opportunity
+    print(DR.find_variables_per_opportunity(DR.get_opportunities()[0]))
+    # -> Get all experiments' id associated with an opportunity
+    print(DR.find_experiments_per_opportunity(DR.get_opportunities()[0]))
+    # -> Get information about the shapes of the variables of all variables groups
+    rep = dict()
+    for elt in DR.get_variables_groups():
+        rep[elt.id] = dict(spatial_shape=set(), frequency=set(), temporal_shape=set(), physical_parameter=set())
+        for var in elt.get_variables():
+            for key in ["spatial_shape", "frequency", "temporal_shape", "physical_parameter"]:
+                rep[elt.id][key] = rep[elt.id][key].union(set([elt.get("name", "???") if isinstance(elt, dict) else elt
+                                                            for elt in var.__getattribute__(key)]))
+
+    rep = defaultdict(lambda: defaultdict(set))
+    for elt in DR.get_variables_groups():
+        for var in elt.get_variables():
+            param = var.physical_parameter["name"]
+            freq = var.frequency["name"]
+            realm = set([elt if isinstance(elt, six.string_types) else elt.get("name", "???") for elt in var.modelling_realm])
+            spt_shp = set([elt if isinstance(elt, six.string_types) else elt.get("name", "???") for elt in var.spatial_shape])
+            tmp_shp = set([elt if isinstance(elt, six.string_types) else elt.get("name", "???") for elt in var.temporal_shape])
+            for (rlm, freq, sshp, tshp) in zip(realm, [freq, ], spt_shp, tmp_shp):
+                rep[rlm][param].add(f"{freq} // {sshp} // {tshp}")
+    pprint.pprint(rep)
+    # pprint.pprint(rep_data)
+
+
+
+
+# from copy import deepcopy
+# DR0 = deepcopy(DR)
+
+print(type(DR))
+
+opp = DR.get_opportunities()[0]
+
+
+# simpler way to do this?
+def get_name(x):
+    return x.vs.get_element(x.DR_type, x.id, 'name')
+
+title = 'Ocean Extremes'
+opp = [opp for opp in DR.get_opportunities() if get_name(opp) == 'Ocean Extremes'][0]
+
+# print()
+# print(opp)
+
+# print()
+# for theme in opp.get_themes():
+#     print(theme)
+
+# print()
+# for expt_group in opp.get_experiments_groups():
+#     print(expt_group)
+
+# print()
+# for var_group in opp.get_variables_groups():
+#     print(var_group)
+
+# print()
+
+
+# print(len(opp_vars), len(opp_expts))
+
+# var = opp_vars[0]
+
+# copied this out of VS_content.json, surely there's a way to get this lookup info from DR object?
+lookup = {
+    "priority_level": {
+        # note, this info is keyed on the uid, which is found in the export
+        # e.g. in dreq_release_export.json for v1.0beta:
+        # "Name": "High",
+        # "Notes": "High priority should be used sparingly",
+        # "UID": "527f5c94-8c97-11ef-944e-41a8eb05f654",
+        # "Value": 2,
+        "527f5c94-8c97-11ef-944e-41a8eb05f654": {
+            "name": "High",
+            "notes": "High priority should be used sparingly",
+            "value": 2
+        },
+        "527f5c95-8c97-11ef-944e-41a8eb05f654": {
+            "name": "Medium",
+            "value": 3
+        },
+        "527f5c96-8c97-11ef-944e-41a8eb05f654": {
+            "name": "Low",
+            "value": 4
+        },
+        "527f5c97-8c97-11ef-944e-41a8eb05f654": {
+            "name": "Core",
+            "notes": "Top priority -- adopted by panels",
+            "value": 1
+        }
+    }
+}
+priority_levels = [info['name'] for info in lookup['priority_level'].values()]
+
+def get_unique_var_name(var):
+    return var.compound_name
+
+
+use_opps = []
+use_opps.append('Baseline Climate Variables for Earth System Modelling')
+use_opps.append('Synoptic systems and impacts')
+# use_opps.append('Climate impact assessments on freshwater ecosystems')
+# use_opps.append('Ocean Extremes')
+
+use_opps = 'all'
+
+# lookup table of opportunities by their title
+opps = {}
+for opp in DR.get_opportunities():
+    title = get_name(opp)
+    assert title not in opps, f'opp title not unique: {title}'
+    opps[title] = opp
+
+request = {} # dict to hold aggregated request
+
+check = not True
+
+if use_opps == 'all':
+    use_opps = list(opps.keys())
+
+use_opps = sorted(use_opps)
+for title in use_opps:
+
+    opp = opps[title]
+
+    if check:
+        opp_expts = set()
+        for expt_group in opp.experiments_groups:
+            opp_expts.update([expt.name for expt in expt_group.get_experiments()])
+        opp_expts0 = opp_expts
+
+    # -> Get all experiments' id associated with an opportunity
+    opp_expts = DR.find_experiments_per_opportunity(opp)
+    opp_expts = set([expt.name for expt in opp_expts])
+
+    if check:
+        assert opp_expts == opp_expts0
+        del opp_expts0
+
+    if check:
+        # -> Get all variables' id associated with an opportunity
+        opp_vars = DR.find_variables_per_opportunity(opp)
+        opp_vars0 = set([get_unique_var_name(var) for var in opp_vars])
+
+    # Loop over variable groups to get opportunity's variables separated by priority level
+    opp_vars = {p : set() for p in priority_levels}
+    for vg in opp.variables_groups:
+
+        assert isinstance(vg.priority, list) and len(vg.priority) == 1
+        priority_info = lookup['priority_level'][vg.priority[0]]
+        priority_level = priority_info['name']  # e.g. "Core", "High", ...
+
+        for var in vg.variables:
+            var_name = get_unique_var_name(var)
+            # Add this variable to the list of requested variables at the specified priority
+            opp_vars[priority_level].add(var_name)
+
+    if check:
+        opp_vars1 = set()
+        for priority_level in opp_vars:
+            opp_vars1.update(opp_vars[priority_level])
+        assert opp_vars1 == opp_vars0  # confirm that DR.find_variables_per_opportunity(opp) lumps all priority levels together
+        del opp_vars0, opp_vars1
+
+    # Aggregate this Opportunity's request into the master list of requests
+    for expt_name in opp_expts:
+        if expt_name not in request:
+            # If we haven't encountered this experiment yet, initialize an expt_request object for it
+            request[expt_name] = {p : set() for p in priority_levels}
+
+        # Add this Opportunity's variables request to the expt_request object
+        for priority_level, var_names in opp_vars.items():
+            request[expt_name][priority_level].update(opp_vars[priority_level])
+
+
+# Remove any overlaps in variable lists between different priority levels
+priority_hierarchy = ['Core', 'High', 'Medium', 'Low']  # ordered from highest to lowest priority
+assert set(priority_hierarchy) == set(priority_levels)
+assert len(set(priority_hierarchy)) == len(priority_hierarchy)
+for expt_request in request.values():
+    for k,p in enumerate(priority_hierarchy):
+        for p_higher in priority_hierarchy[:k]:
+            expt_request[p] = expt_request[p].difference(expt_request[p_higher])
+            # print(p,p_higher)
+            # print()
+
+    # convert sets to lists for json output
+    for p in expt_request:
+        expt_request[p] = sorted(expt_request[p], key=str.lower)
+
+expt_vars = {
+    'Header' : {
+        'Opportunities' : use_opps,
+        'dreq version' : use_dreq_version,
+    },
+    'experiment' : request,
+}
+
+
+if len(expt_vars['experiment']) > 0:
+
+    # Show user what was found
+    print(f'\nFor data request version {use_dreq_version}, number of requested variables found by experiment:')
+    priority_levels = ['Core', 'High', 'Medium', 'Low']
+    for expt, req in sorted(expt_vars['experiment'].items()):
+        d = {p : 0 for p in priority_levels}
+        for p in priority_levels:
+            if p in req:
+                d[p] = len(req[p])
+        n_total = sum(d.values())
+        print(f'  {expt} : ' + ' ,'.join(['{p}={n}'.format(p=p,n=d[p]) for p in priority_levels]) + f', TOTAL={n_total}')
+
+    # Write the results to json
+    filename = 'requested2.json'
+    with open(filename, 'w') as f:
+        json.dump(expt_vars, f, indent=4, sort_keys=True)
+        print('\nWrote requested variables to ' + filename)
+
+else:
+    print(f'\nFor data request version {use_dreq_version}, no requested variables were found')
+
+
+
+

data_request_api/dev/JA/workflow_example_test.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python
+'''
+Example script for basic use of CMIP7 data request content
+
+Getting started
+---------------
+First create an environment with the required dependencies:
+
+    conda env create -n my_dreq_env --file env.yml
+
+(replacing my_dreq_env with your preferred env name). Then activate it and run the script:
+
+    conda activate my_dreq_env
+    python workflow_example.py
+
+will load the data request content and save a json file of requested variables in the current dir.
+To run interactively in ipython:
+
+    run -i workflow_example.py
+
+'''
+# try: tables
+# except: tables = {}
+
+# use_export = 'release'
+# use_export = 'raw'
+
+# if True:
+if False:
+
+    for use_export in tables:
+
+        print(f'\n*** {use_export}  ***')
+
+        VarGroups = tables[use_export]['VarGroups']
+        Opps = tables[use_export]['Opps']
+
+        titles = []
+        titles.append('Temperature variability')
+        titles.append('Ocean Extremes')
+
+        for title in titles:
+            print()
+            opp = Opps.get_attr_record('title', title)
+            print(opp.title)
+            for link in opp.variable_groups:
+                vg = VarGroups.get_record(link)
+                print(vg.name)
+
+    stop
+
+import json
+from data_request_api.stable.content.dreq_api import dreq_content as dc
+from data_request_api.stable.query import dreq_query as dq
+
+from importlib import reload
+reload(dq)
+
+
+test = True
+
+
+# Specify opportunities that modelling group chooses to support
+# This can be a subset:
+use_opps = []
+use_opps.append('Baseline Climate Variables for Earth System Modelling')
+# use_opps.append('Synoptic systems and impacts')
+# use_opps.append('Climate impact assessments on freshwater ecosystems')
+use_opps.append('Ocean Extremes')
+
+# Or to use all opportunities in the data request:
+use_opps = 'all'
+
+
+if test:
+
+    import os
+
+    use_dreq_version = 'v1.0beta'
+
+    use_export = 'release'
+    use_export = 'raw'
+
+    use_consolidated = not True
+
+    load_manually = not True
+
+    if load_manually:
+        # just for testing
+        path = '/home/rja001/code/dreq/CMIP7_DReq_Content/airtable_export'
+        filepath = os.path.join(path, 'dreq_working.json')
+        with open(filepath, 'r') as f:
+            content = json.load(f)
+            print('loaded ' + filepath)
+
+        use_consolidated = False
+
+    else:
+        # Download specified version of data request content (if not locally cached)
+        dc.retrieve(use_dreq_version, export=use_export)
+        # Load content into python dict
+        content = dc.load(use_dreq_version, export=use_export, consolidate=use_consolidated)
+
+
+
+
+else:
+
+    use_dreq_version = 'v1.0beta'
+
+    # Download specified version of data request content (if not locally cached)
+    dc.retrieve(use_dreq_version)
+    # Load content into python dict
+    content = dc.load(use_dreq_version)
+
+
+# Get consolidated list of requested variables that supports these opportunities
+dq.DREQ_VERSION = use_dreq_version
+
+if test:
+
+    use_old_get = False
+
+    if use_old_get:
+
+        if len(content.keys()) == 1:
+            k = 'Data Request'
+            if k in content:
+                content[f'{k} {use_dreq_version}'] = content[k]
+                content.pop(k)
+        expt_vars = dq._get_requested_variables(content, use_opps, priority_cutoff='Low')
+
+    else:
+
+        expt_vars = dq.get_requested_variables(content, use_opps, priority_cutoff='Low', verbose=False, consolidated=use_consolidated)
+
+
+
+
+        if False:
+            # return Opps, VarGroups, PriorityLevel, ExptGroups, Expts
+
+            Opps, VarGroups, PriorityLevel, ExptGroups, Expts = expt_vars
+            del expt_vars
+
+            tables[use_export] = {
+                'Opps' : Opps, 'VarGroups' : VarGroups,
+            }
+            
+            opps = []
+            opps.append( Opps.get_attr_record('title', 'Temperature variability') )
+
+            # opps.append( Opps.get_attr_record('title', 'Ocean Extremes') )
+
+            for title in [            
+                "Ocean Extremes",
+                "Ocean changes, drivers and impacts",
+                "Rapid Evaluation Framework",
+                "Paleoclimate research at the interface between past, present, and future",
+                "Robust Risk Assessment of Tipping Points",
+            ]:
+                opps.append( Opps.get_attr_record('title', title) )
+
+            # achtung!
+            # "Ocean Extremes",
+            # "Ocean changes, drivers and impacts",
+            # "Rapid Evaluation Framework",
+            # "Paleoclimate research at the interface between past, present, and future",
+            # "Robust Risk Assessment of Tipping Points",
+
+            # opp = opps[0]
+            # opp_expts0 = dq.get_opp_expts(opp, ExptGroups, Expts, verbose=False)
+
+            for opp in opps:
+                print('\n' + opp.title)
+                # opp_expts = dq.get_opp_expts(opp, ExptGroups, Expts, verbose=False)
+                # print(opp_expts.difference(opp_expts0))
+                for link in opp.variable_groups: 
+                    vg = VarGroups.get_record(link)
+                    if isinstance(vg.priority_level, str):
+                        assert PriorityLevel is None
+                        print(vg.priority_level)
+                    else:
+                        link = vg.priority_level[0]
+                        pl = PriorityLevel.get_record(link)
+                        print('  ' + pl.name)
+            stop
+
+
+else:
+    
+    expt_vars = dq.get_requested_variables(content, use_opps, priority_cutoff='Low')
+
+
+if len(expt_vars['experiment']) > 0:
+
+    # Show user what was found
+    print(f'\nFor data request version {use_dreq_version}, number of requested variables found by experiment:')
+    priority_levels = ['Core', 'High', 'Medium', 'Low']
+    for expt, req in expt_vars['experiment'].items():
+        d = {p : 0 for p in priority_levels}
+        for p in priority_levels:
+            if p in req:
+                d[p] = len(req[p])
+        n_total = sum(d.values())
+        print(f'  {expt} : ' + ' ,'.join(['{p}={n}'.format(p=p,n=d[p]) for p in priority_levels]) + f', TOTAL={n_total}')
+
+    # Write the results to json
+    # filename = 'requested.json'
+    filename = f'requested_{use_export}.json'
+
+    with open(filename, 'w') as f:
+        json.dump(expt_vars, f, indent=4, sort_keys=True)
+        print('\nWrote requested variables to ' + filename)
+
+else:
+    print(f'\nFor data request version {use_dreq_version}, no requested variables were found')
+
+# To remove locally cached version:
+# dc.delete(use_dreq_version)
+# To remove all locally cached versions:
+# dc.delete()

data_request_api/dev/MM/checksum.py

@@ -0,0 +1,68 @@
+'''
+https://gist.github.com/matthew-mizielinski/643ca1841dc760edf230473a8b81d396
+'''
+
+import copy
+import json
+import hashlib
+
+
+def set_checksum(dictionary, overwrite=True):
+    """
+    Calculate the checksum for the ``dictionary``, then add the
+    value to ``dictionary`` under the ``checksum`` key. ``dictionary``
+    is modified in place.
+
+    Parameters
+    ----------
+    dictionary: dict
+        The dictionary to set the checksum to.
+    overwrite: bool
+        Overwrite the existing checksum (default True).
+
+    Raises
+    ------
+    RuntimeError
+        If the ``checksum`` key already exists and ``overwrite`` is
+        False.
+    """
+    if 'checksum' in dictionary:
+        if not overwrite:
+            raise RuntimeError('Checksum already exists.')
+        del dictionary['checksum']
+    checksum = _checksum(dictionary)
+    dictionary['checksum'] = checksum
+
+
+def validate_checksum(dictionary):
+    """
+    Validate the checksum in the ``dictionary``.
+
+    Parameters
+    ----------
+    dictionary: dict
+        The dictionary containing the ``checksum`` to validate.
+
+    Raises
+    ------
+    KeyError
+        If the ``checksum`` key does not exist.
+    RuntimeError
+        If the ``checksum`` value is invalid.
+    """
+    if 'checksum' not in dictionary:
+        raise KeyError('No checksum to validate')
+    dictionary_copy = copy.deepcopy(dictionary)
+    del dictionary_copy['checksum']
+    checksum = _checksum(dictionary_copy)
+    if dictionary['checksum'] != checksum:
+        msg = ('Expected checksum   "{}"\n'
+               'Calculated checksum "{}"').format(dictionary['checksum'],
+                                                  checksum)
+        raise RuntimeError(msg)
+
+
+def _checksum(obj):
+    obj_str = json.dumps(obj, sort_keys=True)
+    checksum_hex = hashlib.md5(obj_str.encode('utf8')).hexdigest()
+    return 'md5: {}'.format(checksum_hex)