CMIP7-data-request-api 1.1.2__py3-none-any.whl

data_request_api/dev/MM/walking_data_request.ipynb

@@ -0,0 +1,380 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "ee4ccd2a-6455-4ce7-81a8-cad76b950c91",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# imports\n",
+    "from collections import defaultdict\n",
+    "import json\n",
+    "from dataclasses import dataclass"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "c21eafe7-973b-4df9-ab51-3294fe83bd5e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Define data classes for each object type.\n",
+    "# Have only pulled out a very limited set of objects and their parameters as a demonstration\n",
+    "\n",
+    "@dataclass\n",
+    "class Experiment:\n",
+    "    record_id: str\n",
+    "    experiment_name: str\n",
+    "    mips: list\n",
+    "    experiment_groups: list\n",
+    "\n",
+    "\n",
+    "@dataclass\n",
+    "class ExperimentGroup:\n",
+    "    record_id: str\n",
+    "    experiments: list\n",
+    "    name: str\n",
+    "    opportunities: list\n",
+    "\n",
+    "\n",
+    "@dataclass\n",
+    "class Opportunity:\n",
+    "    record_id: str\n",
+    "    description: str\n",
+    "    experiment_groups: list\n",
+    "    variable_groups: list\n",
+    "\n",
+    "\n",
+    "@dataclass\n",
+    "class VariableGroup:\n",
+    "    record_id: str\n",
+    "    title: str\n",
+    "    variables: list\n",
+    "    priority: str\n",
+    "\n",
+    "\n",
+    "@dataclass\n",
+    "class Variable:\n",
+    "    record_id: str\n",
+    "    standard_name: str\n",
+    "    frequency: str\n",
+    "    name: str"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "abcc77b8-d0d8-4630-9479-9c18c89676d5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# build dictionaries allowing Data request network to be navigated\n",
+    "\n",
+    "def build_dictionaries(filename):\n",
+    "    \"\"\"\n",
+    "    load json in filename and build dictionaries of data request objects\n",
+    "    \"\"\"\n",
+    "    # Dictionaries to be returned\n",
+    "    object_dict = {}\n",
+    "    structured_dict = defaultdict(dict)\n",
+    "\n",
+    "    with open(filename) as fh:\n",
+    "        raw_data = json.load(fh)\n",
+    "\n",
+    "    # load experiments\n",
+    "    for record_id, entry in raw_data['Experiments']['records'].items():\n",
+    "        try:\n",
+    "            expt = Experiment(\n",
+    "                record_id=record_id, \n",
+    "                experiment_name=entry[' Experiment'].strip(), \n",
+    "                mips=entry['MIP'],\n",
+    "                experiment_groups=entry['Experiment Group'])\n",
+    "        except:\n",
+    "            print(\"Could not interpret experiment: \", entry)\n",
+    "        structured_dict['experiment'][record_id] = expt\n",
+    "        object_dict[record_id] = expt\n",
+    "\n",
+    "    # load experiment groups\n",
+    "    for record_id, entry in raw_data['Experiment Group']['records'].items():\n",
+    "        try:\n",
+    "            expt_group = ExperimentGroup(\n",
+    "                record_id=record_id,\n",
+    "                experiments=entry['Experiments'],\n",
+    "                name=entry['Name'],\n",
+    "                opportunities=entry['Opportunities']\n",
+    "            )\n",
+    "        except:\n",
+    "            print(\"Could not interpret experiment_group:\", entry)\n",
+    "        structured_dict['experiment_group'][record_id] = expt_group\n",
+    "        object_dict[record_id] = expt_group\n",
+    "\n",
+    "    # load opportunities\n",
+    "    for record_id, entry in raw_data['Opportunity']['records'].items():\n",
+    "        try:\n",
+    "            opportunity = Opportunity(\n",
+    "                record_id=record_id,\n",
+    "                description=entry['Description'],\n",
+    "                experiment_groups=entry['Experiment Groups'],\n",
+    "                variable_groups=entry['Variable Groups']\n",
+    "            )\n",
+    "        except:\n",
+    "            print(\"could not interpret opportunity\", entry)\n",
+    "        structured_dict['opportunity'][record_id] = opportunity\n",
+    "        object_dict[record_id] = opportunity\n",
+    "\n",
+    "    # load variable groups\n",
+    "    for record_id, entry in raw_data['Variable Group']['records'].items():\n",
+    "        try:\n",
+    "            variable_group = VariableGroup(\n",
+    "                record_id=record_id,\n",
+    "                title=entry['Title'],\n",
+    "                variables=entry['Variables'],\n",
+    "                priority=entry['Priority Level']\n",
+    "            )\n",
+    "        except:\n",
+    "            print(\"Could not interpret variable group\", entry)\n",
+    "        structured_dict['variable_group'][record_id] = variable_group\n",
+    "        object_dict[record_id] = variable_group\n",
+    "\n",
+    "    # load variables\n",
+    "    for record_id, entry in raw_data['Variables']['records'].items():\n",
+    "        try:\n",
+    "            variable = Variable(\n",
+    "                record_id=record_id,\n",
+    "                standard_name=entry['CF Standard Name (from MIP Variables)'],\n",
+    "                frequency=entry['Frequency'][0],\n",
+    "                name=entry['Compound Name']\n",
+    "            )\n",
+    "        except:\n",
+    "            print(\"Could not interpret as variable:\", entry)\n",
+    "        structured_dict['variable'][record_id] = variable\n",
+    "        object_dict[record_id] = variable\n",
+    "\n",
+    "    # return dictionaries\n",
+    "    return object_dict, structured_dict"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "4a639eb1-5bdf-411b-a3e2-0774c46b9c55",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Could not interpret experiment_group: {'Experiments': ['recFIIkEdefZynPhh'], 'Name': 'spin-up', 'Title': 'The spin-up'}\n",
+      "could not interpret opportunity {'Experiment Groups': ['rec0J4qnPn2vktlhO'], 'References': ['recw7gRoApscR5fsW'], 'Status': 'New', 'Title of Opportunity': 'Benchmarking'}\n",
+      "Could not interpret as variable: {'Atmosphere author team review': 'In progress', 'CF Standard Name (from MIP Variables)': 'air_temperature', 'Cell Measures': ['recb1V00ayWZnmm79'], 'Cell Methods': ['recfYDayRm62sFUsp'], 'Coordinates': 'height2m', 'Description': 'For models with fractional land areas (SFTLF) we are not currently able to diagnose near-surface air temperatures over just the land part of coastal gridboxes, with the current CMIP diagnostic set. The TAS diagnostic will be a weighted mean of the air temperatures over land and sea. If the diagnostic (lets call it LTAS) is available, we can then diagnose the air temperatures just over land, and in conjunction with TAS and SFTLF diagnose the air temperatures just over sea/sea-ice in coastal gridboxes. This will enable consistent comparisons with observational datasets (e.g. CRUTEM5 which is just air temperature over land) over regional areas with large coastal areas. \\nReference: Jones, GS, 2020,  \"Apples and oranges\": on comparing near surface temperatures from climate models with observations, QJRMS, 146, Issue 733, 3747-3771, https://doi.org/10.1002/qj.3871.\\n', 'Extra Dimensions': 'latitude, longitude, time', 'Frequency': ['mon'], 'Modeling Realm': ['atmos'], 'Physical Parameter': ['reccZFphgzh8sN3yC'], 'Processing Note': \"The diagnostic is used within models that have fractional land areas to calculate coastal gridbox mean TAS, i.e., the mean of air temperatures over sea and air temperatures over land (weighted by the land/ocean areas) in models with fractional land areas. So it shouldn't be too difficult to produce the diagnostic for CMIP.\\n\", 'Spatial Shape': ['rec1eFSEpR5EWjdLK'], 'Status': 'Under review', 'Table': ['Amon'], 'Temporal Shape': ['recF1jvxHxPzKHG62'], 'Theme': ['Atmosphere'], 'Title': 'Near-surface air temperature over land', 'Type': 'real'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "object_dict, structured_dict = build_dictionaries('request_basic_dump2.json')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "c4ffda0e-0174-4a70-9053-6547a2d8d873",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# dictionary to allow lookup up of experiments\n",
+    "experiment_name_to_record_id = {i.experiment_name: i.record_id \n",
+    "                                for i in structured_dict['experiment'].values()}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "082b69c9-976a-4d5a-861c-e1679f2a42d6",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'rec7mTVv08z3iBObw'"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "experiment_name_to_record_id['historical']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "c6af3294-447f-4424-a992-f24381c805f9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'recR7YVoWYdbQp3wk'"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "experiment_name_to_record_id['amip']\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "d0afab7c-5319-4ae6-a51d-4c80672d8602",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Walk data request network from experiment to variable list:\n",
+    "\n",
+    "def walk_experiment_to_variables(experiment_id, object_dict):\n",
+    "    \"\"\"\n",
+    "    Navigate data request from specified experiment ids to record ids of corresponding variables\n",
+    "    \"\"\"\n",
+    "    expt_groups = set(object_dict[experiment_id].experiment_groups)\n",
+    "    print(\"found {} connected experiment groups\".format(len(expt_groups)))\n",
+    "\n",
+    "    opportunities = set()\n",
+    "    for expt_group_id in expt_groups:\n",
+    "        expt_group_object = object_dict[expt_group_id]\n",
+    "        for opportunity_id in expt_group_object.opportunities:\n",
+    "            opportunities.add(opportunity_id)\n",
+    "\n",
+    "    print(\"found {} connected opportunities\".format(len(opportunities)))\n",
+    "\n",
+    "    variable_groups = set()\n",
+    "    for opportunity_id in opportunities:\n",
+    "        opportunity_object = object_dict[opportunity_id]\n",
+    "        for variable_group_id in opportunity_object.variable_groups:\n",
+    "            variable_groups.add(variable_group_id)\n",
+    "\n",
+    "    print(\"found {} connected variable groups\".format(len(variable_groups)))\n",
+    "\n",
+    "    variables = set()\n",
+    "    for variable_group_id in variable_groups:\n",
+    "        variable_group_object = object_dict[variable_group_id]\n",
+    "        for variable_id in variable_group_object.variables:\n",
+    "            variables.add(variable_id)\n",
+    "\n",
+    "    print(\"found {} connected variables\".format(len(variables)))\n",
+    "\n",
+    "    return list(variables)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "d8f55064-b6f4-4227-825d-7ee328ebfd88",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "found 2 connected experiment groups\n",
+      "found 4 connected opportunities\n",
+      "found 9 connected variable groups\n",
+      "found 239 connected variables\n"
+     ]
+    }
+   ],
+   "source": [
+    "historical_variables = walk_experiment_to_variables(experiment_name_to_record_id['historical'], object_dict)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "2d932550-6b13-487f-9ac7-450c533c53a7",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['recGfiAQ8b62b8D0H',\n",
+       " 'recc29R75UVqSGpFC',\n",
+       " 'reck2Ofq8KEbUw4Yz',\n",
+       " 'recH0aJ4cZJt7up4d',\n",
+       " 'recQMtSbbOQwkDBcL',\n",
+       " 'recyHFBDBY1oy7jib',\n",
+       " 'recIWQRAqz9K78QhF',\n",
+       " 'recS5sYw7lg0By4Dn',\n",
+       " 'reczhZLOX2BxzRT93',\n",
+       " 'rec57DB6wHX5cC9uo']"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Show first few variable record_ids \n",
+    "historical_variables[:10]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "37f8ba4e-b84b-4109-8c15-b06c3579015d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "False"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# problem: record_ids for variables don't appear to match up in the json file used here\n",
+    "any([i in object_dict for i in historical_variables])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7269f54c-00f3-43c9-95fb-a9309b7357e6",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}