hpcflow 0.1.15__py3-none-any.whl → 0.2.0a271__py3-none-any.whl

hpcflow/examples.ipynb

@@ -0,0 +1,1037 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from tempfile import gettempdir\n",
+    "\n",
+    "from hpcflow.api import (\n",
+    "    WorkflowTemplate, Workflow, TaskSchema, Task, Parameter, SchemaInput, load_config,\n",
+    "    InputValue, ElementSet, InputSource, ValueSequence, ElementPropagation, Action,\n",
+    "    Command, ActionEnvironment\n",
+    ")\n",
+    "\n",
+    "# Usually the user will have a known configuration directory; this step won't be \n",
+    "# necessary. (I am doing this to avoid hpcflow reading the config file for the old version\n",
+    "# of hpcflow.)\n",
+    "load_config(config_dir=gettempdir())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Note we can also import the App instance like this:\n",
+    "from hpcflow.api import hpcflow\n",
+    "\n",
+    "# In MatFlow, this will be *slightly* less confusing, like this: `from matflow import MatFlow`\n",
+    "# The App instance shouldn't normally need to be imported.\n",
+    "\n",
+    "print(hpcflow.version)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Example 1: simple workflow"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Workflow generation using the API"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Define some parameters (when end users use MatFlow, parameter types they/we define will \n",
+    "# be domain-specific and more descriptive -- not just p1, p2 etc! The same applies for\n",
+    "# task schemas.):\n",
+    "p1 = Parameter('p1')\n",
+    "p2 = Parameter('p2')\n",
+    "p3 = Parameter('p3')\n",
+    "p4 = Parameter('p4')\n",
+    "\n",
+    "\n",
+    "# Define some actions that transform the parameters in some way via commands:\n",
+    "act_1 = Action(\n",
+    "    commands=[\n",
+    "        Command(\n",
+    "            command='echo \"$((<<parameter:p1>> + 100))\"',\n",
+    "            stdout=\"<<parameter:p2>>\",\n",
+    "        ),\n",
+    "    ],\n",
+    "    environments=[ActionEnvironment(environment=hpcflow.envs.null_env)],\n",
+    ")\n",
+    "\n",
+    "act_2 = Action(\n",
+    "    commands=[\n",
+    "        Command(\n",
+    "            command='echo \"$((<<parameter:p2>> + <<parameter:p3>>))\"',\n",
+    "            stdout=\"<<parameter:p4>>\",\n",
+    "        ),\n",
+    "    ],\n",
+    "    environments=[ActionEnvironment(environment=hpcflow.envs.null_env)],\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Define some task schemas\n",
+    "s1 = TaskSchema(objective='t1', actions=[act_1], inputs=[p1], outputs=[p2])\n",
+    "s2 = TaskSchema(objective='t2', actions=[act_2], inputs=[p2, p3], outputs=[p4])\n",
+    "\n",
+    "# Define some tasks:\n",
+    "t1 = Task(schemas=[s1], inputs=[InputValue(p1, 101)])\n",
+    "t2 = Task(schemas=[s2], inputs=[InputValue(p3, 301)]) # note: p2 will originate from t1 output\n",
+    "\n",
+    "# Generate a workflow template:\n",
+    "wkt = WorkflowTemplate(name='w1', tasks=[t1, t2])\n",
+    "\n",
+    "# Write a persistent workflow from the template:\n",
+    "# note: by default the workflow directory will be the workflow name + a date-timestamp\n",
+    "#       here we pass the name explicitly, so there will be no date-timestamp, and we\n",
+    "#       overwrite. This avoids generating loads of workflow directories during testing.\n",
+    "wk = Workflow.from_template(wkt, name=wkt.name, overwrite=True)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Examining the workflow structure"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We can access tasks by their name:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t1"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "... or by index:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks[0]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Each task is associated with one or more elements:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t1.elements"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Each element has input and output values, for each of the input/output types defined in the schema:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t1.elements[0].inputs, wk.tasks.t1.elements[0].outputs"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The parameter (input or output) values can be accessed by name:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t1.elements[0].inputs.p1"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The `ElementParameter` object contains metadata about the parameter as well as the parameter value, which can be accessed by the `value` attribute:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t1.elements[0].inputs.p1.value"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "There is a lower-level `Element` function for accessing parameter data by \"path\":"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t1.elements[0].get(\"inputs.p1\")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Using `get` with no path argument returns all parameter data for that element:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t1.elements[0].get()"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Checking dependencies"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We can find the dependencies/dependents between tasks like this:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t2.get_task_dependencies(as_objects=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t1.get_dependent_tasks(as_objects=True)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "These dependencies derive from the `p2` parameter, which is output by task `t1` and \"consumed\" by task `t2`. We can also check dependency relationship at the element level:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t2.elements[0].get_element_dependencies(as_objects=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t1.elements[0].get_dependent_elements(as_objects=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Adding more elements to an existing workflow"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We can add more elements to a given task, using the `WorkflowTask.add_elements` method:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t2.add_elements(inputs=[InputValue(p3, 302)])"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now, two elements exist in the second task:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t2.elements"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Both elements in task `t2` depend on the single element in task `t1`:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t1.elements[0].get_dependent_elements(as_objects=True)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Accessing the original `Task` and `WorkflowTemplate` objects"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "When a `Task` object is added to a workflow, it is converted into a `WorkflowTask` object. However, we can still access (a copy of) the original `Task` object, using the `template` attribute:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t1.template"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We can also access (a copy of) the original workflow template in a similar way:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.template"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# TODO: write out a workflow template YAML string here! e.g. `wk.template.to_YAML_string()`"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Element sets"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You might notice that the `Task` object accessed from a `Workflow` has no `inputs` attribute, even though we originally passed an `inputs` argument to its constructor, e.g. like this: `t1 = Task(schemas=[s1], inputs=[InputValue(p1, 101)])`. This is because arguments passed to the `Task` object are converted to an *element set*. Element sets, defined by the `ElementSet` class, represent the arguments that we can pass when constructing a task, or when adding new elements to an existing task. An equivalent way of constructing the above task would be like this:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# t1 = Task(schemas=[s1], inputs=[InputValue(p1, 101)])\n",
+    "t1_b = Task(schemas=[s1], element_sets=[ElementSet(inputs=[InputValue(p1, 101)])])"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This is more verbose, and so will not usually be used, but it shows how the arguments to the task are stored internally. Element set are also what are used when adding elements to an existing task. Previously, we wrote:\n",
+    "\n",
+    "```wk.tasks.t2.add_elements(inputs=[InputValue(p3, 302)])```.\n",
+    "\n",
+    "This could be written more verbosely with an element set like this:\n",
+    "\n",
+    "```wk.tasks.t2.add_elements(element_sets=[ElementSet(inputs=[InputValue(p3, 302)])])```.\n",
+    "\n",
+    "The element set concept exists to support adding elements after a task is initially created, and helps with knowing where elements originate from."
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We can access element sets via the `Task` object (via the `WorkflowTask.template` attribute):"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t2.template.element_sets"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Element sets contain the original parametrisation:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t2.template.element_sets[0].inputs"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Example 2: using default values and overriding input sources"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Define some parameters\n",
+    "p1 = Parameter('p1')\n",
+    "p2 = Parameter('p2')\n",
+    "p3 = Parameter('p3')\n",
+    "p4 = Parameter('p4')"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The `inputs` argument of the `TaskSchema` constructor in fact takes a list `SchemaInput` objects, but if we pass a `Parameter` object instead, it will transformed into a `SchemaInput`. Likewise, the `outputs` argument more precisely takes a list of `SchemaOutput` objects.\n",
+    "\n",
+    "The distinction allows us to provide default values for a give input type within a particular schema:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "p1_input = SchemaInput(p1, default_value=10)\n",
+    "p2_input = SchemaInput(p2, default_value=30)\n",
+    "\n",
+    "# Define some task schemas\n",
+    "s1 = TaskSchema(objective='t1', actions=[act_1], inputs=[p1_input], outputs=[p2])\n",
+    "s2 = TaskSchema(objective='t2', actions=[act_2], inputs=[p2_input, p3], outputs=[p4])"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now we can define a `Task` object without passing an input value for the `p1` parameter, because the default value can be used instead:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Define some tasks:\n",
+    "t1 = Task(schemas=[s1])\n",
+    "t2 = Task(schemas=[s2], inputs=[InputValue(p3, 301)])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Generate the workflow:\n",
+    "wkt = WorkflowTemplate(name='w1', tasks=[t1, t2])\n",
+    "wk = Workflow.from_template(wkt, name=wkt.name, overwrite=True) "
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We can check that the default value was used by inspecting the relevent `ElementSet` object. In particular, the `input_sources` attribute is a map between each input parameter type and a list of `InputSource` objects:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t1.template.element_sets[0].input_sources"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "There is some default behaviour regarding the input sources. If an parameter is defined locally (meaning within the task constructor), it's local value will be used. It the parameter is not provided locally, but is output from an upstream task, that value will be used. If a parameter is not defined locally nor is output from an upstream task, the default value will be used, if defined. Otherwise a `MissingInputs` exception will be raised."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t2.template.element_sets[0].input_sources"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We can override the default behaviour, and choose a different input source if required. We can demonstrate this by adding another element (set) to task `t2` and specify input sources manually:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t2.add_elements(\n",
+    "    inputs=[\n",
+    "        InputValue(p2_input, 201),\n",
+    "        InputValue(p3, 301),\n",
+    "    ],\n",
+    "    input_sources={\n",
+    "        'p2': [InputSource.default()],\n",
+    "    }\n",
+    ")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "In the above case, the locally specified value of `p2` is not used, because we specified that the default value should be used. The default behaviour will still be used for any parameter types not included in the `input_sources` dict."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t2.template.element_sets[1].input_sources"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Example 3: multiple elements per task: sequences"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Define some parameters\n",
+    "p1 = Parameter('p1')\n",
+    "p2 = Parameter('p2')\n",
+    "p3 = Parameter('p3')\n",
+    "p4 = Parameter('p4')\n",
+    "\n",
+    "# Define some task schemas\n",
+    "s1 = TaskSchema(objective='t1', actions=[act_1], inputs=[p1], outputs=[p2])\n",
+    "s2 = TaskSchema(objective='t2', actions=[act_2], inputs=[p2, p3], outputs=[p4])"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Multiple elements can be generated for a task by using the `sequences` argument:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Define some tasks:\n",
+    "t1 = Task(schemas=[s1], inputs=[InputValue(p1, 101)])\n",
+    "t2 = Task(\n",
+    "    schemas=[s2],\n",
+    "    sequences=[\n",
+    "        ValueSequence('inputs.p3', values=[301, 302, 303], nesting_order=0),        \n",
+    "    ]\n",
+    ")\n",
+    "\n",
+    "# Note the \"parameter path\" (e.g. \"inputs.p3\") must be given to a `ValueSequence` object;\n",
+    "# this is because sequences can also be generated for resources (e.g. \"resources.num_cores\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Generate the workflow:\n",
+    "wkt = WorkflowTemplate(name='w1', tasks=[t1, t2])\n",
+    "wk = Workflow.from_template(wkt, name=wkt.name, overwrite=True)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Example 4: Propagating new elements downstream"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "When adding elements to a task, we can optionally propagate new elements to downstream tasks. Consider the workflow below, with three tasks, where each task depends on the previous:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# define an action for the new task schema \"t3\":\n",
+    "act_3 = Action(\n",
+    "    commands=[Command(\"echo <<parameter:p4>>\")],\n",
+    "    environments=[ActionEnvironment(hpcflow.envs.null_env)]\n",
+    ")\n",
+    "\n",
+    "s1 = TaskSchema(objective='t1', actions=[act_1], inputs=[p1], outputs=[p2])\n",
+    "s2 = TaskSchema(objective='t2', actions=[act_2], inputs=[p2, p3], outputs=[p4])\n",
+    "s3 = TaskSchema(objective='t3', actions=[act_3], inputs=[p4], outputs=[])\n",
+    "\n",
+    "t1 = Task(schemas=[s1], inputs=[InputValue(p1, 101)])\n",
+    "t2 = Task(schemas=[s2], inputs=[InputValue(p3, 301)])\n",
+    "t3 = Task(schemas=[s3], inputs=[])\n",
+    "\n",
+    "wkt = WorkflowTemplate(name='w1', tasks=[t1, t2, t3])\n",
+    "wk = Workflow.from_template(wkt, name=wkt.name, overwrite=True)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "There is a single element in each task:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print([task.num_elements for task in wk.tasks])"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now if we add a new element to the first task, we can specify with the `propagate_to` argument that we wish to generate new elements in downstream tasks as well. The elements generated in the downstream tasks will be restricted to depending only on the new elements generated in their respective upstream tasks."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for elem in wk.tasks.t1.elements:\n",
+    "    print(elem)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# TODO: waiting for feat/submissions fix\n",
+    "wk.tasks.t1.add_elements(\n",
+    "    inputs=[InputValue(p1, 102)],\n",
+    "    propagate_to={\"t2\": {}, \"t3\": {}},\n",
+    ")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now we have an additional element in each task:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print([task.num_elements for task in wk.tasks])"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Note that it is sufficient to specify `propagate_to` as an empty list to signify that you wish to propagate new elements downstream. There are cases where you will need to specify a nesting order for the newly generated downstream elements, in which case, `propagate_to` can be specified as a list of `ElementPropagation` objects, like this:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t1.add_elements(\n",
+    "    inputs=[InputValue(p1, 103)],\n",
+    "    propagate_to=[\n",
+    "        ElementPropagation(task=wk.tasks.t2),\n",
+    "        ElementPropagation(task=wk.tasks.t3)\n",
+    "    ]\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print([task.num_elements for task in wk.tasks])"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The `ElementPropagation` object accepts a `nesting_order` argument, which in this case is not needed, because there are no `sequences` in the workflow. If an `ElementPropagation` object is missing for a given downstream task, a default object will be generated, with no nesting order specified. Note that there is not currently a way to propagate to only some downstream tasks. We should refine this behaviour later."
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let's look at the input sources used for all tasks."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for task in wk.tasks:\n",
+    "    print(f\"{task.name=}\")\n",
+    "    for elem_set in task.template.element_sets:\n",
+    "        print(f\"{elem_set.index=}\")\n",
+    "        for k, v in elem_set.input_sources.items():\n",
+    "            print(f\"\\t{k}: {v}\")\n",
+    "    print()"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We can see that a second (and third) element set has been generated for each task due to the `add_elements` calls on the first task. The input sources for each element set in a given task look the same, except an additional argument `element_iters` in the second (and third) element sets for task input sources. For example, for the second element set of the second task, the input sources for `p2` look like this:\n",
+    "\n",
+    "```\n",
+    "[InputSource.task(task_ref=0, task_source_type=output, elements=[3])]\n",
+    "```\n",
+    "\n",
+    "The `element_iters` argument here serves as a *contraint* on the elements from which that input can be sourced. In particular, element index 3, in this case, is the newly added element from the first task."
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Example 5: inserting tasks in the middle of the workflow"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "s1 = TaskSchema(objective='t1', actions=[act_1], inputs=[p1], outputs=[p2])\n",
+    "s2 = TaskSchema(objective='t2', actions=[act_2], inputs=[p2, p3], outputs=[p4])\n",
+    "\n",
+    "t1 = Task(schemas=[s1], inputs=[InputValue(p1, 101)])\n",
+    "t2 = Task(schemas=[s2], inputs=[InputValue(p3, 301)])\n",
+    "\n",
+    "wkt = WorkflowTemplate(name='w1', tasks=[t1, t2])\n",
+    "wk = Workflow.from_template(wkt, name=wkt.name, overwrite=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# define an action for the new task schema \"t3\":\n",
+    "act_3 = Action(\n",
+    "    commands=[\n",
+    "        Command(\n",
+    "            command=\"echo <<parameter:p2>>\",\n",
+    "            stdout=\"<<parameter:p2>>\",\n",
+    "            stderr=\"<<parameter:p3>>\",\n",
+    "        ),\n",
+    "    ],\n",
+    "    environments=[ActionEnvironment(hpcflow.envs.null_env)]\n",
+    ")\n",
+    "\n",
+    "s3 = TaskSchema(objective='t3', actions=[act_3], inputs=[p2], outputs=[p2, p3])\n",
+    "t3 = Task(schemas=[s3])\n",
+    "wk.add_task(t3, new_index=1) # this will add the task in between the existing tasks"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Note that when adding a new task in the middle of the workflow, no new elements will be propagated to downstream tasks. However, in this case, we might want to add a new element to `t2` that uses outputs from the newly inserted task `t3`, which we can do like this:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wk.tasks.t2.add_elements()"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We can inspect the element sets and input sources to see what this has done (note that input source `task_ref` refers to task `insert_ID`s, since this does not change as we add/remove tasks):"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for task in wk.tasks:\n",
+    "    print(f\"{task.name=} {task.insert_ID=} {task.num_elements=}\")\n",
+    "    for elem_set in task.template.element_sets:\n",
+    "        print(f\"{elem_set.index=}\")\n",
+    "        for k, v in elem_set.input_sources.items():\n",
+    "            print(f\"\\t{k}: {v}\")\n",
+    "    print()"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "So the second element set that we added to `t2` gets its `p2` and `p3` values from the newly added middle task `t3`."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "hpcflow-new2-zEAkg0Xp-py3.10",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.8"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "ddf37d14159059670ae5501c968490e36316a9e314066be903fdc4622421273c"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}