hpcflow-new2 0.2.0a190__py3-none-any.whl → 0.2.0a199__py3-none-any.whl

hpcflow/sdk/submission/jobscript.py

@@ -3,16 +3,18 @@ Model of information submitted to a scheduler.
 """
 
 from __future__ import annotations
+from collections import defaultdict
 
 import os
 import shutil
 import socket
 import subprocess
-from textwrap import indent
-from typing import cast, overload, TYPE_CHECKING
+from textwrap import dedent, indent
+from typing import TextIO, cast, overload, TYPE_CHECKING
 from typing_extensions import override
 
 import numpy as np
+from hpcflow.sdk.core import SKIPPED_EXIT_CODE
 from hpcflow.sdk.core.enums import EARStatus
 from hpcflow.sdk.core.errors import (
     JobscriptSubmissionFailure,
@@ -21,8 +23,9 @@ from hpcflow.sdk.core.errors import (
 
 from hpcflow.sdk.typing import hydrate
 from hpcflow.sdk.core.json_like import ChildObjectSpec, JSONLike
-from hpcflow.sdk.core.utils import parse_timestamp, current_timestamp
+from hpcflow.sdk.core.utils import nth_value, parse_timestamp, current_timestamp
 from hpcflow.sdk.log import TimeIt
+from hpcflow.sdk.submission.schedulers import QueuedScheduler
 from hpcflow.sdk.submission.schedulers.direct import DirectScheduler
 from hpcflow.sdk.submission.shells import get_shell, DEFAULT_SHELL_NAMES
 
@@ -36,32 +39,63 @@ if TYPE_CHECKING:
     from ..core.actions import ElementActionRun
     from ..core.element import ElementResources
     from ..core.loop_cache import LoopIndex
-    from ..core.types import JobscriptSubmissionFailureArgs
+    from ..core.types import JobscriptSubmissionFailureArgs, BlockActionKey
     from ..core.workflow import WorkflowTask, Workflow
+    from ..persistence.base import PersistentStore
     from .submission import Submission
     from .shells.base import Shell
-    from .schedulers import Scheduler, QueuedScheduler
+    from .schedulers import Scheduler
     from .enums import JobscriptElementState
     from .types import (
         JobScriptCreationArguments,
         JobScriptDescriptor,
-        ResolvedDependencies,
+        ResolvedJobscriptBlockDependencies,
         SchedulerRef,
         VersionInfo,
     )
+    from ..core.cache import ObjectCache
+from hpcflow.sdk.submission.submission import JOBSCRIPT_SUBMIT_TIME_KEYS
+
+
+def is_jobscript_array(
+    resources: ElementResources, num_elements: int, store: PersistentStore
+) -> bool:
+    """Return True if a job array should be used for the specified `ElementResources`."""
+    if resources.scheduler in ("direct", "direct_posix"):
+        if resources.use_job_array:
+            raise ValueError(
+                f"`use_job_array` not supported by scheduler: {resources.scheduler!r}"
+            )
+        return False
+
+    if resources.combine_scripts:
+        return False
+
+    run_parallelism = store._features.EAR_parallelism
+    if resources.use_job_array is None:
+        if num_elements > 1 and run_parallelism:
+            return True
+        else:
+            return False
+    else:
+        if resources.use_job_array and not run_parallelism:
+            raise ValueError(
+                f"Store type {store!r} does not support element parallelism, so jobs "
+                f"cannot be submitted as scheduler arrays."
+            )
+        return resources.use_job_array
 
 
 @TimeIt.decorator
 def generate_EAR_resource_map(
     task: WorkflowTask,
     loop_idx: LoopIndex[str, int],
+    cache: ObjectCache,
 ) -> tuple[Sequence[ElementResources], Sequence[int], NDArray, NDArray]:
     """
     Generate an integer array whose rows represent actions and columns represent task
     elements and whose values index unique resources.
     """
-    # TODO: assume single iteration for now; later we will loop over Loop tasks for each
-    # included task and call this func with specific loop indices
     none_val = -1
     resources: list[ElementResources] = []
     resource_hashes: list[int] = []
@@ -69,16 +103,16 @@ def generate_EAR_resource_map(
     arr_shape = (task.num_actions, task.num_elements)
     resource_map = np.empty(arr_shape, dtype=int)
     EAR_ID_map = np.empty(arr_shape, dtype=int)
-    # EAR_idx_map = np.empty(
-    #     shape=arr_shape,
-    #     dtype=[("EAR_idx", np.int32), ("run_idx", np.int32), ("iteration_idx", np.int32)],
-    # )
     resource_map[:] = none_val
     EAR_ID_map[:] = none_val
-    # EAR_idx_map[:] = (none_val, none_val, none_val)  # TODO: add iteration_idx as well
 
-    for element in task.elements[:]:
-        for iter_i in element.iterations:
+    assert cache.elements is not None
+    assert cache.iterations is not None
+
+    for elem_id in task.element_IDs:
+        element = cache.elements[elem_id]
+        for iter_ID_i in element.iteration_IDs:
+            iter_i = cache.iterations[iter_ID_i]
             if iter_i.loop_idx != loop_idx:
                 continue
             if iter_i.EARs_initialised:  # not strictly needed (actions will be empty)
@@ -188,12 +222,12 @@ def group_resource_map_into_jobscripts(
 def resolve_jobscript_dependencies(
     jobscripts: Mapping[int, JobScriptCreationArguments],
     element_deps: Mapping[int, Mapping[int, Sequence[int]]],
-) -> Mapping[int, dict[int, ResolvedDependencies]]:
+) -> Mapping[int, dict[int, ResolvedJobscriptBlockDependencies]]:
     """
     Discover concrete dependencies between jobscripts.
     """
     # first pass is to find the mappings between jobscript elements:
-    jobscript_deps: dict[int, dict[int, ResolvedDependencies]] = {}
+    jobscript_deps: dict[int, dict[int, ResolvedJobscriptBlockDependencies]] = {}
     for js_idx, elem_deps in element_deps.items():
         # keys of new dict are other jobscript indices on which this jobscript (js_idx)
         # depends:
@@ -258,7 +292,9 @@ def resolve_jobscript_dependencies(
 
 
 def _reindex_dependencies(
-    jobscripts: Mapping[int, JobScriptCreationArguments], from_idx: int, to_idx: int
+    jobscripts: Mapping[int, JobScriptCreationArguments],
+    from_idx: int,
+    to_idx: int,
 ):
     for ds_js_idx, ds_js in jobscripts.items():
         if ds_js_idx <= from_idx:
@@ -270,7 +306,7 @@ def _reindex_dependencies(
 
 @TimeIt.decorator
 def merge_jobscripts_across_tasks(
-    jobscripts: Mapping[int, JobScriptCreationArguments]
+    jobscripts: Mapping[int, JobScriptCreationArguments],
 ) -> Mapping[int, JobScriptCreationArguments]:
     """Try to merge jobscripts between tasks.
 
@@ -284,55 +320,413 @@ def merge_jobscripts_across_tasks(
     merged: set[int] = set()
 
     for js_idx, js in jobscripts.items():
-        # for now only attempt to merge a jobscript with a single dependency:
-        if len(js["dependencies"]) != 1:
+        if not js["dependencies"]:
             continue
-        deps = js["dependencies"]
-        js_j_idx, dep_info = next(iter(deps.items()))
-        js_j = jobscripts[js_j_idx]  # the jobscript we are merging `js` into
 
-        # can only merge if resources are the same and is array dependency:
-        if js["resource_hash"] == js_j["resource_hash"] and dep_info["is_array"]:
-            num_loop_idx = len(
-                js_j["task_loop_idx"]
-            )  # TODO: should this be: `js_j["task_loop_idx"][0]`?
+        closest_idx = cast("int", max(js["dependencies"]))
+        closest_js = jobscripts[closest_idx]
+        other_deps = {k: v for k, v in js["dependencies"].items() if k != closest_idx}
+
+        # if all `other_deps` are also found within `closest_js`'s dependencies, then we
+        # can merge `js` into `closest_js`:
+        merge = True
+        for dep_idx, dep_i in other_deps.items():
+            try:
+                if closest_js["dependencies"][dep_idx] != dep_i:
+                    merge = False
+            except KeyError:
+                merge = False
+
+        if merge:
+            js_j = closest_js  # the jobscript we are merging `js` into
+            js_j_idx = closest_idx
+            dep_info = js["dependencies"][js_j_idx]
 
-            # append task_insert_IDs
-            js_j["task_insert_IDs"].append(js["task_insert_IDs"][0])
-            js_j["task_loop_idx"].append(js["task_loop_idx"][0])
+            # can only merge if resources are the same and is array dependency:
+            if js["resource_hash"] == js_j["resource_hash"] and dep_info["is_array"]:
+                num_loop_idx = len(
+                    js_j["task_loop_idx"]
+                )  # TODO: should this be: `js_j["task_loop_idx"][0]`?
 
-            add_acts = [(a, b, num_loop_idx) for a, b, _ in js["task_actions"]]
+                # append task_insert_IDs
+                js_j["task_insert_IDs"].append(js["task_insert_IDs"][0])
+                js_j["task_loop_idx"].append(js["task_loop_idx"][0])
 
-            js_j["task_actions"].extend(add_acts)
-            for k, v in js["task_elements"].items():
-                js_j["task_elements"][k].extend(v)
+                add_acts = [(a, b, num_loop_idx) for a, b, _ in js["task_actions"]]
 
-            # append to elements and elements_idx list
-            js_j["EAR_ID"] = np.vstack((js_j["EAR_ID"], js["EAR_ID"]))
+                js_j["task_actions"].extend(add_acts)
+                for k, v in js["task_elements"].items():
+                    js_j["task_elements"][k].extend(v)
 
-            # mark this js as defunct
-            merged.add(id(js))
+                # append to elements and elements_idx list
+                js_j["EAR_ID"] = np.vstack((js_j["EAR_ID"], js["EAR_ID"]))
 
-            # update dependencies of any downstream jobscripts that refer to this js
-            _reindex_dependencies(jobscripts, js_idx, js_j_idx)
+                # mark this js as defunct
+                merged.add(id(js))
+
+                # update dependencies of any downstream jobscripts that refer to this js
+                _reindex_dependencies(jobscripts, js_idx, js_j_idx)
 
     # remove is_merged jobscripts:
     return {k: v for k, v in jobscripts.items() if id(v) not in merged}
 
 
 @TimeIt.decorator
-def jobscripts_to_list(
-    jobscripts: Mapping[int, JobScriptCreationArguments]
-) -> Iterator[JobScriptCreationArguments]:
-    """Convert the jobscripts dict to a list, normalising jobscript indices so they refer
-    to list indices; also remove `resource_hash`."""
-    lst: list[JobScriptCreationArguments] = []
-    for new_idx, (js_idx, js) in enumerate(jobscripts.items()):
-        if js_idx != new_idx:
-            # need to reindex jobscripts that depend on this one
-            _reindex_dependencies(jobscripts, js_idx, new_idx)
-        del js["resource_hash"]
-        yield js
+def resolve_jobscript_blocks(
+    jobscripts: Mapping[int, JobScriptCreationArguments],
+) -> list[dict[str, Any]]:
+    """For contiguous, dependent, non-array jobscripts with identical resource
+    requirements, combine into multi-block jobscripts.
+
+    Parameters
+    ----------
+    jobscripts
+        Dict whose values must be dicts with keys "is_array", "resource_hash" and
+        "dependencies".
+    run_parallelism
+        True if the store supports run parallelism
+
+    """
+    js_new: list[
+        list[JobScriptCreationArguments]
+    ] = []  # TODO: not the same type, e.g. dependencies have tuple keys,
+    new_idx: dict[
+        int, tuple[int, int]
+    ] = {}  # track new positions by new jobscript index and block index
+    new_idx_inv: dict[int, list[int]] = defaultdict(list)
+    prev_hash = None
+    blocks: list[JobScriptCreationArguments] = []
+    js_deps_rec: dict[int, set[int]] = {}  # recursive
+    for js_idx, js_i in jobscripts.items():
+
+        cur_js_idx = len(js_new)
+        new_deps_js_j = {
+            new_idx[i][0] for i in cast("Sequence[int]", js_i["dependencies"])
+        }
+        new_deps_js_j_rec = [
+            k for i in new_deps_js_j for j in new_idx_inv[i] for k in js_deps_rec[j]
+        ]
+
+        js_deps_rec[js_idx] = new_deps_js_j.union(new_deps_js_j_rec)
+
+        # recursive dependencies of js_i (which we're looking to merge), excluding the
+        # dependency on the current jobscript:
+        js_j_deps_rec_no_cur = js_deps_rec[js_idx] - set([cur_js_idx])
+
+        # recursive dependencies of the current jobscript:
+        cur_deps_rec = {
+            j for i in new_idx_inv[cur_js_idx] for j in js_deps_rec[i] if j != cur_js_idx
+        }
+
+        # can we mege js_i into the current jobscript, as far as dependencies are
+        # concerned?
+        deps_mergable = cur_js_idx in new_deps_js_j
+        if deps_mergable and js_j_deps_rec_no_cur:
+            deps_mergable = js_j_deps_rec_no_cur == cur_deps_rec
+
+        if js_i["is_array"]:
+            # array jobs cannot be merged into the same jobscript
+
+            # append existing block:
+            if blocks:
+                js_new.append(blocks)
+                prev_hash = None
+                blocks = []
+
+            new_idx[js_idx] = (len(js_new), 0)
+            new_idx_inv[len(js_new)].append(js_idx)
+            js_new.append([js_i])
+            continue
+
+        if js_idx == 0 or prev_hash is None:
+            # (note: zeroth index will always exist)
+
+            # start a new block:
+            blocks.append(js_i)
+            new_idx[js_idx] = (len(js_new), len(blocks) - 1)
+            new_idx_inv[len(js_new)].append(js_idx)
+
+            # set resource hash to compare with the next jobscript
+            prev_hash = js_i["resource_hash"]
+
+        elif js_i["resource_hash"] == prev_hash and deps_mergable:
+            # merge with previous jobscript by adding another block
+            # only merge if this jobscript's dependencies include the current jobscript,
+            # and any other dependencies are included in the current jobscript's
+            # dependencies
+            blocks.append(js_i)
+            new_idx[js_idx] = (len(js_new), len(blocks) - 1)
+            new_idx_inv[len(js_new)].append(js_idx)
+
+        else:
+            # cannot merge, append the new jobscript data:
+            js_new.append(blocks)
+
+            # start a new block:
+            blocks = [js_i]
+            new_idx[js_idx] = (len(js_new), len(blocks) - 1)
+            new_idx_inv[len(js_new)].append(js_idx)
+
+            # set resource hash to compare with the next jobscript
+            prev_hash = js_i["resource_hash"]
+
+    # append remaining blocks:
+    if blocks:
+        js_new.append(blocks)
+        prev_hash = None
+        blocks = []
+
+    # re-index dependencies:
+    js_new_: list[dict[str, Any]] = []
+    for js_i_idx, js_new_i in enumerate(js_new):
+
+        resources = None
+        is_array = None
+        for block_j in js_new_i:
+            for k, v in new_idx.items():
+                dep_data = block_j["dependencies"].pop(k, None)
+                if dep_data:
+                    block_j["dependencies"][v] = dep_data
+
+            del block_j["resource_hash"]
+            resources = block_j.pop("resources", None)
+            is_array = block_j.pop("is_array")
+
+        js_new_.append(
+            {
+                "resources": resources,
+                "is_array": is_array,
+                "blocks": js_new[js_i_idx],
+            }
+        )
+
+    return js_new_
+
+
+@hydrate
+class JobscriptBlock(JSONLike):
+    """A rectangular block of element-actions to run within a jobscript.
+
+    Parameters
+    ----------
+    task_insert_IDs: list[int]
+        The task insertion IDs.
+    task_actions: list[tuple]
+        The actions of the tasks.
+        ``task insert ID, action_idx, index into task_loop_idx`` for each ``JS_ACTION_IDX``
+    task_elements: dict[int, list[int]]
+        The elements of the tasks.
+        Maps ``JS_ELEMENT_IDX`` to list of ``TASK_ELEMENT_IDX`` for each ``TASK_INSERT_ID``
+    EAR_ID:
+        Element action run information.
+    task_loop_idx: list[dict]
+        Description of what loops are in play.
+    dependencies: dict[tuple[int, int], dict]
+        Description of dependencies. Keys are tuples of (jobscript index,
+        jobscript-block index) of the dependency.
+    index: int
+        The index of the block within the parent jobscript.
+    jobscript: ~hpcflow.app.Jobscript
+        The parent jobscript.
+
+    """
+
+    def __init__(
+        self,
+        index: int,
+        task_insert_IDs: list[int],
+        task_loop_idx: list[dict[str, int]],
+        task_actions: list[tuple[int, int, int]] | None = None,
+        task_elements: dict[int, list[int]] | None = None,
+        EAR_ID: NDArray | None = None,
+        dependencies: (
+            dict[tuple[int, int], ResolvedJobscriptBlockDependencies] | None
+        ) = None,
+        jobscript: Jobscript | None = None,
+    ):
+        self.jobscript = jobscript
+        self._index = index
+        self._task_insert_IDs = task_insert_IDs
+        self._task_actions = task_actions
+        self._task_elements = task_elements
+        self._task_loop_idx = task_loop_idx
+        self._EAR_ID = EAR_ID
+        self._dependencies = dependencies
+
+        self._all_EARs = None  # assigned on first access to `all_EARs` property
+
+    @property
+    def index(self) -> int:
+        return self._index
+
+    @property
+    def submission(self) -> Submission:
+        assert self.jobscript is not None
+        return self.jobscript.submission
+
+    @property
+    def task_insert_IDs(self) -> Sequence[int]:
+        """
+        The insertion IDs of tasks in this jobscript-block.
+        """
+        return self._task_insert_IDs
+
+    @property
+    @TimeIt.decorator
+    def task_actions(self) -> NDArray:
+        """
+        The IDs of actions of each task in this jobscript-block.
+        """
+        assert self.jobscript is not None
+        return self.workflow._store.get_jobscript_block_task_actions_array(
+            sub_idx=self.submission.index,
+            js_idx=self.jobscript.index,
+            blk_idx=self.index,
+            task_actions_arr=self._task_actions,
+        )
+
+    @property
+    @TimeIt.decorator
+    def task_elements(self) -> Mapping[int, Sequence[int]]:
+        """
+        The IDs of elements of each task in this jobscript-block.
+        """
+        assert self.jobscript is not None
+        return self.workflow._store.get_jobscript_block_task_elements_map(
+            sub_idx=self.submission.index,
+            js_idx=self.jobscript.index,
+            blk_idx=self.index,
+            task_elems_map=self._task_elements,
+        )
+
+    @property
+    @TimeIt.decorator
+    def EAR_ID(self) -> NDArray:
+        """
+        The array of EAR IDs in this jobscript-block.
+        """
+        assert self.jobscript is not None
+        return self.workflow._store.get_jobscript_block_run_ID_array(
+            sub_idx=self.submission.index,
+            js_idx=self.jobscript.index,
+            blk_idx=self.index,
+            run_ID_arr=self._EAR_ID,
+        )
+
+    @property
+    @TimeIt.decorator
+    def dependencies(
+        self,
+    ) -> Mapping[tuple[int, int], ResolvedJobscriptBlockDependencies]:
+        """
+        The dependency descriptor.
+        """
+        assert self.jobscript is not None
+        return self.workflow._store.get_jobscript_block_dependencies(
+            sub_idx=self.submission.index,
+            js_idx=self.jobscript.index,
+            blk_idx=self.index,
+            js_dependencies=self._dependencies,
+        )
+
+    @property
+    def task_loop_idx(self) -> Sequence[Mapping[str, int]]:
+        """
+        The description of where various task loops are.
+        """
+        return self._task_loop_idx
+
+    @property
+    @TimeIt.decorator
+    def num_actions(self) -> int:
+        """
+        The maximal number of actions in the jobscript-block.
+        """
+        return self.EAR_ID.shape[0]
+
+    @property
+    @TimeIt.decorator
+    def num_elements(self) -> int:
+        """
+        The maximal number of elements in the jobscript-block.
+        """
+        return self.EAR_ID.shape[1]
+
+    @property
+    def workflow(self) -> Workflow:
+        """
+        The associated workflow.
+        """
+        assert self.jobscript is not None
+        return self.jobscript.workflow
+
+    @property
+    @TimeIt.decorator
+    def all_EARs(self) -> Sequence[ElementActionRun]:
+        """
+        Description of EAR information for this jobscript-block.
+        """
+        assert self.jobscript is not None
+        return [i for i in self.jobscript.all_EARs if i.id_ in self.EAR_ID]
+
+    @override
+    def _postprocess_to_dict(self, d: dict[str, Any]) -> dict[str, Any]:
+        dct = super()._postprocess_to_dict(d)
+        del dct["_all_EARs"]
+        dct["_dependencies"] = [[list(k), v] for k, v in self.dependencies.items()]
+        dct = {k.lstrip("_"): v for k, v in dct.items()}
+        dct["EAR_ID"] = cast("NDArray", dct["EAR_ID"]).tolist()
+        return dct
+
+    @classmethod
+    def from_json_like(cls, json_like, shared_data=None):
+        json_like["EAR_ID"] = (
+            np.array(json_like["EAR_ID"]) if json_like["EAR_ID"] is not None else None
+        )
+        if json_like["dependencies"] is not None:
+            # transform list to dict with tuple keys, and transform string keys in
+            # `js_element_mapping` to integers:
+            deps_processed = {}
+            for i in json_like["dependencies"]:
+                deps_processed_i = {
+                    "js_element_mapping": {
+                        int(k): v for k, v in i[1]["js_element_mapping"].items()
+                    },
+                    "is_array": i[1]["is_array"],
+                }
+                deps_processed[tuple(i[0])] = deps_processed_i
+            json_like["dependencies"] = deps_processed
+
+        return super().from_json_like(json_like, shared_data)
+
+    def _get_EARs_arr(self) -> NDArray:
+        """
+        Get all associated EAR objects as a 2D array.
+        """
+        return np.array(self.all_EARs).reshape(self.EAR_ID.shape)
+
+    def get_task_loop_idx_array(self) -> NDArray:
+        """
+        Get an array of task loop indices.
+        """
+        loop_idx = np.empty_like(self.EAR_ID)
+        loop_idx[:] = np.array([i[2] for i in self.task_actions]).reshape(
+            (len(self.task_actions), 1)
+        )
+        return loop_idx
+
+    @TimeIt.decorator
+    def write_EAR_ID_file(self, fp: TextIO):
+        """Write a text file with `num_elements` lines and `num_actions` delimited tokens
+        per line, representing whether a given EAR must be executed."""
+        assert self.jobscript is not None
+        # can't specify "open" newline if we pass the file name only, so pass handle:
+        np.savetxt(
+            fname=fp,
+            X=(self.EAR_ID).T,
+            fmt="%.0f",
+            delimiter=self.jobscript._EAR_files_delimiter,
+        )
 
 
 @hydrate
@@ -391,32 +785,25 @@ class Jobscript(JSONLike):
             name="resources",
             class_name="ElementResources",
         ),
+        ChildObjectSpec(
+            name="blocks",
+            class_name="JobscriptBlock",
+            is_multiple=True,
+            parent_ref="jobscript",
+        ),
     )
 
-    @classmethod
-    def __is_QueuedScheduler(cls, value) -> TypeIs[QueuedScheduler]:
-        return isinstance(value, cls._app.QueuedScheduler)
-
     def __init__(
         self,
-        task_insert_IDs: list[int],
-        task_actions: list[tuple[int, int, int]],
-        task_elements: dict[int, list[int]],
-        EAR_ID: NDArray,
+        index: int,
+        is_array: bool,
         resources: ElementResources,
-        task_loop_idx: list[dict[str, int]],
-        dependencies: dict[int, ResolvedDependencies],
-        submit_time: datetime | None = None,
+        blocks: list[JobscriptBlock],
+        at_submit_metadata: dict[str, Any] | None = None,
         submit_hostname: str | None = None,
         submit_machine: str | None = None,
-        submit_cmdline: list[str] | None = None,
-        scheduler_job_ID: str | None = None,
-        process_ID: int | None = None,
+        shell_idx: int | None = None,
         version_info: VersionInfo | None = None,
-        os_name: str | None = None,
-        shell_name: str | None = None,
-        scheduler_name: str | None = None,
-        running: bool | None = None,
         resource_hash: str | None = None,
         elements: dict[int, list[int]] | None = None,
     ):
@@ -424,74 +811,62 @@ class Jobscript(JSONLike):
             raise AttributeError("resource_hash must not be supplied")
         if elements is not None:
             raise AttributeError("elements must not be supplied")
-        self._task_insert_IDs = task_insert_IDs
-        self._task_loop_idx = task_loop_idx
 
-        # [ (task insert ID, action_idx, index into task_loop_idx) for each JS_ACTION_IDX ]:
-        self._task_actions = task_actions
-
-        # {JS_ELEMENT_IDX: [TASK_ELEMENT_IDX for each TASK_INSERT_ID] }:
-        self._task_elements = task_elements
+        if not isinstance(blocks[0], JobscriptBlock):
+            blocks = [
+                JobscriptBlock(**i, index=idx, jobscript=self)
+                for idx, i in enumerate(blocks)
+            ]
 
-        self._EAR_ID = EAR_ID
+        self._index = index
+        self._blocks = blocks
+        self._at_submit_metadata = at_submit_metadata or {
+            k: None for k in JOBSCRIPT_SUBMIT_TIME_KEYS
+        }
+        self._is_array = is_array
         self._resources = resources
-        self._dependencies = dependencies
 
         # assigned on parent `Submission.submit` (or retrieved form persistent store):
-        self._submit_time = submit_time
         self._submit_hostname = submit_hostname
         self._submit_machine = submit_machine
-        self._submit_cmdline = submit_cmdline
+        self._shell_idx = shell_idx
 
-        self._scheduler_job_ID = scheduler_job_ID
-        self._process_ID = process_ID
         self._version_info = version_info
 
-        # assigned as submit-time:
-        # TODO: these should now always be set in `resources` so shouldn't need these:
-        self._os_name = os_name
-        self._shell_name = shell_name
-        self._scheduler_name = scheduler_name
-
         # assigned by parent Submission
         self._submission: Submission | None = None
-        # assigned by parent Submission
-        self._index: int | None = None
         # assigned on first access to `scheduler` property
         self._scheduler_obj: Scheduler | None = None
         # assigned on first access to `shell` property
         self._shell_obj: Shell | None = None
         # assigned on first access to `submit_time` property
         self._submit_time_obj: datetime | None = None
-        self._running = running
         # assigned on first access to `all_EARs` property
         self._all_EARs: list[ElementActionRun] | None = None
 
+        self._set_parent_refs()
+
     def __repr__(self) -> str:
         return (
             f"{self.__class__.__name__}("
             f"index={self.index!r}, "
-            f"task_insert_IDs={self.task_insert_IDs!r}, "
+            f"blocks={self.blocks!r}, "
             f"resources={self.resources!r}, "
-            f"dependencies={self.dependencies!r}"
             f")"
         )
 
     @override
     def _postprocess_to_dict(self, d: dict[str, Any]) -> dict[str, Any]:
         dct = super()._postprocess_to_dict(d)
-        del dct["_index"]
         del dct["_scheduler_obj"]
         del dct["_shell_obj"]
         del dct["_submit_time_obj"]
         del dct["_all_EARs"]
         dct = {k.lstrip("_"): v for k, v in dct.items()}
-        dct["EAR_ID"] = cast("NDArray", dct["EAR_ID"]).tolist()
         return dct
 
     @classmethod
     def from_json_like(cls, json_like, shared_data=None):
-        json_like["EAR_ID"] = np.array(json_like["EAR_ID"])
         return super().from_json_like(json_like, shared_data)
 
     @property
@@ -499,54 +874,36 @@ class Jobscript(JSONLike):
         """
         Alias for the workflow app in job scripts.
         """
-        return self._workflow_app_alias
+        return self.submission.WORKFLOW_APP_ALIAS
 
     def get_commands_file_name(
-        self, js_action_idx: int | str, shell: Shell | None = None
+        self, block_act_key: BlockActionKey, shell: Shell | None = None
     ) -> str:
         """
         Get the name of a file containing commands for a particular jobscript action.
         """
         return self._app.RunDirAppFiles.get_commands_file_name(
-            js_idx=self.index,
-            js_action_idx=js_action_idx,
+            block_act_key,
             shell=shell or self.shell,
         )
 
     @property
-    def task_insert_IDs(self) -> Sequence[int]:
-        """
-        The insertion IDs of tasks in this jobscript.
-        """
-        return self._task_insert_IDs
-
-    @property
-    def task_actions(self) -> Sequence[tuple[int, int, int]]:
-        """
-        The IDs of actions of each task in this jobscript.
-        """
-        return self._task_actions
-
-    @property
-    def task_elements(self) -> Mapping[int, Sequence[int]]:
-        """
-        The IDs of elements of each task in this jobscript.
-        """
-        return self._task_elements
+    def blocks(self) -> Sequence[JobscriptBlock]:
+        return self._blocks
 
     @property
-    def EAR_ID(self) -> NDArray:
-        """
-        The array of EAR IDs.
-        """
-        return self._EAR_ID
+    def at_submit_metadata(self) -> dict[str, Any]:
+        return self.workflow._store.get_jobscript_at_submit_metadata(
+            sub_idx=self.submission.index,
+            js_idx=self.index,
+            metadata_attr=self._at_submit_metadata,
+        )
 
     @property
-    def all_EAR_IDs(self) -> Iterable[int]:
-        """
-        The IDs of all EARs in this jobscript.
-        """
-        return self.EAR_ID.flatten()
+    def all_EAR_IDs(self) -> NDArray:
+        """Return all run IDs of this jobscripts (across all blocks), removing missing
+        run IDs (i.e. -1 values)"""
+        return np.concatenate([i.EAR_ID[i.EAR_ID >= 0] for i in self.blocks])
 
     @property
     @TimeIt.decorator
@@ -554,11 +911,10 @@ class Jobscript(JSONLike):
         """
         Description of EAR information for this jobscript.
         """
-        if not self._all_EARs:
-            self._all_EARs = self.workflow.get_EARs_from_IDs(self.all_EAR_IDs)
-        return self._all_EARs
+        return self.workflow.get_EARs_from_IDs(self.all_EAR_IDs)
 
     @property
+    @TimeIt.decorator
     def resources(self) -> ElementResources:
         """
         The common resources that this jobscript requires.
@@ -566,18 +922,20 @@ class Jobscript(JSONLike):
         return self._resources
 
     @property
-    def task_loop_idx(self) -> Sequence[Mapping[str, int]]:
-        """
-        The description of where various task loops are.
-        """
-        return self._task_loop_idx
-
-    @property
-    def dependencies(self) -> Mapping[int, ResolvedDependencies]:
+    @TimeIt.decorator
+    def dependencies(self) -> Mapping[tuple[int, int], dict[str, bool]]:
         """
-        The dependency descriptor.
+        The dependency descriptor, accounting for all blocks within this jobscript.
         """
-        return self._dependencies
+        deps = {}
+        for block in self.blocks:
+            for (js_idx, blk_idx), v in block.dependencies.items():
+                if js_idx == self.index:
+                    # block dependency is internal to this jobscript
+                    continue
+                else:
+                    deps[js_idx, blk_idx] = {"is_array": v["is_array"]}
+        return deps
 
     @property
     @TimeIt.decorator
@@ -598,14 +956,15 @@ class Jobscript(JSONLike):
         return max((ear.end_time for ear in self.all_EARs if ear.end_time), default=None)
 
     @property
-    def submit_time(self) -> datetime | None:
+    def submit_time(self):
         """
         When the jobscript was submitted, if known.
         """
-        if self._submit_time_obj is None and self._submit_time is not None:
-            self._submit_time_obj = parse_timestamp(
-                self._submit_time, self.workflow.ts_fmt
-            )
+        if self._submit_time_obj is None:
+            if _submit_time := self.at_submit_metadata["submit_time"]:
+                self._submit_time_obj = parse_timestamp(
+                    _submit_time, self.workflow.ts_fmt
+                )
         return self._submit_time_obj
 
     @property
@@ -622,26 +981,30 @@ class Jobscript(JSONLike):
         """
         return self._submit_machine
 
+    @property
+    def shell_idx(self):
+        return self._shell_idx
+
     @property
     def submit_cmdline(self) -> list[str] | None:
         """
-        The command line used to do the commit, if known.
+        The command line used to submit the jobscript, if known.
         """
-        return self._submit_cmdline
+        return self.at_submit_metadata["submit_cmdline"]
 
     @property
     def scheduler_job_ID(self) -> str | None:
         """
         The job ID from the scheduler, if known.
         """
-        return self._scheduler_job_ID
+        return self.at_submit_metadata["scheduler_job_ID"]
 
     @property
     def process_ID(self) -> int | None:
         """
         The process ID from direct execution, if known.
         """
-        return self._process_ID
+        return self.at_submit_metadata["process_ID"]
 
     @property
     def version_info(self) -> VersionInfo | None:
@@ -674,59 +1037,32 @@ class Jobscript(JSONLike):
         return self.submission.workflow
 
     @property
-    def num_actions(self) -> int:
+    def is_array(self) -> bool:
         """
-        The number of actions in this jobscript.
+        Whether to generate an array job.
         """
-        return self.EAR_ID.shape[0]
+        return self._is_array
 
     @property
-    def num_elements(self) -> int:
+    def os_name(self) -> str:
         """
-        The number of elements in this jobscript.
+        The name of the OS to use.
         """
-        return self.EAR_ID.shape[1]
-
-    @property
-    def is_array(self) -> bool:
-        """
-        Whether to generate an array job.
-        """
-        if self.scheduler_name == "direct":
-            return False
-
-        support_EAR_para = self.workflow._store._features.EAR_parallelism
-        if self.resources.use_job_array is None:
-            return self.num_elements > 1 and support_EAR_para
-        if self.resources.use_job_array and not support_EAR_para:
-            raise ValueError(
-                f"Store type {self.workflow._store!r} does not support element "
-                f"parallelism, so jobs cannot be submitted as scheduler arrays."
-            )
-        return self.resources.use_job_array
-
-    @property
-    def os_name(self) -> str:
-        """
-        The name of the OS to use.
-        """
-        name = self._os_name or self.resources.os_name
-        assert name is not None
-        return name
+        assert self.resources.os_name
+        return self.resources.os_name
 
     @property
-    def shell_name(self) -> str | None:
-        """
-        The name of the shell to use.
-        """
-        return self._shell_name or self.resources.shell
+    def shell_name(self) -> str:
+        assert self.resources.shell
+        return self.resources.shell
 
     @property
-    def scheduler_name(self) -> str | None:
+    def scheduler_name(self) -> str:
         """
         The name of the scheduler to use.
         """
-        return self._scheduler_name or self.resources.scheduler
+        assert self.resources.scheduler
+        return self.resources.scheduler
 
     def _get_submission_os_args(self) -> dict[str, str]:
         return {"linux_release_file": self._app.config.linux_release_file}
@@ -784,21 +1120,8 @@ class Jobscript(JSONLike):
         return f"js_{self.index}_EAR_IDs.txt"
 
     @property
-    def element_run_dir_file_name(self) -> str:
-        """
-        The name of a file containing run directory names.
-        """
-        return f"js_{self.index}_run_dirs.txt"
-
-    @property
-    def direct_stdout_file_name(self) -> str:
-        """File for direct execution stdout."""
-        return f"js_{self.index}_stdout.log"
-
-    @property
-    def direct_stderr_file_name(self) -> str:
-        """File for direct execution stderr."""
-        return f"js_{self.index}_stderr.log"
+    def combined_script_indices_file_name(self) -> str:
+        return f"js_{self.index}_script_indices.txt"
 
     @property
     def direct_win_pid_file_name(self) -> str:
@@ -810,42 +1133,212 @@ class Jobscript(JSONLike):
         """The name of the jobscript file."""
         return f"js_{self.index}{self.shell.JS_EXT}"
 
+    @property
+    def jobscript_functions_name(self):
+        assert self.shell_idx is not None
+        return self.submission.get_jobscript_functions_name(self.shell, self.shell_idx)
+
     @property
     def EAR_ID_file_path(self) -> Path:
         """
         The path to the file containing EAR IDs for this jobscript.
         """
-        return self.submission.path / self.EAR_ID_file_name
+        return self.submission.js_run_ids_path / self.EAR_ID_file_name
 
     @property
-    def element_run_dir_file_path(self) -> Path:
+    def combined_script_indices_file_path(self) -> Path:
         """
-        The path to the file containing run directory names for this jobscript.
+        The path to the file containing script indices, in the case this is a
+        ``combine_scripts=True`` jobscript.
         """
-        return self.submission.path / self.element_run_dir_file_name
+        return (
+            self.submission.js_script_indices_path
+            / self.combined_script_indices_file_name
+        )
 
     @property
     def jobscript_path(self) -> Path:
         """
         The path to the file containing the jobscript file.
         """
-        return self.submission.path / self.jobscript_name
+        return self.submission.js_path / self.jobscript_name
 
     @property
-    def direct_stdout_path(self) -> Path:
+    def jobscript_functions_path(self) -> Path:
+        """
+        The path to the file containing the supporting shell functions."""
+        assert self.shell_idx is not None
+        return self.submission.get_jobscript_functions_path(self.shell, self.shell_idx)
+
+    @property
+    def std_path(self) -> Path:
+        """Directory in which to store jobscript standard out and error stream files."""
+        return self.submission.js_std_path / str(self.index)
+
+    @property
+    def direct_std_out_err_path(self) -> Path:
+        """File path of combined standard output and error streams.
+
+        Notes
+        -----
+        This path will only exist if `resources.combine_jobscript_std` is True. Otherwise,
+        see `direct_stdout_path` and `direct_stderr_path` for the separate stream paths.
+
         """
-        The path to the file containing the stdout from directly executed commands
-        for this jobscript.
+        return self.get_std_out_err_path()
+
+    @property
+    def direct_stdout_path(self) -> Path:
+        """File path to which the jobscript's standard output is saved, for direct
+        execution only.
+
+        Notes
+        -----
+        This returned path be the same as that from `get_stderr_path` if
+        `resources.combine_jobscript_std` is True.
+
         """
-        return self.submission.path / self.direct_stdout_file_name
+        assert not self.is_scheduled
+        return self.get_stdout_path()
 
     @property
     def direct_stderr_path(self) -> Path:
+        """File path to which the jobscript's standard error is saved, for direct
+        execution only.
+
+        Notes
+        -----
+        This returned path be the same as that from `get_stdout_path` if
+        `resources.combine_jobscript_std` is True.
+
+        """
+        assert not self.is_scheduled
+        return self.get_stderr_path()
+
+    def __validate_get_std_path_array_idx(self, array_idx: int | None = None):
+        if array_idx is None and self.is_array:
+            raise ValueError(
+                "`array_idx` must be specified, since this jobscript is an array job."
+            )
+        elif array_idx is not None and not self.is_array:
+            raise ValueError(
+                "`array_idx` should not be specified, since this jobscript is not an "
+                "array job."
+            )
+
+    def _get_stdout_path(self, array_idx: int | None = None) -> Path:
+        """File path to the separate standard output stream.
+
+        Notes
+        -----
+        This path will only exist if `resources.combine_jobscript_std` is False.
+        Otherwise, see `get_std_out_err_path` for the combined stream path.
+
         """
-        The path to the file containing the stderr from directly executed commands
-        for this jobscript.
+        self.__validate_get_std_path_array_idx(array_idx)
+        return self.std_path / self.scheduler.get_stdout_filename(
+            js_idx=self.index, job_ID=self.scheduler_job_ID, array_idx=array_idx
+        )
+
+    def _get_stderr_path(self, array_idx: int | None = None) -> Path:
+        """File path to the separate standard error stream.
+
+        Notes
+        -----
+        This path will only exist if `resources.combine_jobscript_std` is False.
+        Otherwise, see `get_std_out_err_path` for the combined stream path.
+
         """
-        return self.submission.path / self.direct_stderr_file_name
+        self.__validate_get_std_path_array_idx(array_idx)
+        return self.std_path / self.scheduler.get_stderr_filename(
+            js_idx=self.index, job_ID=self.scheduler_job_ID, array_idx=array_idx
+        )
+
+    def get_std_out_err_path(self, array_idx: int | None = None) -> Path:
+        """File path of combined standard output and error streams.
+
+        Notes
+        -----
+        This path will only exist if `resources.combine_jobscript_std` is True. Otherwise,
+        see `get_stdout_path` and `get_stderr_path` for the separate stream paths.
+
+        """
+        self.__validate_get_std_path_array_idx(array_idx)
+        return self.std_path / self.scheduler.get_std_out_err_filename(
+            js_idx=self.index, job_ID=self.scheduler_job_ID, array_idx=array_idx
+        )
+
+    def get_stdout_path(self, array_idx: int | None = None) -> Path:
+        """File path to which the jobscript's standard output is saved.
+
+        Notes
+        -----
+        This returned path be the same as that from `get_stderr_path` if
+        `resources.combine_jobscript_std` is True.
+
+        """
+        if self.resources.combine_jobscript_std:
+            return self.get_std_out_err_path(array_idx=array_idx)
+        else:
+            return self._get_stdout_path(array_idx=array_idx)
+
+    def get_stderr_path(self, array_idx: int | None = None) -> Path:
+        """File path to which the jobscript's standard error is saved.
+
+        Notes
+        -----
+        This returned path be the same as that from `get_stdout_path` if
+        `resources.combine_jobscript_std` is True.
+
+        """
+        if self.resources.combine_jobscript_std:
+            return self.get_std_out_err_path(array_idx=array_idx)
+        else:
+            return self._get_stderr_path(array_idx=array_idx)
+
+    def get_stdout(self, array_idx: int | None = None) -> str:
+        """Retrieve the contents of the standard output stream file.
+
+        Notes
+        -----
+        In the case of non-array jobscripts, this will return the whole standard output,
+        even if that includes multiple elements/actions.
+
+        """
+        return self.workflow.get_text_file(self.get_stdout_path(array_idx))
+
+    def get_stderr(self, array_idx: int | None = None) -> str:
+        """Retrieve the contents of the standard error stream file.
+
+        Notes
+        -----
+        In the case of non-array jobscripts, this will return the whole standard error,
+        even if that includes multiple elements/actions.
+
+        """
+        return self.workflow.get_text_file(self.get_stderr_path(array_idx))
+
+    def print_stdout(self, array_idx: int | None = None) -> None:
+        """Print the contents of the standard output stream file.
+
+        Notes
+        -----
+        In the case of non-array jobscripts, this will print the whole standard output,
+        even if that includes multiple elements/actions.
+
+        """
+        print(self.get_stdout(array_idx))
+
+    def print_stderr(self, array_idx: int | None = None) -> None:
+        """Print the contents of the standard error stream file.
+
+        Notes
+        -----
+        In the case of non-array jobscripts, this will print the whole standard error,
+        even if that includes multiple elements/actions.
+
+        """
+        print(self.get_stderr(array_idx))
 
     @property
     def direct_win_pid_file_path(self) -> Path:
@@ -853,14 +1346,44 @@ class Jobscript(JSONLike):
         The path to the file containing PIDs for directly executed commands for this
         jobscript. Windows only.
         """
-        return self.submission.path / self.direct_win_pid_file_name
+        return self.submission.js_win_pids_path / self.direct_win_pid_file_name
 
-    def _set_submit_time(self, submit_time: datetime) -> None:
-        self._submit_time = submit_time
+    @property
+    def is_scheduled(self) -> bool:
+        return self.scheduler_name not in ("direct", "direct_posix")
+
+    def _update_at_submit_metadata(
+        self,
+        submit_cmdline: list[str] | None = None,
+        scheduler_job_ID: str | None = None,
+        process_ID: int | None = None,
+        submit_time: str | None = None,
+    ):
+        """Update persistent store and in-memory record of at-submit metadata for this
+        jobscript.
+
+        """
         self.workflow._store.set_jobscript_metadata(
             sub_idx=self.submission.index,
             js_idx=self.index,
-            submit_time=submit_time.strftime(self.workflow.ts_fmt),
+            submit_cmdline=submit_cmdline,
+            scheduler_job_ID=scheduler_job_ID,
+            process_ID=process_ID,
+            submit_time=submit_time,
+        )
+
+        if submit_cmdline is not None:
+            self._at_submit_metadata["submit_cmdline"] = submit_cmdline
+        if scheduler_job_ID is not None:
+            self._at_submit_metadata["scheduler_job_ID"] = scheduler_job_ID
+        if process_ID is not None:
+            self._at_submit_metadata["process_ID"] = process_ID
+        if submit_time is not None:
+            self._at_submit_metadata["submit_time"] = submit_time
+
+    def _set_submit_time(self, submit_time: datetime) -> None:
+        self._update_at_submit_metadata(
+            submit_time=submit_time.strftime(self.workflow.ts_fmt)
         )
 
     def _set_submit_hostname(self, submit_hostname: str) -> None:
@@ -879,31 +1402,26 @@ class Jobscript(JSONLike):
             submit_machine=submit_machine,
         )
 
-    def _set_submit_cmdline(self, submit_cmdline: list[str]) -> None:
-        self._submit_cmdline = submit_cmdline
+    def _set_shell_idx(self, shell_idx: int) -> None:
+        self._shell_idx = shell_idx
         self.workflow._store.set_jobscript_metadata(
             sub_idx=self.submission.index,
             js_idx=self.index,
-            submit_cmdline=submit_cmdline,
+            shell_idx=shell_idx,
         )
 
+    def _set_submit_cmdline(self, submit_cmdline: list[str]) -> None:
+        self._update_at_submit_metadata(submit_cmdline=submit_cmdline)
+
     def _set_scheduler_job_ID(self, job_ID: str) -> None:
         """For scheduled submission only."""
-        self._scheduler_job_ID = job_ID
-        self.workflow._store.set_jobscript_metadata(
-            sub_idx=self.submission.index,
-            js_idx=self.index,
-            scheduler_job_ID=job_ID,
-        )
+        assert self.is_scheduled
+        self._update_at_submit_metadata(scheduler_job_ID=job_ID)
 
     def _set_process_ID(self, process_ID: int) -> None:
         """For direct submission only."""
-        self._process_ID = process_ID
-        self.workflow._store.set_jobscript_metadata(
-            sub_idx=self.submission.index,
-            js_idx=self.index,
-            process_ID=process_ID,
-        )
+        assert not self.is_scheduled
+        self._update_at_submit_metadata(process_ID=process_ID)
 
     def _set_version_info(self, version_info: VersionInfo) -> None:
         self._version_info = version_info
@@ -913,145 +1431,38 @@ class Jobscript(JSONLike):
             version_info=version_info,
         )
 
-    def _set_os_name(self) -> None:
-        """Set the OS name for this jobscript. This is invoked at submit-time."""
-        self._os_name = self.resources.os_name
-        self.workflow._store.set_jobscript_metadata(
-            sub_idx=self.submission.index,
-            js_idx=self.index,
-            os_name=self._os_name,
-        )
-
-    def _set_shell_name(self) -> None:
-        """Set the shell name for this jobscript. This is invoked at submit-time."""
-        self._shell_name = self.resources.shell
-        self.workflow._store.set_jobscript_metadata(
-            sub_idx=self.submission.index,
-            js_idx=self.index,
-            shell_name=self._shell_name,
-        )
-
-    def _set_scheduler_name(self) -> None:
-        """Set the scheduler name for this jobscript. This is invoked at submit-time."""
-        self._scheduler_name = self.resources.scheduler
-        if self._scheduler_name:
-            self.workflow._store.set_jobscript_metadata(
-                sub_idx=self.submission.index,
-                js_idx=self.index,
-                scheduler_name=self._scheduler_name,
-            )
-
-    def get_task_loop_idx_array(self) -> NDArray:
-        """
-        Get an array of task loop indices.
-        """
-        loop_idx = np.empty_like(self.EAR_ID)
-        loop_idx[:] = np.array([i[2] for i in self.task_actions]).reshape(
-            (len(self.task_actions), 1)
-        )
-        return loop_idx
-
-    @TimeIt.decorator
-    def write_EAR_ID_file(self):
-        """Write a text file with `num_elements` lines and `num_actions` delimited tokens
-        per line, representing whether a given EAR must be executed."""
-
-        with self.EAR_ID_file_path.open(mode="wt", newline="\n") as fp:
-            # can't specify "open" newline if we pass the file name only, so pass handle:
-            np.savetxt(
-                fname=fp,
-                X=(self.EAR_ID).T,
-                fmt="%.0f",
-                delimiter=self._EAR_files_delimiter,
-            )
-
-    @TimeIt.decorator
-    def write_element_run_dir_file(self, run_dirs: list[list[Path]]):
-        """Write a text file with `num_elements` lines and `num_actions` delimited tokens
-        per line, representing the working directory for each EAR.
-
-        We assume a given task element's actions all run in the same directory, but in
-        general a jobscript "element" may cross task boundaries, so we need to provide
-        the directory for each jobscript-element/jobscript-action combination.
-
-        """
-        run_dirs_paths = self.shell.prepare_element_run_dirs(run_dirs)
-        with self.element_run_dir_file_path.open(mode="wt", newline="\n") as fp:
-            # can't specify "open" newline if we pass the file name only, so pass handle:
-            np.savetxt(
-                fname=fp,
-                X=np.array(run_dirs_paths),
-                fmt="%s",
-                delimiter=self._EAR_files_delimiter,
-            )
-
     @TimeIt.decorator
     def compose_jobscript(
         self,
+        shell,
         deps: dict[int, tuple[str, bool]] | None = None,
         os_name: str | None = None,
-        shell_name: str | None = None,
-        os_args: dict[str, Any] | None = None,
-        shell_args: dict[str, Any] | None = None,
         scheduler_name: str | None = None,
         scheduler_args: dict[str, Any] | None = None,
     ) -> str:
         """Prepare the jobscript file string."""
-
-        os_name = os_name or self.os_name
-        shell_name = shell_name or self.shell_name
         scheduler_name = scheduler_name or self.scheduler_name
-
-        if not os_name:
-            raise RuntimeError(
-                f"Jobscript {self.index} `os_name` is not yet set. Pass the `os_name` as "
-                f"a method argument to compose the jobscript for a given `os_name`."
-            )
-        if not shell_name:
-            raise RuntimeError(
-                f"Jobscript {self.index} `shell_name` is not yet set. Pass the "
-                f"`shell_name` as a method argument to compose the jobscript for a given "
-                f"`shell_name`."
-            )
-        if not scheduler_name:
-            scheduler_name = self._app.config.default_scheduler
-
-        shell = self._get_shell(
-            os_name=os_name,
-            shell_name=shell_name,
-            os_args=os_args or self._get_submission_os_args(),
-            shell_args=shell_args or self._get_submission_shell_args(),
-        )
+        assert scheduler_name
+        assert os_name
         scheduler = self._app.get_scheduler(
             scheduler_name=scheduler_name,
             os_name=os_name,
             scheduler_args=scheduler_args or self._get_submission_scheduler_args(),
         )
-
-        cfg_invocation = self._app.config._file.get_invocation(
-            self._app.config._config_key
-        )
-        if env_setup := cfg_invocation["environment_setup"]:
-            env_setup = indent(env_setup.strip(), shell.JS_ENV_SETUP_INDENT)
-            env_setup += "\n\n" + shell.JS_ENV_SETUP_INDENT
-        else:
-            env_setup = shell.JS_ENV_SETUP_INDENT
-
-        header_args = shell.process_JS_header_args(
-            {
-                "workflow_app_alias": self.workflow_app_alias,
-                "env_setup": env_setup,
-                "app_invoc": list(self._app.run_time_info.invocation_command),
-                "run_log_file": self._app.RunDirAppFiles.get_log_file_name(),
-                "config_dir": str(self._app.config.config_directory),
-                "config_invoc_key": self._app.config.config_key,
-                "workflow_path": self.workflow.path,
-                "sub_idx": self.submission.index,
-                "js_idx": self.index,
-                "EAR_file_name": self.EAR_ID_file_name,
-                "element_run_dirs_file_path": self.element_run_dir_file_name,
-            }
-        )
+        app_caps = self._app.package_name.upper()
+        header_args = {
+            "app_caps": app_caps,
+            "jobscript_functions_name": self.jobscript_functions_name,
+            "jobscript_functions_dir": self.submission.JS_FUNCS_DIR_NAME,
+            "sub_idx": self.submission.index,
+            "js_idx": self.index,
+            "run_IDs_file_name": self.EAR_ID_file_name,
+            "run_IDs_file_dir": self.submission.JS_RUN_IDS_DIR_NAME,
+            "tmp_dir_name": self.submission.TMP_DIR_NAME,
+            "log_dir_name": self.submission.LOG_DIR_NAME,
+            "app_std_dir_name": self.submission.APP_STD_DIR_NAME,
+            "scripts_dir_name": self.submission.SCRIPTS_DIR_NAME,
+        }
 
         shebang = shell.JS_SHEBANG.format(
             shebang_executable=" ".join(shell.shebang_executable),
@@ -1059,14 +1470,15 @@ class Jobscript(JSONLike):
         )
         header = shell.JS_HEADER.format(**header_args)
 
-        if self.__is_QueuedScheduler(scheduler):
+        if isinstance(scheduler, QueuedScheduler):
             header = shell.JS_SCHEDULER_HEADER.format(
                 shebang=shebang,
                 scheduler_options=scheduler.format_options(
                     resources=self.resources,
-                    num_elements=self.num_elements,
+                    num_elements=self.blocks[0].num_elements,  # only used for array jobs
                     is_array=self.is_array,
                     sub_idx=self.submission.index,
+                    js_idx=self.index,
                 ),
                 header=header,
             )
@@ -1085,32 +1497,104 @@ class Jobscript(JSONLike):
                 wait_command=wait_cmd,
             )
 
-        main = shell.JS_MAIN.format(
-            num_actions=self.num_actions,
-            EAR_files_delimiter=self._EAR_files_delimiter,
-            workflow_app_alias=self.workflow_app_alias,
-            commands_file_name=self.get_commands_file_name(r"${JS_act_idx}", shell=shell),
-            run_stream_file=self._app.RunDirAppFiles.get_std_file_name(),
-        )
-
         out = header
 
-        if self.is_array:
-            if not self.__is_QueuedScheduler(scheduler):
-                raise Exception("can only schedule arrays of jobs to a queue")
-            out += shell.JS_ELEMENT_ARRAY.format(
-                scheduler_command=scheduler.js_cmd,
-                scheduler_array_switch=scheduler.array_switch,
-                scheduler_array_item_var=scheduler.array_item_var,
-                num_elements=self.num_elements,
-                main=main,
+        if self.resources.combine_scripts:
+            run_cmd = shell.JS_RUN_CMD_COMBINED.format(
+                workflow_app_alias=self.workflow_app_alias
             )
-
+            out += run_cmd + "\n"
         else:
-            out += shell.JS_ELEMENT_LOOP.format(
-                num_elements=self.num_elements,
-                main=indent(main, shell.JS_INDENT),
+            run_cmd = shell.JS_RUN_CMD.format(workflow_app_alias=self.workflow_app_alias)
+
+            if self.resources.write_app_logs:
+                run_log_enable_disable = shell.JS_RUN_LOG_PATH_ENABLE.format(
+                    run_log_file_name=self.submission.get_app_log_file_name(
+                        run_ID=shell.format_env_var_get(f"{app_caps}_RUN_ID")
+                    )
+                )
+            else:
+                run_log_enable_disable = shell.JS_RUN_LOG_PATH_DISABLE
+
+            block_run = shell.JS_RUN.format(
+                EAR_files_delimiter=self._EAR_files_delimiter,
+                app_caps=app_caps,
+                run_cmd=run_cmd,
+                sub_tmp_dir=self.submission.tmp_path,
+                run_log_enable_disable=run_log_enable_disable,
             )
+            if len(self.blocks) == 1:
+                # forgo element and action loops if not necessary:
+                block = self.blocks[0]
+                if block.num_actions > 1:
+                    block_act = shell.JS_ACT_MULTI.format(
+                        num_actions=block.num_actions,
+                        run_block=indent(block_run, shell.JS_INDENT),
+                    )
+                else:
+                    block_act = shell.JS_ACT_SINGLE.format(run_block=block_run)
+
+                main = shell.JS_MAIN.format(
+                    action=block_act,
+                    app_caps=app_caps,
+                    block_start_elem_idx=0,
+                )
+
+                out += shell.JS_BLOCK_HEADER.format(app_caps=app_caps)
+                if self.is_array:
+                    if not isinstance(scheduler, QueuedScheduler):
+                        raise Exception("can only schedule arrays of jobs to a queue")
+                    out += shell.JS_ELEMENT_MULTI_ARRAY.format(
+                        scheduler_command=scheduler.js_cmd,
+                        scheduler_array_switch=scheduler.array_switch,
+                        scheduler_array_item_var=scheduler.array_item_var,
+                        num_elements=block.num_elements,
+                        main=main,
+                    )
+                elif block.num_elements == 1:
+                    out += shell.JS_ELEMENT_SINGLE.format(
+                        block_start_elem_idx=0,
+                        main=main,
+                    )
+                else:
+                    out += shell.JS_ELEMENT_MULTI_LOOP.format(
+                        block_start_elem_idx=0,
+                        num_elements=block.num_elements,
+                        main=indent(main, shell.JS_INDENT),
+                    )
+
+            else:
+                # use a shell loop for blocks, so always write the inner element and action
+                # loops:
+                block_act = shell.JS_ACT_MULTI.format(
+                    num_actions=shell.format_array_get_item("num_actions", "$block_idx"),
+                    run_block=indent(block_run, shell.JS_INDENT),
+                )
+                main = shell.JS_MAIN.format(
+                    action=block_act,
+                    app_caps=app_caps,
+                    block_start_elem_idx="$block_start_elem_idx",
+                )
+
+                # only non-array jobscripts will have multiple blocks:
+                element_loop = shell.JS_ELEMENT_MULTI_LOOP.format(
+                    block_start_elem_idx="$block_start_elem_idx",
+                    num_elements=shell.format_array_get_item(
+                        "num_elements", "$block_idx"
+                    ),
+                    main=indent(main, shell.JS_INDENT),
+                )
+                out += shell.JS_BLOCK_LOOP.format(
+                    num_elements=shell.format_array(
+                        [i.num_elements for i in self.blocks]
+                    ),
+                    num_actions=shell.format_array([i.num_actions for i in self.blocks]),
+                    num_blocks=len(self.blocks),
+                    app_caps=app_caps,
+                    element_loop=indent(element_loop, shell.JS_INDENT),
+                )
+
+        out += shell.JS_FOOTER
 
         return out
 
@@ -1128,70 +1612,40 @@ class Jobscript(JSONLike):
         """
         Write the jobscript to its file.
         """
+        os_name = os_name or self.os_name
+        shell_name = shell_name or self.shell_name
+        assert os_name
+        assert shell_name
+        shell = self._get_shell(
+            os_name=os_name,
+            shell_name=shell_name,
+            os_args=os_args or self._get_submission_os_args(),
+            shell_args=shell_args or self._get_submission_shell_args(),
+        )
+
         js_str = self.compose_jobscript(
             deps=deps,
+            shell=shell,
             os_name=os_name,
-            shell_name=shell_name,
-            os_args=os_args,
-            shell_args=shell_args,
             scheduler_name=scheduler_name,
             scheduler_args=scheduler_args,
         )
         with self.jobscript_path.open("wt", newline="\n") as fp:
             fp.write(js_str)
-        return self.jobscript_path
-
-    @TimeIt.decorator
-    def make_artifact_dirs(self) -> list[list[Path]]:
-        """
-        Create the directories that will hold artifacts associated with this jobscript.
-        """
-        EARs_arr = np.array(self.all_EARs).reshape(self.EAR_ID.shape)
-        task_loop_idx_arr = self.get_task_loop_idx_array()
-
-        return [
-            [
-                self.__make_action_dir(
-                    EARs_arr[js_act_idx, js_elem_idx],
-                    task_loop_idx_arr[js_act_idx, js_elem_idx].item(),
-                    js_act_idx,
-                    js_elem_idx,
-                )
-                for js_act_idx in range(self.num_actions)
-            ]
-            for js_elem_idx in range(self.num_elements)
-        ]
-
-    def __make_action_dir(
-        self, EAR_i: ElementActionRun, l_idx: int, js_act_idx: int, js_elem_idx: int
-    ) -> Path:
-        t_iID = EAR_i.task.insert_ID
-        r_idx = EAR_i.index
-        loop_idx_i = self.task_loop_idx[l_idx]
-        task_dir = self.workflow.tasks.get(insert_ID=t_iID).get_dir_name(loop_idx_i)
-        elem_dir = EAR_i.element.dir_name
-
-        EAR_dir = self.workflow.execution_path / task_dir / elem_dir / f"r_{r_idx}"
-        EAR_dir.mkdir(exist_ok=True, parents=True)
 
-        # copy (TODO: optionally symlink) any input files:
-        for path in cast("dict[Any, str]", EAR_i.get("input_files", {})).values():
-            if path:
-                shutil.copy(path, EAR_dir)
-        return EAR_dir.relative_to(self.workflow.path)
+        return self.jobscript_path
 
     @TimeIt.decorator
-    def _launch_direct_js_win(self) -> int:
+    def _launch_direct_js_win(self, submit_cmd: list[str]) -> int:
         # this is a "trick" to ensure we always get a fully detached new process (with no
         # parent); the `powershell.exe -Command` process exits after running the inner
         # `Start-Process`, which is where the jobscript is actually invoked. I could not
         # find a way using `subprocess.Popen()` to ensure the new process was fully
         # detached when submitting jobscripts via a Jupyter notebook in Windows.
 
-        assert self.submit_cmdline is not None
         # Note we need powershell.exe for this "launcher process", but the shell used for
         # the jobscript itself need not be powershell.exe
-        exe_path, arg_list = self.submit_cmdline[0], self.submit_cmdline[1:]
+        exe_path, arg_list = submit_cmd[0], submit_cmd[1:]
 
         # note powershell-escaped quotes, in case of spaces in arguments (this seems to
         # work okay even though we might have switch like arguments in this list, like
@@ -1223,23 +1677,30 @@ class Jobscript(JSONLike):
         return int(self.direct_win_pid_file_path.read_text())
 
     @TimeIt.decorator
-    def _launch_direct_js_posix(self) -> int:
+    def _launch_direct_js_posix(self, submit_cmd: list[str]) -> int:
         # direct submission; submit jobscript asynchronously:
         # detached process, avoid interrupt signals propagating to the subprocess:
-        assert self.submit_cmdline is not None
-        with self.direct_stdout_path.open(
-            "wt"
-        ) as fp_stdout, self.direct_stderr_path.open("wt") as fp_stderr:
+
+        def _launch(fp_stdout: TextIO, fp_stderr: TextIO) -> int:
             # note: Popen copies the file objects, so this works!
             proc = subprocess.Popen(
-                args=self.submit_cmdline,
+                args=submit_cmd,
                 stdout=fp_stdout,
                 stderr=fp_stderr,
-                cwd=self.workflow.path,
+                cwd=str(self.workflow.path),
                 start_new_session=True,
             )
             return proc.pid
 
+        if self.resources.combine_jobscript_std:
+            with self.direct_std_out_err_path.open("wt") as fp_std:
+                return _launch(fp_std, fp_std)
+        else:
+            with self.direct_stdout_path.open(
+                "wt"
+            ) as fp_stdout, self.direct_stderr_path.open("wt") as fp_stderr:
+                return _launch(fp_stdout, fp_stderr)
+
     @TimeIt.decorator
     def _launch_queued(
         self, submit_cmd: list[str], print_stdout: bool
@@ -1271,31 +1732,39 @@ class Jobscript(JSONLike):
         # map each dependency jobscript index to the JS ref (job/process ID) and if the
         # dependency is an array dependency:
         deps: dict[int, tuple[str, bool]] = {}
-        for js_idx, deps_i in self.dependencies.items():
+        for (js_idx, _), deps_i in self.dependencies.items():
             dep_js_ref, dep_js_is_arr = scheduler_refs[js_idx]
             # only submit an array dependency if both this jobscript and the dependency
             # are array jobs:
             dep_is_arr = deps_i["is_array"] and self.is_array and dep_js_is_arr
             deps[js_idx] = (dep_js_ref, dep_is_arr)
 
-        if not self.submission.JS_parallelism and self.index > 0:
-            # add fake dependencies to all previously submitted jobscripts to avoid
-            # simultaneous execution:
-            for js_idx, (js_ref, _) in scheduler_refs.items():
-                if js_idx not in deps:
-                    deps[js_idx] = (js_ref, False)
+        if self.index > 0:
+            # prevent this jobscript executing if jobscript parallelism is not available:
+            use_parallelism = (
+                self.submission.JS_parallelism is True
+                or {0: "direct", 1: "scheduled"}[self.is_scheduled]
+                == self.submission.JS_parallelism
+            )
+            if not use_parallelism:
+                # add fake dependencies to all previously submitted jobscripts to avoid
+                # simultaneous execution:
+                for js_idx, (js_ref, _) in scheduler_refs.items():
+                    if js_idx not in deps:
+                        deps[js_idx] = (js_ref, False)
+
+        # make directory for jobscripts stdout/err stream files:
+        self.std_path.mkdir(exist_ok=True)
+
+        with self.EAR_ID_file_path.open(mode="wt", newline="\n") as ID_fp:
+            for block in self.blocks:
+                block.write_EAR_ID_file(ID_fp)
 
-        run_dirs = self.make_artifact_dirs()
-        self.write_EAR_ID_file()
-        self.write_element_run_dir_file(run_dirs)
         js_path = self.shell.prepare_JS_path(self.write_jobscript(deps=deps))
         submit_cmd = self.scheduler.get_submit_command(self.shell, js_path, deps)
         self._app.submission_logger.info(
             f"submitting jobscript {self.index!r} with command: {submit_cmd!r}"
         )
-        self._set_submit_cmdline(submit_cmd)
-        self._set_submit_hostname(socket.gethostname())
-        self._set_submit_machine(self._app.config.get("machine"))
 
         err_args: JobscriptSubmissionFailureArgs = {
             "submit_cmd": submit_cmd,
@@ -1305,23 +1774,23 @@ class Jobscript(JSONLike):
         job_ID: str | None = None
         process_ID: int | None = None
         try:
-            if self.__is_QueuedScheduler(self.scheduler):
+            if isinstance(self.scheduler, QueuedScheduler):
                 # scheduled submission, wait for submission so we can parse the job ID:
                 stdout, stderr = self._launch_queued(submit_cmd, print_stdout)
                 err_args["stdout"] = stdout
                 err_args["stderr"] = stderr
             else:
                 if os.name == "nt":
-                    process_ID = self._launch_direct_js_win()
+                    process_ID = self._launch_direct_js_win(submit_cmd)
                 else:
-                    process_ID = self._launch_direct_js_posix()
+                    process_ID = self._launch_direct_js_posix(submit_cmd)
         except Exception as subprocess_exc:
             err_args["subprocess_exc"] = subprocess_exc
             raise JobscriptSubmissionFailure(
                 "Failed to execute submit command.", **err_args
             )
 
-        if self.__is_QueuedScheduler(self.scheduler):
+        if isinstance(self.scheduler, QueuedScheduler):
             # scheduled submission
             if stderr:
                 raise JobscriptSubmissionFailure(
@@ -1348,13 +1817,15 @@ class Jobscript(JSONLike):
             # direct submission
             assert process_ID is not None
             self._set_process_ID(process_ID)
-            # a downstream direct jobscript might need to wait for this jobscript, which
-            # means this jobscript's process ID must be committed:
-            self.workflow._store._pending.commit_all()
-            ref = f"{process_ID}"
+            ref = str(process_ID)
 
+        self._set_submit_cmdline(submit_cmd)
         self._set_submit_time(current_timestamp())
 
+        # a downstream direct jobscript might need to wait for this jobscript, which
+        # means this jobscript's process ID must be committed:
+        self.workflow._store._pending.commit_all()
+
         return ref
 
     @property
@@ -1363,41 +1834,35 @@ class Jobscript(JSONLike):
         return self.index in self.submission.submitted_jobscripts
 
     @property
-    def scheduler_js_ref(self):
+    def scheduler_js_ref(self) -> str | None | tuple[int | None, list[str] | None]:
         """
         The reference to the submitted job for the jobscript.
         """
-        if self.__is_QueuedScheduler(self.scheduler):
+        if isinstance(self.scheduler, QueuedScheduler):
             return self.scheduler_job_ID
         else:
             return (self.process_ID, self.submit_cmdline)
 
-    @property
-    def scheduler_ref(self) -> SchedulerRef:
-        """
-        The generalised scheduler reference descriptor.
-        """
-        return {"js_refs": [self.scheduler_js_ref], "num_js_elements": self.num_elements}
-
     @overload
     def get_active_states(
         self, as_json: Literal[False] = False
-    ) -> Mapping[int, JobscriptElementState]:
+    ) -> Mapping[int, Mapping[int, JobscriptElementState]]:
         ...
 
     @overload
-    def get_active_states(self, as_json: Literal[True]) -> dict[int, str]:
+    def get_active_states(
+        self, as_json: Literal[True]
+    ) -> Mapping[int, Mapping[int, str]]:
         ...
 
     @TimeIt.decorator
     def get_active_states(
         self, as_json: bool = False
-    ) -> Mapping[int, JobscriptElementState] | dict[int, str]:
+    ) -> Mapping[int, Mapping[int, JobscriptElementState | str]]:
         """If this jobscript is active on this machine, return the state information from
         the scheduler."""
-
-        out: dict[int, JobscriptElementState] = {}
-
+        # this returns: {BLOCK_IDX: {JS_ELEMENT_IDX: STATE}}
+        out: Mapping[int, Mapping[int, JobscriptElementState]] = {}
         if self.is_submitted:
             self._app.submission_logger.debug(
                 "checking if the jobscript is running according to EAR submission "
@@ -1407,7 +1872,7 @@ class Jobscript(JSONLike):
             not_run_states = EARStatus.get_non_running_submitted_states()
             all_EAR_states = set(ear.status for ear in self.all_EARs)
             self._app.submission_logger.debug(
-                f"Unique EAR states are: {all_EAR_states!r}"
+                f"Unique EAR states are: {tuple(i.name for i in all_EAR_states)!r}"
             )
             if all_EAR_states.issubset(not_run_states):
                 self._app.submission_logger.debug(
@@ -1419,29 +1884,483 @@ class Jobscript(JSONLike):
                     "Checking if jobscript is running according to the scheduler/process "
                     "ID."
                 )
-                out_d = self.scheduler.get_job_state_info(**self.scheduler_ref)
+                out_d = self.scheduler.get_job_state_info(js_refs=[self.scheduler_js_ref])
                 if out_d:
-                    out_i = out_d[next(iter(out_d))]  # first item only
-                    # if value is single-length dict with `None` key, then transform
-                    # to one key for each jobscript element:
-                    if tuple(out_i) == (None,):
-                        out = {idx: out_i[None] for idx in range(self.num_elements)}
+                    # remove scheduler ref (should be only one):
+                    assert len(out_d) == 1
+                    out_i = nth_value(cast("dict", out_d), 0)
+
+                    if self.is_array:
+                        # out_i is a dict keyed by array index; there will be exactly one
+                        # block:
+                        out = {0: out_i}
                     else:
-                        out = cast("Any", out_i)
+                        # out_i is a single state:
+                        out = {
+                            idx: {i: out_i for i in range(block.num_elements)}
+                            for idx, block in enumerate(self.blocks)
+                        }
 
             else:
                 raise NotSubmitMachineError()
 
         self._app.submission_logger.info(f"Jobscript is {'in' if not out else ''}active.")
         if as_json:
-            return {idx: state.name for idx, state in out.items()}
+            return {
+                block_idx: {k: v.name for k, v in block_data.items()}
+                for block_idx, block_data in out.items()
+            }
         return out
 
-    def cancel(self) -> None:
+    def compose_combined_script(
+        self, action_scripts: list[list[tuple[str, Path, bool]]]
+    ) -> tuple[str, list[list[int]], list[int], list[int]]:
         """
-        Cancel this jobscript.
+        Prepare the combined-script file string, if applicable.
         """
-        self._app.submission_logger.info(
-            f"Cancelling jobscript {self.index} of submission {self.submission.index}"
+
+        # use an index array for action scripts:
+        script_names: list[str] = []
+        requires_dir: list[bool] = []
+        script_data: dict[str, tuple[int, Path]] = {}
+        script_indices: list[list[int]] = []
+        for i in action_scripts:
+            indices_i: list[int] = []
+            for name_j, path_j, req_dir_i in i:
+                if name_j in script_data:
+                    idx = script_data[name_j][0]
+                else:
+                    idx = len(script_names)
+                    script_names.append(name_j)
+                    requires_dir.append(req_dir_i)
+                    script_data[name_j] = (idx, path_j)
+                indices_i.append(idx)
+            script_indices.append(indices_i)
+
+        if not self.resources.combine_scripts:
+            raise TypeError(
+                f"Jobscript {self.index} is not a `combine_scripts` jobscript."
+            )
+
+        tab_indent = "    "
+
+        script_funcs_lst: list[str] = []
+        for act_name, (_, snip_path) in script_data.items():
+            main_func_name = snip_path.stem
+            with snip_path.open("rt") as fp:
+                script_str = fp.read()
+            script_funcs_lst.append(
+                dedent(
+                    """\
+                def {act_name}(*args, **kwargs):
+                {script_str}
+                    return {main_func_name}(*args, **kwargs)
+                """
+                ).format(
+                    act_name=act_name,
+                    script_str=indent(script_str, tab_indent),
+                    main_func_name=main_func_name,
+                )
+            )
+
+        app_caps = self._app.package_name.upper()
+        if self.resources.write_app_logs:
+            sub_log_path = f'os.environ["{app_caps}_LOG_PATH"]'
+        else:
+            sub_log_path = '""'
+
+        py_imports = dedent(
+            """\
+            import os
+            from collections import defaultdict
+            from pathlib import Path
+            import traceback
+            import time
+            from typing import Dict
+
+            import {app_module} as app
+
+            from hpcflow.sdk.core.errors import UnsetParameterDataErrorBase
+
+            log_path = {log_path}
+            wk_path = os.getenv("{app_caps}_WK_PATH")
+            """
+        ).format(
+            app_module=self._app.module,
+            app_caps=app_caps,
+            log_path=sub_log_path,
+        )
+
+        py_main_block_workflow_load = dedent(
+            """\
+                app.load_config(
+                    log_file_path=log_path,
+                    config_dir=r"{cfg_dir}",
+                    config_key=r"{cfg_invoc_key}",
+                )
+                wk = app.Workflow(wk_path)
+            """
+        ).format(
+            cfg_dir=self._app.config.config_directory,
+            cfg_invoc_key=self._app.config.config_key,
+            app_caps=app_caps,
+        )
+
+        func_invoc_lines = dedent(
+            """\
+                import pprint
+                if not run.action.is_OFP and run.action.script_data_out_has_direct:
+                    outputs = func(**func_kwargs)
+                elif run.action.is_OFP:
+                    out_name = run.action.output_file_parsers[0].output.typ
+                    outputs = {out_name: func(**func_kwargs)}
+                else:
+                    outputs = {}
+                    func(**func_kwargs)
+            """
+        )
+
+        script_funcs = "\n".join(script_funcs_lst)
+        script_names_str = "[" + ", ".join(f"{i}" for i in script_names) + "]"
+        main = dedent(
+            """\
+            {py_imports}
+            
+            sub_std_path = Path(os.environ["{app_caps}_SUB_STD_DIR"], f"js_{js_idx}.txt")            
+            with app.redirect_std_to_file(sub_std_path):
+            {py_main_block_workflow_load}
+
+                with open(os.environ["{app_caps}_RUN_ID_FILE"], mode="r") as fp:
+                    lns = fp.read().strip().split("\\n")
+                    run_IDs = [[int(i) for i in ln.split("{run_ID_delim}")] for ln in lns]
+
+                get_all_runs_tic = time.perf_counter()
+                run_IDs_flat = [j for i in run_IDs for j in i]
+                runs = wk.get_EARs_from_IDs(run_IDs_flat, as_dict=True)
+                run_skips : Dict[int, bool] = {{k: v.skip for k, v in runs.items()}}
+                get_all_runs_toc = time.perf_counter()
+
+                with open(os.environ["{app_caps}_SCRIPT_INDICES_FILE"], mode="r") as fp:
+                    lns = fp.read().strip().split("\\n")
+                    section_idx = -1
+                    script_indices = []
+                    for ln in lns:
+                        if ln.startswith("#"):
+                            section_idx += 1
+                            continue
+                        ln_parsed = [int(i) for i in ln.split("{script_idx_delim}")]
+                        if section_idx == 0:
+                            num_elements = ln_parsed
+                        elif section_idx == 1:
+                            num_actions = ln_parsed
+                        else:
+                            script_indices.append(ln_parsed)
+
+                port = int(os.environ["{app_caps}_RUN_PORT"])
+                action_scripts = {script_names}
+                requires_dir = {requires_dir!r}
+                run_dirs = wk.get_run_directories()
+
+                get_ins_time_fp = open(f"js_{js_idx}_get_inputs_times.txt", "wt")
+                func_time_fp = open(f"js_{js_idx}_func_times.txt", "wt")
+                run_time_fp = open(f"js_{js_idx}_run_times.txt", "wt")
+                set_start_multi_times_fp = open(f"js_{js_idx}_set_start_multi_times.txt", "wt")
+                set_end_multi_times_fp = open(f"js_{js_idx}_set_end_multi_times.txt", "wt")
+                save_multi_times_fp = open(f"js_{js_idx}_save_multi_times.txt", "wt")
+                loop_term_times_fp = open(f"js_{js_idx}_loop_term_times.txt", "wt")
+
+                get_all_runs_time = get_all_runs_toc - get_all_runs_tic
+                print(f"get_all_runs_time: {{get_all_runs_time:.4f}}")
+ 
+            app.logger.info(
+                f"running {num_blocks} jobscript block(s) in combined jobscript index "
+                f"{js_idx}."
+            )
+
+            block_start_elem_idx = 0
+            for block_idx in range({num_blocks}):
+
+                app.logger.info(f"running block index {{block_idx}}.")
+
+                os.environ["{app_caps}_BLOCK_IDX"] = str(block_idx)
+
+                block_run_IDs = [
+                    run_IDs[block_start_elem_idx + i]
+                    for i in range(num_elements[block_idx])
+                ]
+
+                for block_act_idx in range(num_actions[block_idx]):
+                
+                    app.logger.info(
+                        f"running block action index {{block_act_idx}} "
+                        f"(in block {{block_idx}})."
+                    )
+
+                    os.environ["{app_caps}_BLOCK_ACT_IDX"] = str(block_act_idx)
+
+                    block_act_run_IDs = [i[block_act_idx] for i in block_run_IDs]
+
+                    block_act_std_path = Path(
+                        os.environ["{app_caps}_SUB_STD_DIR"],
+                        f"js_{js_idx}_blk_{{block_idx}}_blk_act_{{block_act_idx}}.txt",
+                    )
+                    with app.redirect_std_to_file(block_act_std_path):
+                        # set run starts for all runs of the block/action:
+                        block_act_run_dirs = [run_dirs[i] for i in block_act_run_IDs]
+                        block_act_runs = [runs[i] for i in block_act_run_IDs]
+
+                        block_act_run_IDs_non_skipped = []
+                        block_act_run_dirs_non_skipped = []
+                        for i, j in zip(block_act_run_IDs, block_act_run_dirs):
+                            if not run_skips[i]:
+                                block_act_run_IDs_non_skipped.append(i)
+                                block_act_run_dirs_non_skipped.append(j)
+                                                     
+                        if block_act_run_IDs_non_skipped:
+                            set_start_multi_tic = time.perf_counter()
+                            app.logger.info("setting run starts.")
+                            wk.set_multi_run_starts(block_act_run_IDs_non_skipped, block_act_run_dirs_non_skipped, port)
+                            app.logger.info("finished setting run starts.")
+                            set_start_multi_toc = time.perf_counter()
+                            set_start_multi_time = set_start_multi_toc - set_start_multi_tic
+                            print(f"{{set_start_multi_time:.4f}}", file=set_start_multi_times_fp, flush=True)
+
+                    all_act_outputs = {{}}
+                    run_end_dat = defaultdict(list)
+                    block_act_key=({js_idx}, block_idx, block_act_idx)
+
+                    for block_elem_idx in range(num_elements[block_idx]):
+
+                        js_elem_idx = block_start_elem_idx + block_elem_idx
+                        run_ID = block_act_run_IDs[block_elem_idx]
+
+                        app.logger.info(
+                            f"run_ID is {{run_ID}}; block element index: {{block_elem_idx}}; "
+                            f"block action index: {{block_act_idx}}; in block {{block_idx}}."
+                        )
+
+                        if run_ID == -1:                            
+                            continue
+
+                        run = runs[run_ID]
+
+                        skip = run_skips[run_ID]
+                        if skip:
+                            app.logger.info(f"run_ID: {{run_ID}}; run is set to skip; skipping.")
+                            # set run end
+                            run_end_dat[block_act_key].append((run, {skipped_exit_code}, None))
+                            continue
+                            
+                        run_tic = time.perf_counter()
+
+                        os.environ["{app_caps}_BLOCK_ELEM_IDX"] = str(block_elem_idx)
+                        os.environ["{app_caps}_JS_ELEM_IDX"] = str(js_elem_idx)                        
+                        os.environ["{app_caps}_RUN_ID"] = str(run_ID)
+
+                        std_path = Path(os.environ["{app_caps}_SUB_STD_DIR"], f"{{run_ID}}.txt")
+                        with app.redirect_std_to_file(std_path):
+
+                            if {write_app_logs!r}:
+                                new_log_path = Path(
+                                    os.environ["{app_caps}_SUB_LOG_DIR"],
+                                    f"{run_log_name}",
+                                )
+                                # TODO: this doesn't work!
+                                app.logger.info(
+                                    f"run_ID: {{run_ID}}; moving log path to {{new_log_path}}"
+                                )
+                                app.config.log_path = new_log_path
+
+                            run_dir = run_dirs[run_ID]
+
+                            script_idx = script_indices[block_idx][block_act_idx]
+                            req_dir = requires_dir[script_idx]
+                            if req_dir:
+                                app.logger.info(f"run_ID: {{run_ID}}; changing to run directory: {{run_dir}}")
+                                os.chdir(run_dir)
+                            
+                            # retrieve script inputs:
+                            app.logger.info(f"run_ID: {{run_ID}}; retrieving script inputs.")
+                            get_ins_tic = time.perf_counter()
+                            try:
+                                with run.raise_on_failure_threshold() as unset_params:
+                                    app.logger.info(f"run_ID: {{run_ID}}; writing script input files.")
+                                    run.write_script_input_files(block_act_key)
+
+                                    app.logger.info(f"run_ID: {{run_ID}}; retrieving funcion kwargs.")
+                                    func_kwargs = run.get_py_script_func_kwargs(
+                                        raise_on_unset=False,
+                                        add_script_files=True,
+                                        blk_act_key=block_act_key,
+                                    )
+                                    app.logger.info(
+                                        f"run_ID: {{run_ID}}; script inputs have keys: "
+                                        f"{{tuple(func_kwargs.keys())!r}}."
+                                    )                                
+                            except UnsetParameterDataErrorBase:
+                                # not all required parameter data is set, so fail this run:
+                                exit_code = 1
+                                run_end_dat[block_act_key].append((run, exit_code, None))
+                                app.logger.info(
+                                    f"run_ID: {{run_ID}}; some parameter data is unset, "
+                                    f"so cannot run; setting exit code to 1."
+                                )
+                                continue # don't run the function
+
+                            get_ins_toc = time.perf_counter()
+
+                            func = action_scripts[script_idx]
+                            app.logger.info(f"run_ID: {{run_ID}}; function to run is: {{func.__name__}}")
+
+
+                        try:
+                            func_tic = time.perf_counter()
+                            app.logger.info(f"run_ID: {{run_ID}}; invoking function.")
+            {func_invoc_lines}
+                            
+                        except Exception:
+                            print(f"Exception caught during execution of script function {{func.__name__}}.")
+                            traceback.print_exc()
+                            exit_code = 1
+                            outputs = {{}}
+                        else:
+                            app.logger.info(f"run_ID: {{run_ID}}; finished function invocation.")
+                            exit_code = 0
+                        finally:
+                            func_toc = time.perf_counter()
+                            
+                        with app.redirect_std_to_file(std_path):
+                            # set run end
+                            block_act_key=({js_idx}, block_idx, block_act_idx)
+                            run_end_dat[block_act_key].append((run, exit_code, run_dir))
+
+                            # store outputs to save at end:
+                            app.logger.info(f"run_ID: {{run_ID}}; setting outputs to save.")
+                            for name_i, out_i in outputs.items():
+                                p_id = run.data_idx[f"outputs.{{name_i}}"]
+                                all_act_outputs[p_id] = out_i
+                            app.logger.info(f"run_ID: {{run_ID}}; finished setting outputs to save.")
+                                
+                            if req_dir:
+                                app.logger.info(f"run_ID: {{run_ID}}; changing directory back")
+                                os.chdir(os.environ["{app_caps}_SUB_TMP_DIR"])
+
+                            if {write_app_logs!r}:
+                                app.logger.info(f"run_ID: {{run_ID}}; moving log path back to " + {sub_log_path!r})
+                                app.config.log_path = {sub_log_path}
+
+                        run_toc = time.perf_counter()
+
+                        get_ins_time = get_ins_toc - get_ins_tic
+                        func_time = func_toc - func_tic
+                        run_time = run_toc - run_tic
+
+                        print(f"{{get_ins_time:.4f}}", file=get_ins_time_fp)
+                        print(f"{{func_time:.4f}}", file=func_time_fp)
+                        print(f"{{run_time:.4f}}", file=run_time_fp)
+
+                    with app.redirect_std_to_file(block_act_std_path):
+                    
+                        if all_act_outputs:
+                            # save outputs of all elements of this action
+                            save_all_tic = time.perf_counter()
+                            app.logger.info(
+                                f"saving outputs of block action index {{block_act_idx}} "
+                                f"in block {{block_idx}}."
+                            )                            
+                            wk.set_parameter_values(all_act_outputs)
+                            app.logger.info(
+                                f"finished saving outputs of block action index {{block_act_idx}} "
+                                f"in block {{block_idx}}."
+                            )
+                            save_all_toc = time.perf_counter()
+                            save_all_time_i = save_all_toc - save_all_tic
+                            print(f"{{save_all_time_i:.4f}}", file=save_multi_times_fp, flush=True)
+
+                        all_loop_term_tic = time.perf_counter()
+                        app.logger.info(f"run_ID: {{run_ID}}; checking for loop terminations")
+                        for run_i in block_act_runs:
+                            if not run_skips[run_i.id_]:
+                                skipped_IDs_i = wk._check_loop_termination(run_i)
+                                for skip_ID in skipped_IDs_i:
+                                    run_skips[skip_ID] =  2 # SkipReason.LOOP_TERMINATION
+                                    if skip_ID in runs:
+                                        runs[skip_ID]._skip = 2 # mutates runs within `run_end_dat`
+                        app.logger.info(f"run_ID: {{run_ID}}; finished checking for loop terminations.")
+                                    
+                        all_loop_term_toc = time.perf_counter()
+                        all_loop_term_time_i = all_loop_term_toc - all_loop_term_tic
+                        print(f"{{all_loop_term_time_i:.4f}}", file=loop_term_times_fp, flush=True)
+
+                        # set run end for all elements of this action
+                        app.logger.info(f"run_ID: {{run_ID}}; setting run ends.")
+                        set_multi_end_tic = time.perf_counter()
+                        wk.set_multi_run_ends(run_end_dat)
+                        set_multi_end_toc = time.perf_counter()
+                        set_multi_end_time = set_multi_end_toc - set_multi_end_tic
+                        app.logger.info(f"run_ID: {{run_ID}}; finished setting run ends.")
+                        print(f"{{set_multi_end_time:.4f}}", file=set_end_multi_times_fp, flush=True)
+
+                block_start_elem_idx += num_elements[block_idx]
+
+            get_ins_time_fp.close()
+            func_time_fp.close()
+            run_time_fp.close()
+            set_start_multi_times_fp.close()
+            set_end_multi_times_fp.close()
+            save_multi_times_fp.close()
+            loop_term_times_fp.close()
+        """
+        ).format(
+            py_imports=py_imports,
+            py_main_block_workflow_load=indent(py_main_block_workflow_load, tab_indent),
+            app_caps=self._app.package_name.upper(),
+            script_idx_delim=",",  # TODO
+            script_names=script_names_str,
+            requires_dir=requires_dir,
+            num_blocks=len(self.blocks),
+            run_ID_delim=self._EAR_files_delimiter,
+            run_log_name=self.submission.get_app_log_file_name(run_ID="{run_ID}"),
+            js_idx=self.index,
+            write_app_logs=self.resources.write_app_logs,
+            sub_log_path=sub_log_path,
+            skipped_exit_code=SKIPPED_EXIT_CODE,
+            func_invoc_lines=indent(func_invoc_lines, tab_indent * 4),
+        )
+
+        script = dedent(
+            """\
+            {script_funcs}
+            if __name__ == "__main__":
+            {main}
+        """
+        ).format(script_funcs=script_funcs, main=indent(main, tab_indent))
+
+        num_elems = [i.num_elements for i in self.blocks]
+        num_acts = [len(i) for i in action_scripts]
+
+        return script, script_indices, num_elems, num_acts
+
+    def write_script_indices_file(
+        self, indices: list[list[int]], num_elems: list[int], num_acts: list[int]
+    ) -> None:
+        """
+        Write a text file containing the action script index for each block and action
+        in a `combined_scripts` script.
+        """
+        delim = ","  # TODO: refactor?
+        with self.combined_script_indices_file_path.open("wt") as fp:
+            fp.write("# number of elements per block:\n")
+            fp.write(delim.join(str(i) for i in num_elems) + "\n")
+            fp.write("# number of actions per block:\n")
+            fp.write(delim.join(str(i) for i in num_acts) + "\n")
+            fp.write("# script indices:\n")
+            for block in indices:
+                fp.write(delim.join(str(i) for i in block) + "\n")
+
+    def get_app_std_path(self) -> Path:
+        std_dir = self.submission.get_app_std_path(
+            self.workflow.submissions_path,
+            self.submission.index,
         )
-        self.scheduler.cancel_jobs(**self.scheduler_ref, jobscripts=[self])
+        return std_dir / f"js_{self.index}.txt"  # TODO: refactor