hpcflow-new2 0.2.0a190__py3-none-any.whl → 0.2.0a200__py3-none-any.whl

hpcflow/sdk/submission/schedulers/__init__.py

@@ -30,6 +30,13 @@ class Scheduler(ABC, Generic[JSRefType], AppAware):
     """
     Abstract base class for schedulers.
 
+    Note
+    ----
+    Do not make immediate subclasses of this class other than
+    :py:class:`DirectScheduler` and :py:class:`QueuedScheduler`;
+    subclass those two instead. Code (e.g., in :py:class:`Jobscript`)
+    assumes that this model is followed and does not check it.
+
     Parameters
     ----------
     shell_args: str
@@ -119,8 +126,8 @@ class Scheduler(ABC, Generic[JSRefType], AppAware):
 
     @abstractmethod
     def get_job_state_info(
-        self, *, js_refs: Sequence[JSRefType] | None = None, num_js_elements: int = 0
-    ) -> Mapping[str, Mapping[int | None, JobscriptElementState]]:
+        self, *, js_refs: Sequence[JSRefType] | None = None
+    ) -> Mapping[str, JobscriptElementState | Mapping[int, JobscriptElementState]]:
         """
         Get the state of one or more jobscripts.
         """
@@ -136,12 +143,23 @@ class Scheduler(ABC, Generic[JSRefType], AppAware):
         self,
         js_refs: list[JSRefType],
         jobscripts: list[Jobscript] | None = None,
-        num_js_elements: int = 0,  # Ignored!
     ) -> None:
         """
         Cancel one or more jobscripts.
         """
 
+    @abstractmethod
+    def get_std_out_err_filename(self, js_idx: int, *args, **kwargs) -> str:
+        """File name of combined standard output and error streams."""
+
+    @abstractmethod
+    def get_stdout_filename(self, js_idx: int, *args, **kwargs) -> str:
+        """File name of the standard output stream file."""
+
+    @abstractmethod
+    def get_stderr_filename(self, js_idx: int, *args, **kwargs) -> str:
+        """File name of the standard error stream file."""
+
 
 @hydrate
 class QueuedScheduler(Scheduler[str]):
@@ -226,7 +244,6 @@ class QueuedScheduler(Scheduler[str]):
         """
         while js_refs:
             info: Mapping[str, Any] = self.get_job_state_info(js_refs=js_refs)
-            print(info)
             if not info:
                 break
             js_refs = list(info)
@@ -234,8 +251,26 @@ class QueuedScheduler(Scheduler[str]):
 
     @abstractmethod
     def format_options(
-        self, resources: ElementResources, num_elements: int, is_array: bool, sub_idx: int
+        self,
+        resources: ElementResources,
+        num_elements: int,
+        is_array: bool,
+        sub_idx: int,
+        js_idx: int,
     ) -> str:
         """
         Render options in a way that the scheduler can handle.
         """
+
+    def get_std_out_err_filename(
+        self, js_idx: int, job_ID: str, array_idx: int | None = None
+    ):
+        """File name of combined standard output and error streams.
+
+        Notes
+        -----
+        We use the standard output stream filename format for the combined output and
+        error streams file.
+
+        """
+        return self.get_stdout_filename(js_idx=js_idx, job_ID=job_ID, array_idx=array_idx)

hpcflow/sdk/submission/schedulers/direct.py

@@ -23,6 +23,20 @@ if TYPE_CHECKING:
 DirectRef: TypeAlias = "tuple[int, list[str]]"
 
 
+def _is_process_cmdline_equal(proc: psutil.Process, cmdline: list[str]) -> bool:
+    """Check if the `cmdline` of a psutil `Process` is equal to the specified
+    `cmdline`."""
+    try:
+        if proc.cmdline() == cmdline:
+            return True
+        else:
+            return False
+    except (psutil.NoSuchProcess, psutil.ZombieProcess):
+        # process no longer exists or, on unix, process has completed but still has a
+        # record
+        return False
+
+
 class DirectScheduler(Scheduler[DirectRef]):
     """
     A direct scheduler, that just runs jobs immediately as direct subprocesses.
@@ -95,8 +109,7 @@ class DirectScheduler(Scheduler[DirectRef]):
             except psutil.NoSuchProcess:
                 # process might have completed already
                 continue
-            if proc_i.cmdline() == p_cmdline:
-                # additional check this is the same process that we submitted
+            if _is_process_cmdline_equal(proc_i, p_cmdline):
                 procs.append(proc_i)
         return procs
 
@@ -131,23 +144,18 @@ class DirectScheduler(Scheduler[DirectRef]):
 
     @override
     def get_job_state_info(
-        self,
-        *,
-        js_refs: Sequence[DirectRef] | None = None,
-        num_js_elements: int = 0,
-    ) -> Mapping[str, Mapping[int | None, JobscriptElementState]]:
+        self, *, js_refs: Sequence[DirectRef] | None = None
+    ) -> Mapping[str, JobscriptElementState]:
         """Query the scheduler to get the states of all of this user's jobs, optionally
         filtering by specified job IDs.
 
         Jobs that are not in the scheduler's status output will not appear in the output
         of this method."""
-        info: dict[str, Mapping[int | None, JobscriptElementState]] = {}
+        info: dict[str, JobscriptElementState] = {}
         for p_id, p_cmdline in js_refs or ():
             if self.is_jobscript_active(p_id, p_cmdline):
                 # as far as the "scheduler" is concerned, all elements are running:
-                info[str(p_id)] = {
-                    i: JobscriptElementState.running for i in range(num_js_elements)
-                }
+                info[str(p_id)] = JobscriptElementState.running
 
         return info
 
@@ -156,7 +164,6 @@ class DirectScheduler(Scheduler[DirectRef]):
         self,
         js_refs: list[DirectRef],
         jobscripts: list[Jobscript] | None = None,
-        num_js_elements: int = 0,  # Ignored!
     ):
         """
         Cancel some jobs.
@@ -166,18 +173,13 @@ class DirectScheduler(Scheduler[DirectRef]):
 
         def callback(proc: psutil.Process):
             try:
-                js = js_proc_id[proc.pid]
+                js_proc_id[proc.pid]
             except KeyError:
                 # child process of one of the jobscripts
                 self._app.submission_logger.debug(
                     f"jobscript child process ({proc.pid}) killed"
                 )
                 return
-            assert hasattr(proc, "returncode")
-            print(
-                f"Jobscript {js.index} from submission {js.submission.index} "
-                f"terminated (user-initiated cancel) with exit code {proc.returncode}."
-            )
 
         procs = self.__get_jobscript_processes(js_refs)
         self._app.submission_logger.info(
@@ -185,6 +187,7 @@ class DirectScheduler(Scheduler[DirectRef]):
         )
         js_proc_id = {i.pid: jobscripts[idx] for idx, i in enumerate(procs) if jobscripts}
         self.__kill_processes(procs, timeout=3, on_terminate=callback)
+        print(f"Cancelled {len(procs)} jobscript{'s' if len(procs) > 1 else ''}.")
         self._app.submission_logger.info("jobscripts cancel command executed.")
 
     def is_jobscript_active(self, process_ID: int, process_cmdline: list[str]):
@@ -198,8 +201,19 @@ class DirectScheduler(Scheduler[DirectRef]):
             proc = psutil.Process(process_ID)
         except psutil.NoSuchProcess:
             return False
+        return _is_process_cmdline_equal(proc, process_cmdline)
+
+    def get_std_out_err_filename(self, js_idx: int, **kwargs) -> str:
+        """File name of combined standard output and error streams."""
+        return f"js_{js_idx}_std.log"
+
+    def get_stdout_filename(self, js_idx: int, **kwargs) -> str:
+        """File name of the standard output stream file."""
+        return f"js_{js_idx}_stdout.log"
 
-        return proc.cmdline() == process_cmdline
+    def get_stderr_filename(self, js_idx: int, **kwargs) -> str:
+        """File name of the standard error stream file."""
+        return f"js_{js_idx}_stderr.log"
 
 
 @hydrate

hpcflow/sdk/submission/schedulers/sge.py

@@ -5,7 +5,7 @@ An interface to SGE.
 from __future__ import annotations
 from collections.abc import Sequence
 import re
-from typing import TYPE_CHECKING
+from typing import cast, TYPE_CHECKING
 from typing_extensions import override
 from hpcflow.sdk.typing import hydrate
 from hpcflow.sdk.core.errors import (
@@ -131,7 +131,7 @@ class SGEPosix(QueuedScheduler):
         if resources.SGE_parallel_env is not None:
             # check user-specified `parallel_env` is valid and compatible with
             # `num_cores`:
-            if resources.num_cores and resources.num_cores > 1:
+            if resources.num_cores and resources.num_cores == 1:
                 raise ValueError(
                     f"An SGE parallel environment should not be specified if `num_cores` "
                     f"is 1 (`SGE_parallel_env` was specified as "
@@ -174,16 +174,42 @@ class SGEPosix(QueuedScheduler):
     def __format_array_request(self, num_elements: int) -> str:
         return f"{self.js_cmd} {self.array_switch} 1-{num_elements}"
 
+    def get_stdout_filename(
+        self, js_idx: int, job_ID: str, array_idx: int | None = None
+    ) -> str:
+        """File name of the standard output stream file."""
+        # TODO: untested, might not work!
+        array_idx_str = f".{array_idx}" if array_idx is not None else ""
+        return f"js_{js_idx}.sh.o{job_ID}{array_idx_str}"
+
+    def get_stderr_filename(
+        self, js_idx: int, job_ID: str, array_idx: int | None = None
+    ) -> str:
+        """File name of the standard error stream file."""
+        # TODO: untested, might not work!
+        array_idx_str = f".{array_idx}" if array_idx is not None else ""
+        return f"js_{js_idx}.sh.e{job_ID}{array_idx_str}"
+
     def __format_std_stream_file_option_lines(
-        self, is_array: bool, sub_idx: int
+        self, is_array: bool, sub_idx: int, js_idx: int, combine_std: bool
     ) -> Iterator[str]:
-        # note: we can't modify the file names
-        yield f"{self.js_cmd} -o ./artifacts/submissions/{sub_idx}"
-        yield f"{self.js_cmd} -e ./artifacts/submissions/{sub_idx}"
+        # note: if we modify the file names, there is, I believe, no way to include the
+        # job ID; so we don't modify the file names:
+        base = f"./artifacts/submissions/{sub_idx}/js_std/{js_idx}"
+        yield f"{self.js_cmd} -o {base}"
+        if combine_std:
+            yield f"{self.js_cmd} -j y"  # redirect stderr to stdout
+        else:
+            yield f"{self.js_cmd} -e {base}"
 
     @override
     def format_options(
-        self, resources: ElementResources, num_elements: int, is_array: bool, sub_idx: int
+        self,
+        resources: ElementResources,
+        num_elements: int,
+        is_array: bool,
+        sub_idx: int,
+        js_idx: int,
     ) -> str:
         """
         Format the options to the jobscript command.
@@ -194,7 +220,11 @@ class SGEPosix(QueuedScheduler):
         if is_array:
             opts.append(self.__format_array_request(num_elements))
 
-        opts.extend(self.__format_std_stream_file_option_lines(is_array, sub_idx))
+        opts.extend(
+            self.__format_std_stream_file_option_lines(
+                is_array, sub_idx, js_idx, resources.combine_jobscript_std
+            )
+        )
 
         for opt_k, opt_v in self.options.items():
             if opt_v is None:
@@ -264,9 +294,9 @@ class SGEPosix(QueuedScheduler):
 
     def get_job_statuses(
         self,
-    ) -> Mapping[str, Mapping[int | None, JobscriptElementState]]:
-        """Get information about all of this user's jobscripts that currently listed by
-        the scheduler."""
+    ) -> Mapping[str, JobscriptElementState | Mapping[int, JobscriptElementState]]:
+        """Get information about all of this user's jobscripts that are currently listed
+        by the scheduler."""
         cmd = [*self.show_cmd, "-u", "$USER", "-g", "d"]  # "-g d": separate arrays items
         stdout, stderr = run_cmd(cmd, logger=self._app.submission_logger)
         if stderr:
@@ -277,7 +307,7 @@ class SGEPosix(QueuedScheduler):
         elif not stdout:
             return {}
 
-        info: dict[str, dict[int | None, JobscriptElementState]] = {}
+        info: dict[str, dict[int, JobscriptElementState] | JobscriptElementState] = {}
         lines = stdout.split("\n")
         # assuming a job name with spaces means we can't split on spaces to get
         # anywhere beyond the job name, so get the column index of the state heading
@@ -300,13 +330,19 @@ class SGEPosix(QueuedScheduler):
                 else None
             )
 
-            info.setdefault(base_job_ID, {})[arr_idx] = state
+            if arr_idx is not None:
+                entry = cast(
+                    dict[int, JobscriptElementState], info.setdefault(base_job_ID, {})
+                )
+                entry[arr_idx] = state
+            else:
+                info[base_job_ID] = state
         return info
 
     @override
     def get_job_state_info(
-        self, *, js_refs: Sequence[str] | None = None, num_js_elements: int = 0
-    ) -> Mapping[str, Mapping[int | None, JobscriptElementState]]:
+        self, *, js_refs: Sequence[str] | None = None
+    ) -> Mapping[str, JobscriptElementState | Mapping[int, JobscriptElementState]]:
         """Query the scheduler to get the states of all of this user's jobs, optionally
         filtering by specified job IDs.
 
@@ -324,7 +360,6 @@ class SGEPosix(QueuedScheduler):
         self,
         js_refs: list[str],
         jobscripts: list[Jobscript] | None = None,
-        num_js_elements: int = 0,  # Ignored!
     ):
         """
         Cancel submitted jobs.

hpcflow/sdk/submission/schedulers/slurm.py

@@ -5,7 +5,7 @@ An interface to SLURM.
 from __future__ import annotations
 import subprocess
 import time
-from typing import TYPE_CHECKING
+from typing import cast, TYPE_CHECKING
 from typing_extensions import override
 from hpcflow.sdk.typing import hydrate
 from hpcflow.sdk.core.enums import ParallelMode
@@ -344,17 +344,37 @@ class SlurmPosix(QueuedScheduler):
         max_str = f"%{resources.max_array_items}" if resources.max_array_items else ""
         return f"{self.js_cmd} {self.array_switch} 1-{num_elements}{max_str}"
 
+    def get_stdout_filename(
+        self, js_idx: int, job_ID: str, array_idx: int | None = None
+    ) -> str:
+        """File name of the standard output stream file."""
+        array_idx_str = f".{array_idx}" if array_idx is not None else ""
+        return f"js_{js_idx}.sh_{job_ID}{array_idx_str}.out"
+
+    def get_stderr_filename(
+        self, js_idx: int, job_ID: str, array_idx: int | None = None
+    ) -> str:
+        """File name of the standard error stream file."""
+        array_idx_str = f".{array_idx}" if array_idx is not None else ""
+        return f"js_{js_idx}.sh_{job_ID}{array_idx_str}.err"
+
     def __format_std_stream_file_option_lines(
-        self, is_array: bool, sub_idx: int
+        self, is_array: bool, sub_idx: int, js_idx: int, combine_std: bool
     ) -> Iterator[str]:
         pattern = R"%x_%A.%a" if is_array else R"%x_%j"
-        base = f"./artifacts/submissions/{sub_idx}/{pattern}"
-        yield f"{self.js_cmd} -o {base}.out"
-        yield f"{self.js_cmd} -e {base}.err"
+        base = f"./artifacts/submissions/{sub_idx}/js_std/{js_idx}/{pattern}"
+        yield f"{self.js_cmd} --output {base}.out"
+        if not combine_std:
+            yield f"{self.js_cmd} --error {base}.err"
 
     @override
     def format_options(
-        self, resources: ElementResources, num_elements: int, is_array: bool, sub_idx: int
+        self,
+        resources: ElementResources,
+        num_elements: int,
+        is_array: bool,
+        sub_idx: int,
+        js_idx: int,
     ) -> str:
         """
         Format the options to the scheduler.
@@ -365,7 +385,11 @@ class SlurmPosix(QueuedScheduler):
         if is_array:
             opts.append(self.__format_array_request(num_elements, resources))
 
-        opts.extend(self.__format_std_stream_file_option_lines(is_array, sub_idx))
+        opts.extend(
+            self.__format_std_stream_file_option_lines(
+                is_array, sub_idx, js_idx, resources.combine_jobscript_std
+            )
+        )
 
         for opt_k, opt_v in self.options.items():
             if isinstance(opt_v, list):
@@ -468,9 +492,9 @@ class SlurmPosix(QueuedScheduler):
 
     def __parse_job_states(
         self, stdout: str
-    ) -> dict[str, dict[int | None, JobscriptElementState]]:
+    ) -> dict[str, JobscriptElementState | dict[int, JobscriptElementState]]:
         """Parse output from Slurm `squeue` command with a simple format."""
-        info: dict[str, dict[int | None, JobscriptElementState]] = {}
+        info: dict[str, JobscriptElementState | dict[int, JobscriptElementState]] = {}
         for ln in stdout.split("\n"):
             if not ln:
                 continue
@@ -478,9 +502,14 @@ class SlurmPosix(QueuedScheduler):
             base_job_ID, arr_idx = self._parse_job_IDs(job_id)
             state = self.state_lookup.get(job_state, JobscriptElementState.errored)
 
-            entry = info.setdefault(base_job_ID, {})
-            for arr_idx_i in arr_idx or ():
-                entry[arr_idx_i] = state
+            if arr_idx is not None:
+                entry = cast(
+                    dict[int, JobscriptElementState], info.setdefault(base_job_ID, {})
+                )
+                for arr_idx_i in arr_idx:
+                    entry[arr_idx_i] = state
+            else:
+                info[base_job_ID] = state
 
         return info
 
@@ -490,7 +519,7 @@ class SlurmPosix(QueuedScheduler):
             *self.show_cmd,
             "--noheader",
             "--format",
-            R"%40i %30T",
+            R"%200i %30T",  # job ID (<base_job_id>_<index> for array job) and job state
             "--jobs",
             ",".join(job_IDs),
         ]
@@ -515,8 +544,8 @@ class SlurmPosix(QueuedScheduler):
 
     @override
     def get_job_state_info(
-        self, *, js_refs: Sequence[str] | None = None, num_js_elements: int = 0
-    ) -> Mapping[str, Mapping[int | None, JobscriptElementState]]:
+        self, *, js_refs: Sequence[str] | None = None
+    ) -> Mapping[str, JobscriptElementState | Mapping[int, JobscriptElementState]]:
         """Query the scheduler to get the states of all of this user's jobs, optionally
         filtering by specified job IDs.
 
@@ -555,7 +584,6 @@ class SlurmPosix(QueuedScheduler):
         self,
         js_refs: list[str],
         jobscripts: list[Jobscript] | None = None,
-        num_js_elements: int = 0,  # Ignored!
     ):
         """
         Cancel submitted jobs.

hpcflow/sdk/submission/schedulers/utils.py

@@ -1,6 +1,7 @@
 """
 Helper for running a subprocess.
 """
+
 from __future__ import annotations
 import subprocess
 from typing import TYPE_CHECKING
@@ -10,11 +11,15 @@ if TYPE_CHECKING:
     from logging import Logger
 
 
-def run_cmd(cmd: str | Sequence[str], logger: Logger | None = None) -> tuple[str, str]:
+def run_cmd(
+    cmd: str | Sequence[str], logger: Logger | None = None, **kwargs
+) -> tuple[str, str]:
     """Execute a command and return stdout, stderr as strings."""
     if logger:
         logger.debug(f"running shell command: {cmd}")
-    proc = subprocess.run(args=cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    proc = subprocess.run(
+        args=cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs
+    )
     stdout = proc.stdout.decode()
     stderr = proc.stderr.decode()
     return stdout, stderr

hpcflow/sdk/submission/shells/base.py

@@ -13,6 +13,8 @@ if TYPE_CHECKING:
     from typing import Any, ClassVar
     from ..types import JobscriptHeaderArgs, VersionInfo
 
+from hpcflow.sdk.utils.hashing import get_hash
+
 
 @hydrate
 class Shell(ABC):
@@ -30,28 +32,53 @@ class Shell(ABC):
         Arguments to pass to the shell.
     """
 
+    #: Default for executable name.
+    DEFAULT_EXE: ClassVar[str] = "/bin/bash"
     #: File extension for jobscripts.
     JS_EXT: ClassVar[str]
-    #: Default for executable name.
-    DEFAULT_EXE: ClassVar[str]
+    #: Basic indent.
+    JS_INDENT: ClassVar[str]
     #: Indent for environment setup.
     JS_ENV_SETUP_INDENT: ClassVar[str]
     #: Template for the jobscript shebang line.
     JS_SHEBANG: ClassVar[str]
+    #: Template for the jobscript functions file.
+    JS_FUNCS: ClassVar[str]
     #: Template for the common part of the jobscript header.
     JS_HEADER: ClassVar[str]
     #: Template for the jobscript header when scheduled.
     JS_SCHEDULER_HEADER: ClassVar[str]
     #: Template for the jobscript header when directly executed.
     JS_DIRECT_HEADER: ClassVar[str]
-    #: Template for the jobscript body.
+    #: Template for enabling writing of the app log.
+    JS_RUN_LOG_PATH_ENABLE: ClassVar[str]
+    #: Template for disabling writing of the app log.
+    JS_RUN_LOG_PATH_DISABLE: ClassVar[str]
+    #: Template for the run execution command.
+    JS_RUN_CMD: ClassVar[str]
+    #: Template for the execution command for multiple combined runs.
+    JS_RUN_CMD_COMBINED: ClassVar[str]
+    #: Template for setting up run environment variables and executing the run.
+    JS_RUN: ClassVar[str]
+    #: Template for the action-run processing loop in a jobscript.
+    JS_ACT_MULTI: ClassVar[str]
+    #: Template for the single-action-run execution in a jobscript.
+    JS_ACT_SINGLE: ClassVar[str]
+    #: Template for setting up environment variables and running one or more action-runs.
     JS_MAIN: ClassVar[str]
-    #: Template for the array handling code in a jobscript.
-    JS_ELEMENT_ARRAY: ClassVar[str]
+    #: Template for a jobscript-block header.
+    JS_BLOCK_HEADER: ClassVar[str]
+    #: Template for single-element execution.
+    JS_ELEMENT_SINGLE: ClassVar[str]
     #: Template for the element processing loop in a jobscript.
-    JS_ELEMENT_LOOP: ClassVar[str]
-    #: Basic indent.
-    JS_INDENT: ClassVar[str]
+    JS_ELEMENT_MULTI_LOOP: ClassVar[str]
+    #: Template for the array handling code in a jobscript.
+    JS_ELEMENT_MULTI_ARRAY: ClassVar[str]
+    #: Template for the jobscript block loop in a jobscript.
+    JS_BLOCK_LOOP: ClassVar[str]
+    #: Template for the jobscript footer.
+    JS_FOOTER: ClassVar[str]
+
     __slots__ = ("_executable", "os_args")
 
     def __init__(
@@ -67,6 +94,9 @@ class Shell(ABC):
             return False
         return self._executable == other._executable and self.os_args == other.os_args
 
+    def __hash__(self):
+        return get_hash((self._executable, self.os_args))
+
     @property
     def executable(self) -> list[str]:
         """
@@ -85,6 +115,10 @@ class Shell(ABC):
         """Get the command for submitting a non-scheduled jobscript."""
         return self.executable + [js_path]
 
+    def get_command_file_launch_command(self, cmd_file_path: str) -> list[str]:
+        """Get the command for launching the commands file for a given run."""
+        return self.executable + [cmd_file_path]
+
     @abstractmethod
     def get_version_info(self, exclude_os: bool = False) -> VersionInfo:
         """Get shell and operating system information."""
@@ -141,32 +175,46 @@ class Shell(ABC):
         workflow_app_alias: str,
         param_name: str,
         shell_var_name: str,
-        EAR_ID: int,
         cmd_idx: int,
         stderr: bool,
-    ):
+        app_name: str,
+    ) -> str:
+        """
+        Produce code to save a parameter's value into the workflow persistent store.
         """
-        Format instructions to save a parameter.
+
+    @abstractmethod
+    def format_stream_assignment(self, shell_var_name: str, command: str) -> str:
+        """
+        Format a stream assignment.
         """
 
     @abstractmethod
-    def wrap_in_subshell(self, commands: str, abortable: bool) -> str:
+    def format_env_var_get(self, var: str) -> str:
+        """
+        Format retrieval of a shell environment variable.
         """
-        Format commands to run within a child scope.
 
-        This assumes `commands` ends in a newline.
+    @abstractmethod
+    def format_array(self, lst: list) -> str:
+        """
+        Format construction of a shell array.
         """
 
     @abstractmethod
-    def format_loop_check(
-        self, workflow_app_alias: str, loop_name: str, run_ID: int
-    ) -> str:
+    def format_array_get_item(self, arr_name: str, index: int | str) -> str:
         """
-        Format a loop check.
+        Format retrieval of a shell array item at a specified index.
         """
 
     @abstractmethod
-    def format_stream_assignment(self, shell_var_name: str, command: str) -> str:
+    def format_source_functions_file(self, app_name: str, commands: str) -> str:
         """
-        Format a stream assignment.
+        Format sourcing (i.e. invocation) of the jobscript functions file.
+        """
+
+    @abstractmethod
+    def format_commands_file(self, app_name: str, commands: str) -> str:
+        """
+        Format the commands file.
         """