hpcflow 0.1.9__py3-none-any.whl → 0.2.0a271__py3-none-any.whl

hpcflow/sdk/submission/types.py

@@ -0,0 +1,140 @@
+"""
+Types for the submission subsystem.
+"""
+
+from __future__ import annotations
+from typing import Any, TYPE_CHECKING
+from typing_extensions import NotRequired, TypeAlias, TypedDict
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+    from datetime import datetime
+    from numpy.typing import NDArray
+    from ..core.element import ElementResources
+
+
+class JobScriptDescriptor(TypedDict):
+    """
+    Descriptor for a jobscript.
+    """
+
+    #: Resources required by the jobscript.
+    resources: Any
+    #: Elements handled by the jobscript.
+    elements: dict[int, list[int]]
+    #: Dependencies of the jobscript.
+    dependencies: NotRequired[dict[int, ResolvedJobscriptBlockDependencies]]
+    #: Hash of resources.
+    resource_hash: NotRequired[str]
+
+
+class ResolvedJobscriptBlockDependencies(TypedDict):
+    """
+    The resolution of a jobscript block dependency. This represents the dependency of one
+    jobscript block on another.
+    """
+
+    #: Mapping of jobscript elements.
+    js_element_mapping: dict[int, list[int]]
+    #: Whether this is an array mapping.
+    is_array: NotRequired[bool]
+
+
+class JobScriptCreationArguments(TypedDict):
+    """
+    Arguments to pass to create a :class:`Jobscript`.
+    """
+
+    # TODO: this currently represents a mix of arguments for both jobscripts and jobscript
+    # blocks; need to separate
+
+    #: The task insertion IDs.
+    task_insert_IDs: list[int]
+    #: The actions of the tasks.
+    task_actions: list[tuple[int, int, int]]
+    #: The elements of the tasks.
+    task_elements: dict[int, list[int]]
+    #: Element action run information.
+    EAR_ID: NDArray
+    #: Resources to use.
+    resources: NotRequired[ElementResources]
+    #: Description of what loops are in play.
+    task_loop_idx: list[dict[str, int]]
+    #: Description of dependencies.
+    dependencies: dict[int | tuple[int, int], ResolvedJobscriptBlockDependencies]
+    #: Whether this is an array jobscript.
+    is_array: NotRequired[bool]
+    #: When the jobscript was submitted, if known.
+    submit_time: NotRequired[datetime]
+    #: Where the jobscript was submitted, if known.
+    submit_hostname: NotRequired[str]
+    #: Description of what the jobscript was submitted to, if known.
+    submit_machine: NotRequired[str]
+    #: The command line used to do the commit, if known.
+    submit_cmdline: NotRequired[list[str]]
+    #: The job ID from the scheduler, if known.
+    scheduler_job_ID: NotRequired[str]
+    #: The process ID of the subprocess, if known.
+    process_ID: NotRequired[int]
+    #: Version info about the target system.
+    version_info: NotRequired[dict[str, str | list[str]]]
+    #: The name of the OS.
+    os_name: NotRequired[str]
+    #: The name of the shell.
+    shell_name: NotRequired[str]
+    #: The scheduler used.
+    scheduler_name: NotRequired[str]
+    #: Whether the jobscript is currently running.
+    running: NotRequired[bool]
+    #: Do not supply!
+    resource_hash: NotRequired[str]
+    #: Do not supply!
+    elements: NotRequired[dict[int, list[int]]]
+
+
+class SchedulerRef(TypedDict):
+    """
+    Scheduler reference descriptor.
+    """
+
+    #: Jobscript references.
+    js_refs: list  # Internal type is horrible and variable
+    #: Number of jobscript elements.
+    num_js_elements: int
+
+
+class SubmissionPart(TypedDict):
+    """
+    A part of a submission.
+    """
+
+    #: Timestamp for when this part was submitted.
+    submit_time: datetime
+    #: The jobscripts involved in this submission.
+    jobscripts: list[int]
+
+
+# This needs PEP 728 for a better type, alas
+#: Version data.
+VersionInfo: TypeAlias = "dict[str, str | list[str]]"
+
+
+# TODO: This really doesn't belong here?!
+class JobscriptHeaderArgs(TypedDict):
+    """
+    Keyword arguments to use when creating a job script from a
+    :class:`Jobscript`.
+    """
+
+    #: Application invocation. (Arguments, etc.)
+    app_invoc: str | Sequence[str]
+    #: Workflow application alias.
+    workflow_app_alias: NotRequired[str]
+    #: Environment setup.
+    env_setup: NotRequired[str]
+    #: Application name in CAPS
+    app_caps: NotRequired[str]
+    #: Configuration directory.
+    config_dir: NotRequired[str]
+    #: Configuration key.
+    config_invoc_key: NotRequired[Any]

hpcflow/sdk/typing.py

@@ -0,0 +1,194 @@
+"""
+Common type aliases.
+"""
+
+from __future__ import annotations
+from dataclasses import InitVar
+from typing import Any, ClassVar, Final, TypeVar, cast, TYPE_CHECKING
+from typing_extensions import NotRequired, TypeAlias, TypedDict
+from pathlib import Path
+import re
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+    from datetime import datetime
+    from rich.status import Status
+    from .core.object_list import (
+        CommandFilesList,
+        EnvironmentsList,
+        ParametersList,
+        TaskSchemasList,
+    )
+    from .submission.enums import JobscriptElementState
+    from .submission.submission import Submission
+
+
+#: Type of a value that can be treated as a path.
+PathLike: TypeAlias = "str | Path | None"
+
+
+class ParamSource(TypedDict):
+    """
+    A parameter source descriptor.
+    """
+
+    #: Parameter type name.
+    type: NotRequired[str]
+    #: EAR ID.
+    EAR_ID: NotRequired[int]
+    #: Task insertion ID.
+    task_insert_ID: NotRequired[int]
+    #: Action index.
+    action_idx: NotRequired[int]
+    #: Element index.
+    element_idx: NotRequired[int]
+    #: Element set index.
+    element_set_idx: NotRequired[int]
+    #: Element action run index.
+    run_idx: NotRequired[int]
+    #: Sequence index.
+    sequence_idx: NotRequired[int]
+    #: Task index.
+    task_idx: NotRequired[int]
+    #: Name of method used to create the parameter's value(s).
+    value_class_method: NotRequired[str]
+
+
+class KnownSubmission(TypedDict):
+    """
+    Describes a known submission.
+    """
+
+    #: Local ID.
+    local_id: int
+    #: Workflow global ID.
+    workflow_id: str
+    #: Whether the submission is active.
+    is_active: bool
+    #: Submission index.
+    sub_idx: int
+    #: Submission time.
+    submit_time: str
+    #: Path to submission.
+    path: str
+    #: Start time.
+    start_time: str
+    #: Finish time.
+    end_time: str
+
+
+class KnownSubmissionItem(TypedDict):
+    """
+    Describes a known submission.
+    """
+
+    #: Local ID.
+    local_id: int
+    #: Workflow global ID.
+    workflow_id: str
+    #: Path to the workflow.
+    workflow_path: str
+    #: Time of submission.
+    submit_time: str
+    #: Parsed time of submission.
+    submit_time_obj: NotRequired[datetime | None]
+    #: Time of start.
+    start_time: str
+    #: Parsed time of start.
+    start_time_obj: datetime | None
+    #: Time of finish.
+    end_time: str
+    #: Parsed time of finish.
+    end_time_obj: datetime | None
+    #: Submission index.
+    sub_idx: int
+    #: Jobscripts in submission.
+    jobscripts: list[int]
+    #: Active jobscript state.
+    active_jobscripts: Mapping[int, Mapping[int, Mapping[int, JobscriptElementState]]]
+    #: Whether this is deleted.
+    deleted: bool
+    #: Whether this is unloadable.
+    unloadable: bool
+    #: Expanded submission object.
+    submission: NotRequired[Submission]
+
+
+class TemplateComponents(TypedDict):
+    """
+    Components loaded from templates.
+    """
+
+    #: Parameters loaded from templates.
+    parameters: NotRequired[ParametersList]
+    #: Command files loaded from templates.
+    command_files: NotRequired[CommandFilesList]
+    #: Execution environments loaded from templates.
+    environments: NotRequired[EnvironmentsList]
+    #: Task schemas loaded from templates.
+    task_schemas: NotRequired[TaskSchemasList]
+    #: Scripts discovered by templates.
+    scripts: NotRequired[dict[str, Path]]
+    #: Programs discovered by templates.
+    programs: NotRequired[dict[str, Path]]
+    #: Jinja templates discovered by templates.
+    jinja_templates: NotRequired[dict[str, Path]]
+
+
+class MakeWorkflowCommonArgs(TypedDict):
+    """
+    Common keys used in workflow construction in :py:meth:`BaseApp._make_workflow`.
+    """
+
+    path: str | None
+    name: str | None
+    name_add_timestamp: bool | None
+    name_use_dir: bool | None
+    overwrite: bool
+    store: str
+    ts_fmt: str | None
+    ts_name_fmt: str | None
+    store_kwargs: dict[str, Any] | None
+    variables: dict[str, Any] | None
+    status: Status | None
+
+
+#: Simplification of :class:`TemplateComponents` to allow some types of
+#: internal manipulations.
+BasicTemplateComponents: TypeAlias = "dict[str, list[dict]]"
+
+
+DataIndex: TypeAlias = "dict[str, int | list[int]]"
+"""
+The type of indices to data. These are *normally* dictionaries of integers,
+but can have leaves being lists of integers when dealing with element groups
+(i.e., when a downstream element uses outputs from multiple upstream elements,
+rather than just a single upstream element).
+"""
+
+
+_T = TypeVar("_T")
+
+_CLASS_VAR_RE: Final = re.compile(r"ClassVar\[(.*)\]")
+_INIT_VAR_RE: Final = re.compile(r"InitVar\[(.*)\]")
+
+
+def hydrate(cls: type[_T]) -> type[_T]:
+    """
+    Partially hydrates the annotations on fields in a class, so that a @dataclass
+    annotation can recognise that ClassVar-annotated fields are class variables.
+    """
+    anns: dict[str, Any] = {}
+    for f, a in cls.__annotations__.items():
+        if isinstance(a, str):
+            m = _CLASS_VAR_RE.match(a)
+            if m:
+                anns[f] = cast(Any, ClassVar[m[1]])
+                continue
+            m = _INIT_VAR_RE.match(a)
+            if m:
+                anns[f] = cast(Any, InitVar(cast(type, m[1])))
+                continue
+        anns[f] = a
+    cls.__annotations__ = anns
+    return cls

hpcflow/sdk/utils/arrays.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, overload
+
+if TYPE_CHECKING:
+    from numpy.typing import NDArray
+
+
+@overload
+def get_2D_idx(idx: int, num_cols: int) -> tuple[int, int]: ...
+
+
+@overload
+def get_2D_idx(idx: NDArray, num_cols: int) -> tuple[NDArray, NDArray]: ...
+
+
+def get_2D_idx(idx: int | NDArray, num_cols: int) -> tuple[int | NDArray, int | NDArray]:
+    """Convert a 1D index to a 2D index, assuming items are arranged in a row-major
+    order."""
+    row_idx = idx // num_cols
+    col_idx = idx % num_cols
+    return (row_idx, col_idx)
+
+
+def get_1D_idx(
+    row_idx: int | NDArray, col_idx: int | NDArray, num_cols: int
+) -> int | NDArray:
+    """Convert a 2D (row, col) index into a 1D index, assuming items are arranged in a
+    row-major order."""
+    return row_idx * num_cols + col_idx
+
+
+def split_arr(arr: NDArray, metadata_size: int) -> list[tuple[NDArray, NDArray]]:
+    """Split a 1D integer array into a list of tuples, each containing a metadata array
+    and a data array, where the size of each (metadata + data) sub-array is specified as
+    the integer immediately before each (metadata + data) sub-array.
+
+    Parameters
+    ----------
+    arr
+        One dimensional integer array to split.
+    metadata_size
+        How many elements to include in the metadata array. This can be zero.
+
+    Returns
+    -------
+    sub_arrs
+        List of tuples of integer arrays. The integers that define the sizes of the
+        sub-arrays are excluded.
+
+    Examples
+    --------
+    >>> split_arr(np.array([4, 0, 1, 2, 3, 4, 1, 4, 5, 6]), metadata_size=1)
+    [(array([0]), array([1, 2, 3])), (array([1]), array([4, 5, 6]))]
+
+    """
+    count = 0
+    block_start = 0
+    sub_arrs = []
+    while count < len(arr):
+        size = arr[block_start]
+        start = block_start + 1
+        end = start + size
+        metadata_i = arr[start : start + metadata_size]
+        sub_arr_i = arr[start + metadata_size : end]
+        sub_arrs.append((metadata_i, sub_arr_i))
+        count += size + 1
+        block_start = end
+    return sub_arrs

hpcflow/sdk/utils/deferred_file.py

@@ -0,0 +1,55 @@
+from os import PathLike
+from typing import Literal, Union
+
+
+class DeferredFileWriter:
+    """A class that provides a context manager for deferring writing or appending to a
+    file until a write method is called.
+
+    Attributes
+    ----------
+    filename
+        The file path to open
+    mode
+        The mode to use.
+
+    Examples
+    --------
+    >>> with DeferredFileWrite("new_file.txt", "w") as f:
+    ...     # file is not yet created
+    ...     f.write("contents")
+    ...     # file is now created, but not closed
+    ... # file is now closed
+
+    """
+
+    def __init__(self, filename: Union[str, PathLike], mode: Literal["w", "a"], **kwargs):
+        self.filename = filename
+        self.mode = mode
+        self.file = None
+        self.kwargs = kwargs
+        self._is_open = False
+
+    def _ensure_open(self):
+        if not self._is_open:
+            self.file = open(self.filename, self.mode, **self.kwargs)
+            self._is_open = True
+
+    def write(self, data):
+        self._ensure_open()
+        self.file.write(data)
+
+    def writelines(self, lines):
+        self._ensure_open()
+        self.file.writelines(lines)
+
+    def close(self):
+        if self._is_open:
+            self.file.close()
+            self._is_open = False
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()

hpcflow/sdk/utils/hashing.py

@@ -0,0 +1,16 @@
+def get_hash(obj):
+    """Return a hash from an arbitrarily nested structure dicts, lists, tuples, and
+    sets.
+
+    Note the resulting hash is not necessarily stable across sessions or machines.
+    """
+
+    if isinstance(obj, (set, tuple, list)):
+        return hash(tuple([type(obj)] + [get_hash(i) for i in obj]))
+
+    elif not isinstance(obj, dict):
+        return hash(obj)
+
+    new_obj = {k: get_hash(obj[k]) for k in obj}
+
+    return hash(frozenset(sorted(new_obj.items())))

hpcflow/sdk/utils/patches.py

@@ -0,0 +1,31 @@
+from contextlib import contextmanager
+from pathlib import Path
+import sys
+from typing import Any
+
+
+def resolve_path(path):
+    """On Windows Python 3.8, 3.9, and 3.10, `Pathlib.resolve` does
+    not return an absolute path for non-existant paths, when it should.
+
+    See: https://github.com/python/cpython/issues/82852
+
+    """
+    # TODO: this only seems to be used in a test; remove?
+    return Path.cwd() / Path(path).resolve()  # cwd is ignored if already absolute
+
+
+@contextmanager
+def override_module_attrs(module_name: str, overrides: dict[str, Any]):
+    """Context manager to temporarily override module-level attributes. The module must be
+    imported (i.e. within `sys.modules`)."""
+
+    module = sys.modules[module_name]
+    original_values = {k: getattr(module, k) for k in overrides}
+    try:
+        for k, v in overrides.items():
+            setattr(module, k, v)
+        yield
+    finally:
+        for k, v in original_values.items():
+            setattr(module, k, v)

hpcflow/sdk/utils/strings.py

@@ -0,0 +1,69 @@
+from typing import Iterable
+import re
+
+
+def shorten_list_str(
+    lst: Iterable, items: int = 10, end_num: int = 1, placeholder: str = "..."
+) -> str:
+    """Format a list as a string, including only some maximum number of items.
+
+    Parameters
+    ----------
+    lst:
+        The list to format in a shortened form.
+    items:
+        The total number of items to include in the formatted list.
+    end_num:
+        The number of items to include at the end of the formatted list.
+    placeholder
+        The placeholder to use to replace excess items in the formatted list.
+
+    Examples
+    --------
+    >>> shorten_list_str(list(range(20)), items=5)
+    '[0, 1, 2, 3, ..., 19]'
+
+    """
+    lst = list(lst)
+    if len(lst) <= items + 1:  # (don't replace only one item)
+        lst_short = lst
+    else:
+        start_num = items - end_num
+        lst_short = lst[:start_num] + ["..."] + lst[-end_num:]
+
+    return "[" + ", ".join(f"{i}" for i in lst_short) + "]"
+
+
+def extract_py_from_future_imports(py_str: str) -> tuple[str, set[str]]:
+    """
+    Remove any `from __future__ import <feature>` lines from a string of Python code, and
+    return the modified string, and a list of `<feature>`s that were imported.
+
+    Notes
+    -----
+    This is required when generated a combined-scripts jobscript that concatenates
+    multiple Python scripts into one script. If `__future__` statements are included in
+    these individual scripts, they must be moved to the top of the file [1].
+
+    References
+    ----------
+    [1] https://docs.python.org/3/reference/simple_stmts.html#future-statements
+
+    """
+
+    pattern = r"^from __future__ import (.*)\n"
+    if future_imports := (set(re.findall(pattern, py_str, flags=re.MULTILINE) or ())):
+        future_imports = {
+            j.strip() for i in future_imports for j in i.split(",") if j.strip()
+        }
+        py_str = re.sub(pattern, "", py_str, flags=re.MULTILINE)
+
+    return (py_str, future_imports)
+
+
+def capitalise_first_letter(chars: str) -> str:
+    """
+    Convert the first character of a string to upper case (if that makes sense).
+    The rest of the string is unchanged.
+    """
+    return chars[0].upper() + chars[1:]

hpcflow/tests/api/test_api.py

@@ -0,0 +1,32 @@
+import pytest
+from hpcflow.sdk.core.utils import get_file_context
+from hpcflow.app import app as hf
+
+
+@pytest.mark.integration
+def test_api_make_and_submit_workflow(null_config, tmp_path):
+    with get_file_context("hpcflow.tests.data", "workflow_1.yaml") as file_path:
+        wk = hf.make_and_submit_workflow(
+            file_path,
+            path=tmp_path,
+            status=False,
+            add_to_known=False,
+            wait=True,
+        )
+        p2 = wk.tasks[0].elements[0].outputs.p2
+        assert isinstance(p2, hf.ElementParameter)
+        assert p2.value == "201"
+
+
+@pytest.mark.integration
+def test_api_make_and_submit_demo_workflow(null_config, tmp_path):
+    wk = hf.make_and_submit_demo_workflow(
+        "workflow_1",
+        path=tmp_path,
+        status=False,
+        add_to_known=False,
+        wait=True,
+    )
+    p2 = wk.tasks[0].elements[0].outputs.p2
+    assert isinstance(p2, hf.ElementParameter)
+    assert p2.value == "201"