hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a199__py3-none-any.whl

hpcflow/sdk/submission/types.py

@@ -0,0 +1,140 @@
+"""
+Types for the submission subsystem.
+"""
+
+from __future__ import annotations
+from typing import Any, TYPE_CHECKING
+from typing_extensions import NotRequired, TypeAlias, TypedDict
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+    from datetime import datetime
+    from numpy.typing import NDArray
+    from ..core.element import ElementResources
+
+
+class JobScriptDescriptor(TypedDict):
+    """
+    Descriptor for a jobscript.
+    """
+
+    #: Resources required by the jobscript.
+    resources: Any
+    #: Elements handled by the jobscript.
+    elements: dict[int, list[int]]
+    #: Dependencies of the jobscript.
+    dependencies: NotRequired[dict[int, ResolvedJobscriptBlockDependencies]]
+    #: Hash of resources.
+    resource_hash: NotRequired[str]
+
+
+class ResolvedJobscriptBlockDependencies(TypedDict):
+    """
+    The resolution of a jobscript block dependency. This represents the dependency of one
+    jobscript block on another.
+    """
+
+    #: Mapping of jobscript elements.
+    js_element_mapping: dict[int, list[int]]
+    #: Whether this is an array mapping.
+    is_array: NotRequired[bool]
+
+
+class JobScriptCreationArguments(TypedDict):
+    """
+    Arguments to pass to create a :class:`Jobscript`.
+    """
+
+    # TODO: this currently represents a mix of arguments for both jobscripts and jobscript
+    # blocks; need to separate
+
+    #: The task insertion IDs.
+    task_insert_IDs: list[int]
+    #: The actions of the tasks.
+    task_actions: list[tuple[int, int, int]]
+    #: The elements of the tasks.
+    task_elements: dict[int, list[int]]
+    #: Element action run information.
+    EAR_ID: NDArray
+    #: Resources to use.
+    resources: NotRequired[ElementResources]
+    #: Description of what loops are in play.
+    task_loop_idx: list[dict[str, int]]
+    #: Description of dependencies.
+    dependencies: dict[int | tuple[int, int], ResolvedJobscriptBlockDependencies]
+    #: Whether this is an array jobscript.
+    is_array: NotRequired[bool]
+    #: When the jobscript was submitted, if known.
+    submit_time: NotRequired[datetime]
+    #: Where the jobscript was submitted, if known.
+    submit_hostname: NotRequired[str]
+    #: Description of what the jobscript was submitted to, if known.
+    submit_machine: NotRequired[str]
+    #: The command line used to do the commit, if known.
+    submit_cmdline: NotRequired[list[str]]
+    #: The job ID from the scheduler, if known.
+    scheduler_job_ID: NotRequired[str]
+    #: The process ID of the subprocess, if known.
+    process_ID: NotRequired[int]
+    #: Version info about the target system.
+    version_info: NotRequired[dict[str, str | list[str]]]
+    #: The name of the OS.
+    os_name: NotRequired[str]
+    #: The name of the shell.
+    shell_name: NotRequired[str]
+    #: The scheduler used.
+    scheduler_name: NotRequired[str]
+    #: Whether the jobscript is currently running.
+    running: NotRequired[bool]
+    #: Do not supply!
+    resource_hash: NotRequired[str]
+    #: Do not supply!
+    elements: NotRequired[dict[int, list[int]]]
+
+
+class SchedulerRef(TypedDict):
+    """
+    Scheduler reference descriptor.
+    """
+
+    #: Jobscript references.
+    js_refs: list  # Internal type is horrible and variable
+    #: Number of jobscript elements.
+    num_js_elements: int
+
+
+class SubmissionPart(TypedDict):
+    """
+    A part of a submission.
+    """
+
+    #: Timestamp for when this part was submitted.
+    submit_time: datetime
+    #: The jobscripts involved in this submission.
+    jobscripts: list[int]
+
+
+# This needs PEP 728 for a better type, alas
+#: Version data.
+VersionInfo: TypeAlias = "dict[str, str | list[str]]"
+
+
+# TODO: This really doesn't belong here?!
+class JobscriptHeaderArgs(TypedDict):
+    """
+    Keyword arguments to use when creating a job script from a
+    :class:`Jobscript`.
+    """
+
+    #: Application invocation. (Arguments, etc.)
+    app_invoc: str | Sequence[str]
+    #: Workflow application alias.
+    workflow_app_alias: NotRequired[str]
+    #: Environment setup.
+    env_setup: NotRequired[str]
+    #: Application name in CAPS
+    app_caps: NotRequired[str]
+    #: Configuration directory.
+    config_dir: NotRequired[str]
+    #: Configuration key.
+    config_invoc_key: NotRequired[Any]

hpcflow/sdk/typing.py

@@ -1,18 +1,188 @@
 """
 Common type aliases.
 """
-from typing import Tuple, TypeVar
+
+from __future__ import annotations
+from dataclasses import InitVar
+from typing import Any, ClassVar, Final, TypeVar, cast, TYPE_CHECKING
+from typing_extensions import NotRequired, TypeAlias, TypedDict
 from pathlib import Path
+import re
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+    from datetime import datetime
+    from rich.status import Status
+    from .core.object_list import (
+        CommandFilesList,
+        EnvironmentsList,
+        ParametersList,
+        TaskSchemasList,
+    )
+    from .submission.enums import JobscriptElementState
+    from .submission.submission import Submission
+
 
 #: Type of a value that can be treated as a path.
-PathLike = TypeVar("PathLike", str, Path, None)  # TODO: maybe don't need TypeVar?
-
-#: Type of an element index:
-#: (task_insert_ID, element_idx)
-E_idx_type = Tuple[int, int]
-#: Type of an element iteration index:
-#: (task_insert_ID, element_idx, iteration_idx)
-EI_idx_type = Tuple[int, int, int]
-#: Type of an element action run index:
-#: (task_insert_ID, element_idx, iteration_idx, action_idx, run_idx)
-EAR_idx_type = Tuple[int, int, int, int, int]
+PathLike: TypeAlias = "str | Path | None"
+
+
+class ParamSource(TypedDict):
+    """
+    A parameter source descriptor.
+    """
+
+    #: Parameter type name.
+    type: NotRequired[str]
+    #: EAR ID.
+    EAR_ID: NotRequired[int]
+    #: Task insertion ID.
+    task_insert_ID: NotRequired[int]
+    #: Action index.
+    action_idx: NotRequired[int]
+    #: Element index.
+    element_idx: NotRequired[int]
+    #: Element set index.
+    element_set_idx: NotRequired[int]
+    #: Element action run index.
+    run_idx: NotRequired[int]
+    #: Sequence index.
+    sequence_idx: NotRequired[int]
+    #: Task index.
+    task_idx: NotRequired[int]
+    #: Name of method used to create the parameter's value(s).
+    value_class_method: NotRequired[str]
+
+
+class KnownSubmission(TypedDict):
+    """
+    Describes a known submission.
+    """
+
+    #: Local ID.
+    local_id: int
+    #: Workflow global ID.
+    workflow_id: str
+    #: Whether the submission is active.
+    is_active: bool
+    #: Submission index.
+    sub_idx: int
+    #: Submission time.
+    submit_time: str
+    #: Path to submission.
+    path: str
+    #: Start time.
+    start_time: str
+    #: Finish time.
+    end_time: str
+
+
+class KnownSubmissionItem(TypedDict):
+    """
+    Describes a known submission.
+    """
+
+    #: Local ID.
+    local_id: int
+    #: Workflow global ID.
+    workflow_id: str
+    #: Path to the workflow.
+    workflow_path: str
+    #: Time of submission.
+    submit_time: str
+    #: Parsed time of submission.
+    submit_time_obj: NotRequired[datetime | None]
+    #: Time of start.
+    start_time: str
+    #: Parsed time of start.
+    start_time_obj: datetime | None
+    #: Time of finish.
+    end_time: str
+    #: Parsed time of finish.
+    end_time_obj: datetime | None
+    #: Submission index.
+    sub_idx: int
+    #: Jobscripts in submission.
+    jobscripts: list[int]
+    #: Active jobscript state.
+    active_jobscripts: Mapping[int, Mapping[int, Mapping[int, JobscriptElementState]]]
+    #: Whether this is deleted.
+    deleted: bool
+    #: Whether this is unloadable.
+    unloadable: bool
+    #: Expanded submission object.
+    submission: NotRequired[Submission]
+
+
+class TemplateComponents(TypedDict):
+    """
+    Components loaded from templates.
+    """
+
+    #: Parameters loaded from templates.
+    parameters: NotRequired[ParametersList]
+    #: Command files loaded from templates.
+    command_files: NotRequired[CommandFilesList]
+    #: Execution environments loaded from templates.
+    environments: NotRequired[EnvironmentsList]
+    #: Task schemas loaded from templates.
+    task_schemas: NotRequired[TaskSchemasList]
+    #: Scripts discovered by templates.
+    scripts: NotRequired[dict[str, Path]]
+
+
+class MakeWorkflowCommonArgs(TypedDict):
+    """
+    Common keys used in workflow construction in :py:meth:`BaseApp._make_workflow`.
+    """
+
+    path: str | None
+    name: str | None
+    overwrite: bool
+    store: str
+    ts_fmt: str | None
+    ts_name_fmt: str | None
+    store_kwargs: dict[str, Any] | None
+    variables: dict[str, Any] | None
+    status: Status | None
+
+
+#: Simplification of :class:`TemplateComponents` to allow some types of
+#: internal manipulations.
+BasicTemplateComponents: TypeAlias = "dict[str, list[dict]]"
+
+
+DataIndex: TypeAlias = "dict[str, int | list[int]]"
+"""
+The type of indices to data. These are *normally* dictionaries of integers,
+but can have leaves being lists of integers when dealing with element groups
+(i.e., when a downstream element uses outputs from multiple upstream elements,
+rather than just a single upstream element).
+"""
+
+
+_T = TypeVar("_T")
+
+_CLASS_VAR_RE: Final = re.compile(r"ClassVar\[(.*)\]")
+_INIT_VAR_RE: Final = re.compile(r"InitVar\[(.*)\]")
+
+
+def hydrate(cls: type[_T]) -> type[_T]:
+    """
+    Partially hydrates the annotations on fields in a class, so that a @dataclass
+    annotation can recognise that ClassVar-annotated fields are class variables.
+    """
+    anns: dict[str, Any] = {}
+    for f, a in cls.__annotations__.items():
+        if isinstance(a, str):
+            m = _CLASS_VAR_RE.match(a)
+            if m:
+                anns[f] = cast(Any, ClassVar[m[1]])
+                continue
+            m = _INIT_VAR_RE.match(a)
+            if m:
+                anns[f] = cast(Any, InitVar(cast(type, m[1])))
+                continue
+        anns[f] = a
+    cls.__annotations__ = anns
+    return cls

hpcflow/sdk/utils/arrays.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, overload
+
+if TYPE_CHECKING:
+    from numpy.typing import NDArray
+
+
+@overload
+def get_2D_idx(idx: int, num_cols: int) -> tuple[int, int]:
+    ...
+
+
+@overload
+def get_2D_idx(idx: NDArray, num_cols: int) -> tuple[NDArray, NDArray]:
+    ...
+
+
+def get_2D_idx(idx: int | NDArray, num_cols: int) -> tuple[int | NDArray, int | NDArray]:
+    """Convert a 1D index to a 2D index, assuming items are arranged in a row-major
+    order."""
+    row_idx = idx // num_cols
+    col_idx = idx % num_cols
+    return (row_idx, col_idx)
+
+
+def get_1D_idx(
+    row_idx: int | NDArray, col_idx: int | NDArray, num_cols: int
+) -> int | NDArray:
+    """Convert a 2D (row, col) index into a 1D index, assuming items are arranged in a
+    row-major order."""
+    return row_idx * num_cols + col_idx
+
+
+def split_arr(arr: NDArray, metadata_size: int) -> list[tuple[NDArray, NDArray]]:
+    """Split a 1D integer array into a list of tuples, each containing a metadata array
+    and a data array, where the size of each (metadata + data) sub-array is specified as
+    the integer immediately before each (metadata + data) sub-array.
+
+    Parameters
+    ----------
+    arr
+        One dimensional integer array to split.
+    metadata_size
+        How many elements to include in the metadata array. This can be zero.
+
+    Returns
+    -------
+    sub_arrs
+        List of tuples of integer arrays. The integers that define the sizes of the
+        sub-arrays are excluded.
+
+    Examples
+    --------
+    >>> split_arr(np.array([4, 0, 1, 2, 3, 4, 1, 4, 5, 6]), metadata_size=1)
+    [(array([0]), array([1, 2, 3])), (array([1]), array([4, 5, 6]))]
+
+    """
+    count = 0
+    block_start = 0
+    sub_arrs = []
+    while count < len(arr):
+        size = arr[block_start]
+        start = block_start + 1
+        end = start + size
+        metadata_i = arr[start : start + metadata_size]
+        sub_arr_i = arr[start + metadata_size : end]
+        sub_arrs.append((metadata_i, sub_arr_i))
+        count += size + 1
+        block_start = end
+    return sub_arrs

hpcflow/sdk/utils/deferred_file.py

@@ -0,0 +1,55 @@
+from os import PathLike
+from typing import Literal, Union
+
+
+class DeferredFileWriter:
+    """A class that provides a context manager for deferring writing or appending to a
+    file until a write method is called.
+
+    Attributes
+    ----------
+    filename
+        The file path to open
+    mode
+        The mode to use.
+
+    Examples
+    --------
+    >>> with DeferredFileWrite("new_file.txt", "w") as f:
+    ...     # file is not yet created
+    ...     f.write("contents")
+    ...     # file is now created, but not closed
+    ... # file is now closed
+
+    """
+
+    def __init__(self, filename: Union[str, PathLike], mode: Literal["w", "a"], **kwargs):
+        self.filename = filename
+        self.mode = mode
+        self.file = None
+        self.kwargs = kwargs
+        self._is_open = False
+
+    def _ensure_open(self):
+        if not self._is_open:
+            self.file = open(self.filename, self.mode, **self.kwargs)
+            self._is_open = True
+
+    def write(self, data):
+        self._ensure_open()
+        self.file.write(data)
+
+    def writelines(self, lines):
+        self._ensure_open()
+        self.file.writelines(lines)
+
+    def close(self):
+        if self._is_open:
+            self.file.close()
+            self._is_open = False
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()

hpcflow/sdk/utils/hashing.py

@@ -0,0 +1,16 @@
+def get_hash(obj):
+    """Return a hash from an arbitrarily nested structure dicts, lists, tuples, and
+    sets.
+
+    Note the resulting hash is not necessarily stable across sessions or machines.
+    """
+
+    if isinstance(obj, (set, tuple, list)):
+        return hash(tuple([type(obj)] + [get_hash(i) for i in obj]))
+
+    elif not isinstance(obj, dict):
+        return hash(obj)
+
+    new_obj = {k: get_hash(obj[k]) for k in obj}
+
+    return hash(frozenset(sorted(new_obj.items())))

hpcflow/sdk/utils/patches.py

@@ -0,0 +1,12 @@
+from pathlib import Path
+
+
+def resolve_path(path):
+    """On Windows Python 3.8, 3.9, and 3.10, `Pathlib.resolve` does
+    not return an absolute path for non-existant paths, when it should.
+
+    See: https://github.com/python/cpython/issues/82852
+
+    """
+    # TODO: this only seems to be used in a test; remove?
+    return Path.cwd() / Path(path).resolve()  # cwd is ignored if already absolute

hpcflow/sdk/utils/strings.py

@@ -0,0 +1,33 @@
+from typing import Iterable
+
+
+def shorten_list_str(
+    lst: Iterable, items: int = 10, end_num: int = 1, placeholder: str = "..."
+) -> str:
+    """Format a list as a string, including only some maximum number of items.
+
+    Parameters
+    ----------
+    lst:
+        The list to format in a shortened form.
+    items:
+        The total number of items to include in the formatted list.
+    end_num:
+        The number of items to include at the end of the formatted list.
+    placeholder
+        The placeholder to use to replace excess items in the formatted list.
+
+    Examples
+    --------
+    >>> shorten_list_str(list(range(20)), items=5)
+    '[0, 1, 2, 3, ..., 19]'
+
+    """
+    lst = list(lst)
+    if len(lst) <= items + 1:  # (don't replace only one item)
+        lst_short = lst
+    else:
+        start_num = items - end_num
+        lst_short = lst[:start_num] + ["..."] + lst[-end_num:]
+
+    return "[" + ", ".join(f"{i}" for i in lst_short) + "]"

hpcflow/tests/api/test_api.py

@@ -0,0 +1,32 @@
+import pytest
+from hpcflow.sdk.core.utils import get_file_context
+from hpcflow.app import app as hf
+
+
+@pytest.mark.integration
+def test_api_make_and_submit_workflow(null_config, tmp_path):
+    with get_file_context("hpcflow.tests.data", "workflow_1.yaml") as file_path:
+        wk = hf.make_and_submit_workflow(
+            file_path,
+            path=tmp_path,
+            status=False,
+            add_to_known=False,
+            wait=True,
+        )
+        p2 = wk.tasks[0].elements[0].outputs.p2
+        assert isinstance(p2, hf.ElementParameter)
+        assert p2.value == "201"
+
+
+@pytest.mark.integration
+def test_api_make_and_submit_demo_workflow(null_config, tmp_path):
+    wk = hf.make_and_submit_demo_workflow(
+        "workflow_1",
+        path=tmp_path,
+        status=False,
+        add_to_known=False,
+        wait=True,
+    )
+    p2 = wk.tasks[0].elements[0].outputs.p2
+    assert isinstance(p2, hf.ElementParameter)
+    assert p2.value == "201"

hpcflow/tests/conftest.py

@@ -1,8 +1,10 @@
+from __future__ import annotations
+from pathlib import Path
 import pytest
 from hpcflow.app import app as hf
 
 
-def pytest_addoption(parser):
+def pytest_addoption(parser: pytest.Parser):
     parser.addoption(
         "--slurm",
         action="store_true",
@@ -27,9 +29,16 @@ def pytest_addoption(parser):
         default=False,
         help="run integration-like workflow submission tests",
     )
+    parser.addoption(
+        "--repeat",
+        action="store",
+        default=1,
+        type=int,
+        help="number of times to repeat each test",
+    )
 
 
-def pytest_configure(config):
+def pytest_configure(config: pytest.Config):
     config.addinivalue_line("markers", "slurm: mark test as slurm to run")
     config.addinivalue_line("markers", "wsl: mark test as wsl to run")
     config.addinivalue_line(
@@ -42,7 +51,7 @@ def pytest_configure(config):
     hf.run_time_info.in_pytest = True
 
 
-def pytest_collection_modifyitems(config, items):
+def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]):
     if config.getoption("--slurm"):
         # --slurm given in cli: only run slurm tests
         for item in items:
@@ -84,19 +93,31 @@ def pytest_collection_modifyitems(config, items):
                 )
 
 
-def pytest_unconfigure(config):
+def pytest_unconfigure(config: pytest.Config):
     hf.run_time_info.in_pytest = False
 
 
 @pytest.fixture
-def null_config(tmp_path):
+def null_config(tmp_path: Path):
     if not hf.is_config_loaded:
         hf.load_config(config_dir=tmp_path)
     hf.run_time_info.in_pytest = True
 
 
 @pytest.fixture
-def new_null_config(tmp_path):
+def new_null_config(tmp_path: Path):
     hf.load_config(config_dir=tmp_path, warn=False)
     hf.load_template_components(warn=False)
     hf.run_time_info.in_pytest = True
+
+
+@pytest.fixture
+def unload_config():
+    hf.unload_config()
+
+
+def pytest_generate_tests(metafunc):
+    repeats_num = int(metafunc.config.getoption("--repeat"))
+    if repeats_num > 1:
+        metafunc.fixturenames.append("tmp_ct")
+        metafunc.parametrize("tmp_ct", range(repeats_num))

hpcflow/tests/data/multi_path_sequences.yaml

@@ -0,0 +1,29 @@
+template_components:
+  task_schemas:
+    - objective: add_p1_components
+      inputs:
+        - parameter: p1
+      outputs:
+        - parameter: p2
+      actions:
+        - rules:
+            - path: resources.os_name
+              condition: { value.equal_to: posix }
+          commands:
+            - command: echo "$((<<parameter:p1.a>> + <<parameter:p1.b>>))"
+              stdout: <<int(parameter:p2)>>
+        - rules:
+            - path: resources.os_name
+              condition: { value.equal_to: nt }
+          commands:
+            - command: Write-Output ((<<parameter:p1.a>> + <<parameter:p1.b>>))
+              stdout: <<int(parameter:p2)>>
+tasks:
+  - schema: add_p1_components
+    inputs:
+      p1: {}
+    multi_path_sequences:
+      - paths: [inputs.p1.a, inputs.p1.b]
+        values:
+          - [101, 102]
+          - [201, 202]