hpcflow-new2 0.2.0a190__py3-none-any.whl → 0.2.0a200__py3-none-any.whl

hpcflow/sdk/submission/types.py

@@ -1,6 +1,7 @@
 """
 Types for the submission subsystem.
 """
+
 from __future__ import annotations
 from typing import Any, TYPE_CHECKING
 from typing_extensions import NotRequired, TypeAlias, TypedDict
@@ -22,14 +23,15 @@ class JobScriptDescriptor(TypedDict):
     #: Elements handled by the jobscript.
     elements: dict[int, list[int]]
     #: Dependencies of the jobscript.
-    dependencies: NotRequired[dict[int, ResolvedDependencies]]
+    dependencies: NotRequired[dict[int, ResolvedJobscriptBlockDependencies]]
     #: Hash of resources.
     resource_hash: NotRequired[str]
 
 
-class ResolvedDependencies(TypedDict):
+class ResolvedJobscriptBlockDependencies(TypedDict):
     """
-    The resolution of a dependency.
+    The resolution of a jobscript block dependency. This represents the dependency of one
+    jobscript block on another.
     """
 
     #: Mapping of jobscript elements.
@@ -43,6 +45,9 @@ class JobScriptCreationArguments(TypedDict):
     Arguments to pass to create a :class:`Jobscript`.
     """
 
+    # TODO: this currently represents a mix of arguments for both jobscripts and jobscript
+    # blocks; need to separate
+
     #: The task insertion IDs.
     task_insert_IDs: list[int]
     #: The actions of the tasks.
@@ -52,11 +57,13 @@ class JobScriptCreationArguments(TypedDict):
     #: Element action run information.
     EAR_ID: NDArray
     #: Resources to use.
-    resources: ElementResources
+    resources: NotRequired[ElementResources]
     #: Description of what loops are in play.
     task_loop_idx: list[dict[str, int]]
     #: Description of dependencies.
-    dependencies: dict[int, ResolvedDependencies]
+    dependencies: dict[int | tuple[int, int], ResolvedJobscriptBlockDependencies]
+    #: Whether this is an array jobscript.
+    is_array: NotRequired[bool]
     #: When the jobscript was submitted, if known.
     submit_time: NotRequired[datetime]
     #: Where the jobscript was submitted, if known.
@@ -121,23 +128,13 @@ class JobscriptHeaderArgs(TypedDict):
 
     #: Application invocation. (Arguments, etc.)
     app_invoc: str | Sequence[str]
+    #: Workflow application alias.
+    workflow_app_alias: NotRequired[str]
+    #: Environment setup.
+    env_setup: NotRequired[str]
+    #: Application name in CAPS
+    app_caps: NotRequired[str]
     #: Configuration directory.
     config_dir: NotRequired[str]
     #: Configuration key.
     config_invoc_key: NotRequired[Any]
-    #: Name of EAR file.
-    EAR_file_name: NotRequired[str]
-    #: Name of file containing run directories.
-    element_run_dirs_file_path: NotRequired[str]
-    #: Environment setup.
-    env_setup: NotRequired[str]
-    #: Jobscript index.
-    js_idx: NotRequired[int]
-    #: Log file for the run.
-    run_log_file: NotRequired[str]
-    #: Submission index.
-    sub_idx: NotRequired[int]
-    #: Workflow application alias.
-    workflow_app_alias: NotRequired[str]
-    #: Path to workflow.
-    workflow_path: NotRequired[str]

hpcflow/sdk/typing.py

@@ -1,9 +1,10 @@
 """
 Common type aliases.
 """
+
 from __future__ import annotations
 from dataclasses import InitVar
-from typing import ClassVar, Final, TypeVar, cast, TYPE_CHECKING
+from typing import Any, ClassVar, Final, TypeVar, cast, TYPE_CHECKING
 from typing_extensions import NotRequired, TypeAlias, TypedDict
 from pathlib import Path
 import re
@@ -11,6 +12,7 @@ import re
 if TYPE_CHECKING:
     from collections.abc import Mapping
     from datetime import datetime
+    from rich.status import Status
     from .core.object_list import (
         CommandFilesList,
         EnvironmentsList,
@@ -20,6 +22,7 @@ if TYPE_CHECKING:
     from .submission.enums import JobscriptElementState
     from .submission.submission import Submission
 
+
 #: Type of a value that can be treated as a path.
 PathLike: TypeAlias = "str | Path | None"
 
@@ -102,7 +105,7 @@ class KnownSubmissionItem(TypedDict):
     #: Jobscripts in submission.
     jobscripts: list[int]
     #: Active jobscript state.
-    active_jobscripts: Mapping[int, Mapping[int, JobscriptElementState]]
+    active_jobscripts: Mapping[int, Mapping[int, Mapping[int, JobscriptElementState]]]
     #: Whether this is deleted.
     deleted: bool
     #: Whether this is unloadable.
@@ -128,6 +131,22 @@ class TemplateComponents(TypedDict):
     scripts: NotRequired[dict[str, Path]]
 
 
+class MakeWorkflowCommonArgs(TypedDict):
+    """
+    Common keys used in workflow construction in :py:meth:`BaseApp._make_workflow`.
+    """
+
+    path: str | None
+    name: str | None
+    overwrite: bool
+    store: str
+    ts_fmt: str | None
+    ts_name_fmt: str | None
+    store_kwargs: dict[str, Any] | None
+    variables: dict[str, Any] | None
+    status: Status | None
+
+
 #: Simplification of :class:`TemplateComponents` to allow some types of
 #: internal manipulations.
 BasicTemplateComponents: TypeAlias = "dict[str, list[dict]]"
@@ -153,16 +172,16 @@ def hydrate(cls: type[_T]) -> type[_T]:
     Partially hydrates the annotations on fields in a class, so that a @dataclass
     annotation can recognise that ClassVar-annotated fields are class variables.
     """
-    anns = {}
+    anns: dict[str, Any] = {}
     for f, a in cls.__annotations__.items():
         if isinstance(a, str):
             m = _CLASS_VAR_RE.match(a)
             if m:
-                anns[f] = ClassVar[m[1]]
+                anns[f] = cast(Any, ClassVar[m[1]])
                 continue
             m = _INIT_VAR_RE.match(a)
             if m:
-                anns[f] = InitVar(cast(type, m[1]))
+                anns[f] = cast(Any, InitVar(cast(type, m[1])))
                 continue
         anns[f] = a
     cls.__annotations__ = anns

hpcflow/sdk/utils/arrays.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, overload
+
+if TYPE_CHECKING:
+    from numpy.typing import NDArray
+
+
+@overload
+def get_2D_idx(idx: int, num_cols: int) -> tuple[int, int]:
+    ...
+
+
+@overload
+def get_2D_idx(idx: NDArray, num_cols: int) -> tuple[NDArray, NDArray]:
+    ...
+
+
+def get_2D_idx(idx: int | NDArray, num_cols: int) -> tuple[int | NDArray, int | NDArray]:
+    """Convert a 1D index to a 2D index, assuming items are arranged in a row-major
+    order."""
+    row_idx = idx // num_cols
+    col_idx = idx % num_cols
+    return (row_idx, col_idx)
+
+
+def get_1D_idx(
+    row_idx: int | NDArray, col_idx: int | NDArray, num_cols: int
+) -> int | NDArray:
+    """Convert a 2D (row, col) index into a 1D index, assuming items are arranged in a
+    row-major order."""
+    return row_idx * num_cols + col_idx
+
+
+def split_arr(arr: NDArray, metadata_size: int) -> list[tuple[NDArray, NDArray]]:
+    """Split a 1D integer array into a list of tuples, each containing a metadata array
+    and a data array, where the size of each (metadata + data) sub-array is specified as
+    the integer immediately before each (metadata + data) sub-array.
+
+    Parameters
+    ----------
+    arr
+        One dimensional integer array to split.
+    metadata_size
+        How many elements to include in the metadata array. This can be zero.
+
+    Returns
+    -------
+    sub_arrs
+        List of tuples of integer arrays. The integers that define the sizes of the
+        sub-arrays are excluded.
+
+    Examples
+    --------
+    >>> split_arr(np.array([4, 0, 1, 2, 3, 4, 1, 4, 5, 6]), metadata_size=1)
+    [(array([0]), array([1, 2, 3])), (array([1]), array([4, 5, 6]))]
+
+    """
+    count = 0
+    block_start = 0
+    sub_arrs = []
+    while count < len(arr):
+        size = arr[block_start]
+        start = block_start + 1
+        end = start + size
+        metadata_i = arr[start : start + metadata_size]
+        sub_arr_i = arr[start + metadata_size : end]
+        sub_arrs.append((metadata_i, sub_arr_i))
+        count += size + 1
+        block_start = end
+    return sub_arrs

hpcflow/sdk/utils/deferred_file.py

@@ -0,0 +1,55 @@
+from os import PathLike
+from typing import Literal, Union
+
+
+class DeferredFileWriter:
+    """A class that provides a context manager for deferring writing or appending to a
+    file until a write method is called.
+
+    Attributes
+    ----------
+    filename
+        The file path to open
+    mode
+        The mode to use.
+
+    Examples
+    --------
+    >>> with DeferredFileWrite("new_file.txt", "w") as f:
+    ...     # file is not yet created
+    ...     f.write("contents")
+    ...     # file is now created, but not closed
+    ... # file is now closed
+
+    """
+
+    def __init__(self, filename: Union[str, PathLike], mode: Literal["w", "a"], **kwargs):
+        self.filename = filename
+        self.mode = mode
+        self.file = None
+        self.kwargs = kwargs
+        self._is_open = False
+
+    def _ensure_open(self):
+        if not self._is_open:
+            self.file = open(self.filename, self.mode, **self.kwargs)
+            self._is_open = True
+
+    def write(self, data):
+        self._ensure_open()
+        self.file.write(data)
+
+    def writelines(self, lines):
+        self._ensure_open()
+        self.file.writelines(lines)
+
+    def close(self):
+        if self._is_open:
+            self.file.close()
+            self._is_open = False
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()

hpcflow/sdk/utils/hashing.py

@@ -0,0 +1,16 @@
+def get_hash(obj):
+    """Return a hash from an arbitrarily nested structure dicts, lists, tuples, and
+    sets.
+
+    Note the resulting hash is not necessarily stable across sessions or machines.
+    """
+
+    if isinstance(obj, (set, tuple, list)):
+        return hash(tuple([type(obj)] + [get_hash(i) for i in obj]))
+
+    elif not isinstance(obj, dict):
+        return hash(obj)
+
+    new_obj = {k: get_hash(obj[k]) for k in obj}
+
+    return hash(frozenset(sorted(new_obj.items())))

hpcflow/sdk/utils/patches.py

@@ -0,0 +1,12 @@
+from pathlib import Path
+
+
+def resolve_path(path):
+    """On Windows Python 3.8, 3.9, and 3.10, `Pathlib.resolve` does
+    not return an absolute path for non-existant paths, when it should.
+
+    See: https://github.com/python/cpython/issues/82852
+
+    """
+    # TODO: this only seems to be used in a test; remove?
+    return Path.cwd() / Path(path).resolve()  # cwd is ignored if already absolute

hpcflow/sdk/utils/strings.py

@@ -0,0 +1,33 @@
+from typing import Iterable
+
+
+def shorten_list_str(
+    lst: Iterable, items: int = 10, end_num: int = 1, placeholder: str = "..."
+) -> str:
+    """Format a list as a string, including only some maximum number of items.
+
+    Parameters
+    ----------
+    lst:
+        The list to format in a shortened form.
+    items:
+        The total number of items to include in the formatted list.
+    end_num:
+        The number of items to include at the end of the formatted list.
+    placeholder
+        The placeholder to use to replace excess items in the formatted list.
+
+    Examples
+    --------
+    >>> shorten_list_str(list(range(20)), items=5)
+    '[0, 1, 2, 3, ..., 19]'
+
+    """
+    lst = list(lst)
+    if len(lst) <= items + 1:  # (don't replace only one item)
+        lst_short = lst
+    else:
+        start_num = items - end_num
+        lst_short = lst[:start_num] + ["..."] + lst[-end_num:]
+
+    return "[" + ", ".join(f"{i}" for i in lst_short) + "]"

hpcflow/tests/api/test_api.py

@@ -0,0 +1,32 @@
+import pytest
+from hpcflow.sdk.core.utils import get_file_context
+from hpcflow.app import app as hf
+
+
+@pytest.mark.integration
+def test_api_make_and_submit_workflow(null_config, tmp_path):
+    with get_file_context("hpcflow.tests.data", "workflow_1.yaml") as file_path:
+        wk = hf.make_and_submit_workflow(
+            file_path,
+            path=tmp_path,
+            status=False,
+            add_to_known=False,
+            wait=True,
+        )
+        p2 = wk.tasks[0].elements[0].outputs.p2
+        assert isinstance(p2, hf.ElementParameter)
+        assert p2.value == "201"
+
+
+@pytest.mark.integration
+def test_api_make_and_submit_demo_workflow(null_config, tmp_path):
+    wk = hf.make_and_submit_demo_workflow(
+        "workflow_1",
+        path=tmp_path,
+        status=False,
+        add_to_known=False,
+        wait=True,
+    )
+    p2 = wk.tasks[0].elements[0].outputs.p2
+    assert isinstance(p2, hf.ElementParameter)
+    assert p2.value == "201"

hpcflow/tests/conftest.py

@@ -29,6 +29,13 @@ def pytest_addoption(parser: pytest.Parser):
         default=False,
         help="run integration-like workflow submission tests",
     )
+    parser.addoption(
+        "--repeat",
+        action="store",
+        default=1,
+        type=int,
+        help="number of times to repeat each test",
+    )
 
 
 def pytest_configure(config: pytest.Config):
@@ -102,3 +109,15 @@ def new_null_config(tmp_path: Path):
     hf.load_config(config_dir=tmp_path, warn=False)
     hf.load_template_components(warn=False)
     hf.run_time_info.in_pytest = True
+
+
+@pytest.fixture
+def unload_config():
+    hf.unload_config()
+
+
+def pytest_generate_tests(metafunc):
+    repeats_num = int(metafunc.config.getoption("--repeat"))
+    if repeats_num > 1:
+        metafunc.fixturenames.append("tmp_ct")
+        metafunc.parametrize("tmp_ct", range(repeats_num))

hpcflow/tests/data/benchmark_script_runner.yaml

@@ -0,0 +1,26 @@
+doc: |
+  A workflow for benchmarking the overhead introduced by hpcflow in running a Python
+  script `N` times.
+
+template_components:
+  task_schemas:
+    - objective: run_script
+      inputs:
+        - parameter: p1
+      outputs:
+        - parameter: p2
+      actions:
+        - environments:
+            - scope:
+                type: any
+              environment: python_env
+          script: <<script:main_script_test_direct_in_direct_out.py>>
+          script_exe: python_script
+          script_data_in: direct
+          script_data_out: direct
+
+tasks:
+  - schema: run_script
+    inputs:
+      p1: 101
+    repeats: <<var:N[default=1]>>

hpcflow/tests/data/multi_path_sequences.yaml

@@ -0,0 +1,29 @@
+template_components:
+  task_schemas:
+    - objective: add_p1_components
+      inputs:
+        - parameter: p1
+      outputs:
+        - parameter: p2
+      actions:
+        - rules:
+            - path: resources.os_name
+              condition: { value.equal_to: posix }
+          commands:
+            - command: echo "$((<<parameter:p1.a>> + <<parameter:p1.b>>))"
+              stdout: <<int(parameter:p2)>>
+        - rules:
+            - path: resources.os_name
+              condition: { value.equal_to: nt }
+          commands:
+            - command: Write-Output ((<<parameter:p1.a>> + <<parameter:p1.b>>))
+              stdout: <<int(parameter:p2)>>
+tasks:
+  - schema: add_p1_components
+    inputs:
+      p1: {}
+    multi_path_sequences:
+      - paths: [inputs.p1.a, inputs.p1.b]
+        values:
+          - [101, 102]
+          - [201, 202]

hpcflow/tests/data/workflow_test_run_abort.yaml

@@ -2,42 +2,41 @@ doc: |
   A workflow to sleep a specified number of seconds, and then set an output parameter 
   `is_finished` to the string "true". This is used for testing the run-abort
   functionality.
-parameters:
-  - type: sleep_time_seconds
-  - type: is_finished
-task_schemas:
-  - objective: sleep
-    inputs:
-      - parameter: sleep_time_seconds
-    outputs:
-      - parameter: is_finished
-    actions:
-      - abortable: true
-        rules:
-          - path: resources.os_name
-            condition: { value.equal_to: posix }
-        commands:
-          - command: sleep <<parameter:sleep_time_seconds>>
-      - abortable: true
-        rules:
-          - path: resources.os_name
-            condition: { value.equal_to: nt }
-        commands:
-          - command: Start-Sleep <<parameter:sleep_time_seconds>>
-      - rules:
-          - path: resources.os_name
-            condition: { value.equal_to: posix }
-        commands:
-          - command: echo "true"
-            stdout: <<parameter:is_finished>>
-      - rules:
-          - path: resources.os_name
-            condition: { value.equal_to: nt }
-        commands:
-          - command: Write-Output "true"
-            stdout: <<parameter:is_finished>>
+
+template_components:
+  task_schemas:
+    - objective: sleep
+      inputs:
+        - parameter: sleep_time_seconds
+      outputs:
+        - parameter: is_finished
+      actions:
+        - abortable: true
+          rules:
+            - path: resources.os_name
+              condition: { value.equal_to: posix }
+          commands:
+            - command: sleep <<parameter:sleep_time_seconds>>
+        - abortable: true
+          rules:
+            - path: resources.os_name
+              condition: { value.equal_to: nt }
+          commands:
+            - command: Start-Sleep <<parameter:sleep_time_seconds>>
+        - rules:
+            - path: resources.os_name
+              condition: { value.equal_to: posix }
+          commands:
+            - command: echo "true"
+              stdout: <<parameter:is_finished>>
+        - rules:
+            - path: resources.os_name
+              condition: { value.equal_to: nt }
+          commands:
+            - command: Write-Output "true"
+              stdout: <<parameter:is_finished>>
 
 tasks:
   - schema: sleep
     inputs:
-      sleep_time_seconds: 3600 # 1 hour
+      sleep_time_seconds: 300 # 5 minutes

hpcflow/tests/schedulers/sge/test_sge_submission.py

@@ -0,0 +1,36 @@
+import pytest
+from hpcflow.app import app as hf
+
+
+def test_SGE_process_resources_multi_core_with_parallel_env(null_config):
+
+    scheduler_config = {
+        "parallel_environments": {
+            None: {"num_cores": [1, 1, 1]},  # [start, step, stop]
+            "my_parallel_env": {"num_cores": [2, 1, 32]},
+        }
+    }
+
+    scheduler = hf.SGEPosix()
+    resources = hf.ElementResources(num_cores=2, SGE_parallel_env="my_parallel_env")
+
+    scheduler.process_resources(resources, scheduler_config)
+
+    assert resources.num_cores == 2
+    assert resources.SGE_parallel_env == "my_parallel_env"
+
+
+def test_SGE_process_resources_raises_on_single_core_with_parallel_env(null_config):
+
+    scheduler_config = {
+        "parallel_environments": {
+            None: {"num_cores": [1, 1, 1]},  # [start, step, stop]
+            "my_parallel_env": {"num_cores": [2, 1, 32]},
+        }
+    }
+
+    scheduler = hf.SGEPosix()
+    resources = hf.ElementResources(num_cores=1, SGE_parallel_env="my_parallel_env")
+
+    with pytest.raises(ValueError):
+        scheduler.process_resources(resources, scheduler_config)