hpcflow-new2 0.2.0a50__py3-none-any.whl → 0.2.0a52__py3-none-any.whl

hpcflow/sdk/data/template_components/environments.yaml

@@ -1,45 +1 @@
-- name: demo_env_1
-  executables:
-    - label: demo
-      instances:
-        - command: matflow --config-dir="../" demo-software
-          num_cores: 1
-          parallel_mode: null
-    - label: python
-      instances:
-        - command: python
-          num_cores: 1
-          parallel_mode: null
-
 - name: null_env
-
-- name: lammps_parse_env
-  specifiers:
-    lammps_version: 1.2
-  setup: |
-    conda activate lammps_parse_env
-  executables:
-    - label: python
-      instances:
-        - command: python
-          num_cores: 1
-          parallel_mode: null
-
-- name: lammps_env
-  specifiers:
-    lammps_version: 1.3
-    gcc_version: 3.0231.321+1
-  executables:
-    - label: lammps
-      instances:
-        - command: lmp_serial
-          num_cores: 1
-          parallel_mode: null
-
-- name: damask_parse_env
-  executables:
-    - label: python
-      instances:
-        - command: python
-          num_cores: 1
-          parallel_mode: null

hpcflow/sdk/data/template_components/task_schemas.yaml

@@ -1,4 +1,4 @@
-- objective: test_bash_t1
+- objective: test_t1_bash
   inputs:
     - parameter: p1
   outputs:
@@ -12,23 +12,65 @@
         - command: echo "$((<<parameter:p1>> + 100))"
           stdout: <<parameter:p2>>
 
-- objective: demo_task_2
+- objective: test_t1_ps
   inputs:
+    - parameter: p1
+  outputs:
+    - parameter: p2
+  actions:
+    - environments:
+        - scope:
+            type: any
+          environment: null_env
+      commands:
+        - command: Write-Output ((<<parameter:p1>> + 100))
+          stdout: <<parameter:p2>>
+
+- objective: test_t2_bash
+  inputs:
+    - parameter: p2
+  outputs:
     - parameter: p3
-    - parameter: orientations
+  actions:
+    - environments:
+        - scope:
+            type: any
+          environment: null_env
+      commands:
+        - command: echo "$((<<parameter:p2>> + 100))"
+          stdout: <<parameter:p3>>
+
+- objective: test_t2_ps
+  inputs:
+    - parameter: p2
   outputs:
-    - parameter: p4
+    - parameter: p3
   actions:
     - environments:
         - scope:
             type: any
-          environment: demo_env_1
+          environment: null_env
       commands:
-        - command: >
-            echo "hello! p3=<<parameter:p3>> orientations=<<parameter:orientations>>"
-      output_file_parsers:
-        p4:
-          from_files: [t1_outfile_1]
+        - command: Write-Output ((<<parameter:p2>> + 100))
+          stdout: <<parameter:p3>>
+
+# - objective: demo_task_2
+#   inputs:
+#     - parameter: p3
+#     - parameter: orientations
+#   outputs:
+#     - parameter: p4
+#   actions:
+#     - environments:
+#         - scope:
+#             type: any
+#           environment: demo_env_1
+#       commands:
+#         - command: >
+#             echo "hello! p3=<<parameter:p3>> orientations=<<parameter:orientations>>"
+#       output_file_parsers:
+#         p4:
+#           from_files: [t1_outfile_1]
 
 - objective: dummy_task_1
   inputs:

hpcflow/sdk/persistence/__init__.py

@@ -1,43 +1,31 @@
+import copy
 from pathlib import Path
 import random
 import string
-from typing import Type
+import time
+from typing import Type, Union
 
-from hpcflow.sdk.core.errors import WorkflowNotFoundError
+from reretry import retry
 
-from .base import PersistentStore, dropbox_permission_err_retry
-from .json import JSONPersistentStore
-from .zarr import ZarrPersistentStore
+from hpcflow.sdk.persistence.base import PersistentStore
+from hpcflow.sdk.persistence.json import JSONPersistentStore
+from hpcflow.sdk.persistence.zarr import ZarrPersistentStore, ZarrZipPersistentStore
 
-ALL_STORE_FORMATS = ("zarr", "json")
+ALL_STORE_CLS = {
+    "zarr": ZarrPersistentStore,
+    "zip": ZarrZipPersistentStore,
+    "json": JSONPersistentStore,
+    # "json-single": JSONPersistentStore,  # TODO
+}
 DEFAULT_STORE_FORMAT = "zarr"
+ALL_STORE_FORMATS = tuple(ALL_STORE_CLS.keys())
+ALL_CREATE_STORE_FORMATS = tuple(
+    k for k, v in ALL_STORE_CLS.items() if v._features.create
+)
 
 
-def store_cls_from_path(workflow_path: Path) -> Type[PersistentStore]:
-    if ZarrPersistentStore.path_has_store(workflow_path):
-        return ZarrPersistentStore
-    elif JSONPersistentStore.path_has_store(workflow_path):
-        return JSONPersistentStore
-    else:
-        raise WorkflowNotFoundError(
-            f"No workflow of a known store type found at path: {workflow_path!r}."
-        )
-
-
-def store_cls_from_str(store_format: str) -> Type[PersistentStore]:
-    if store_format == "zarr":
-        return ZarrPersistentStore
-    elif store_format == "json":
-        return JSONPersistentStore
-    else:
+def store_cls_from_str_NEW(store_format: str) -> Type[PersistentStore]:
+    try:
+        return ALL_STORE_CLS[store_format]
+    except KeyError:
         raise ValueError(f"Store format {store_format!r} not known.")
-
-
-@dropbox_permission_err_retry
-def temporary_workflow_rename(path):
-    """Rename an existing same-path workflow directory so we can restore it if workflow
-    creation fails"""
-    temp_ext = "".join(random.choices(string.ascii_letters, k=10))
-    replaced_dir = path.with_suffix(f"{path.suffix}.{temp_ext}")
-    path.rename(replaced_dir)
-    return replaced_dir