hpcflow-new2 0.2.0a175__py3-none-any.whl → 0.2.0a176__py3-none-any.whl

hpcflow/_version.py

@@ -1 +1 @@
-__version__ = "0.2.0a175"
+__version__ = "0.2.0a176"

hpcflow/sdk/core/loop.py

@@ -438,7 +438,26 @@ class WorkflowLoop:
         cur_loop_idx = self.num_added_iterations[iters_key] - 1
         all_new_data_idx = {}  # keys are (task.insert_ID and element.index)
 
+        # initialise a new `num_added_iterations` key on each child loop:
+        for child in child_loops:
+            iters_key_dct = {
+                **parent_loop_indices,
+                self.name: cur_loop_idx + 1,
+            }
+            added_iters_key_chd = tuple([iters_key_dct.get(j, 0) for j in child.parents])
+            child._initialise_pending_added_iters(added_iters_key_chd)
+
         for task in self.task_objects:
+
+            new_loop_idx = {
+                **parent_loop_indices,
+                self.name: cur_loop_idx + 1,
+                **{
+                    child.name: 0
+                    for child in child_loops
+                    if task.insert_ID in child.task_insert_IDs
+                },
+            }
             for elem_idx in range(task.num_elements):
                 # element needs to take into account changes made in this code
                 element = task.elements[elem_idx]
@@ -587,10 +606,23 @@ class WorkflowLoop:
 
                         elif orig_inp_src.source_type is InputSourceType.TASK:
                             if orig_inp_src.task_ref not in self.task_insert_IDs:
-                                # source task not part of the loop; copy existing data idx:
-                                inp_dat_idx = element.iterations[0].get_data_idx()[
-                                    inp_key
-                                ]
+                                # TODO: what about groups?
+                                # source the data_idx from the iteration with same parent
+                                # loop indices as the new iteration to add:
+                                src_iters = []
+                                for iter_i in element.iterations:
+                                    skip_iter = False
+                                    for p_k, p_v in parent_loop_indices.items():
+                                        if iter_i.loop_idx.get(p_k) != p_v:
+                                            skip_iter = True
+                                            break
+                                    if not skip_iter:
+                                        src_iters.append(iter_i)
+
+                                # could be multiple, but they should all have the same
+                                # data index for this parameter:
+                                src_iter = src_iters[0]
+                                inp_dat_idx = src_iter.get_data_idx()[inp_key]
                             else:
                                 is_group = False
                                 if (
@@ -673,20 +705,6 @@ class WorkflowLoop:
                 )
                 all_new_data_idx[(task.insert_ID, element.index)] = new_data_idx
 
-                new_loop_idx = {
-                    **parent_loop_indices,
-                    self.name: cur_loop_idx + 1,
-                    **{
-                        child.name: 0
-                        for child in child_loops
-                        if task.insert_ID in child.task_insert_IDs
-                    },
-                }
-                # increment num_added_iterations on child loop for this parent loop index:
-                for i in child_loops:
-                    added_iters_key_chd = tuple([new_loop_idx[j] for j in i.parents])
-                    i._initialise_pending_added_iters(added_iters_key_chd)
-
                 iter_ID_i = self.workflow._store.add_element_iteration(
                     element_ID=element.id_,
                     data_idx=new_data_idx,

hpcflow/tests/unit/test_loop.py

@@ -1159,3 +1159,130 @@ def test_multi_nested_loops(null_config, tmp_path):
         (0, {"inner": 0, "middle_1": 2, "middle_2": 1, "outer": 1}, (22,)),
         (0, {"inner": 1, "middle_1": 2, "middle_2": 1, "outer": 1}, (23,)),
     ]
+
+
+def test_nested_loop_input_from_parent_loop_task(null_config, tmp_path):
+    """Test that an input in a nested-loop task is correctly sourced from latest
+    iteration of the parent loop."""
+    wk = make_workflow(
+        schemas_spec=[
+            [{"p1": None}, ("p2", "p3")],
+            [{"p2": None}, ("p4",)],
+            [{"p4": None, "p3": None}, ("p2", "p1")],  # testing p3 source
+        ],
+        path=tmp_path,
+        local_inputs={0: {"p1": 101}},
+        loops=[
+            hf.Loop(name="inner", tasks=[1, 2], num_iterations=3),
+            hf.Loop(name="outer", tasks=[0, 1, 2], num_iterations=2),
+        ],
+    )
+    pathway = wk.get_iteration_task_pathway(ret_data_idx=True)
+    assert len(pathway) == 14
+    p3_out_idx = [i[2][0]["outputs.p3"] for i in pathway if i[0] == 0]
+    p3_inp_idx = [i[2][0]["inputs.p3"] for i in pathway if i[0] == 2]
+    assert len(p3_out_idx) == 2  # 2 outer iterations
+    assert len(p3_inp_idx) == 6  # 2 * 3 iterations
+    assert p3_inp_idx == [p3_out_idx[0]] * 3 + [p3_out_idx[1]] * 3
+
+
+def test_doubly_nested_loop_input_from_parent_loop_task(null_config, tmp_path):
+    """Test that an input in a doubly-nested-loop task is correctly sourced from latest
+    iteration of the parent loop."""
+    # test source of p6 in final task:
+    wk = make_workflow(
+        schemas_spec=[
+            [{"p5": None}, ("p6", "p1")],
+            [{"p1": None}, ("p2", "p3")],
+            [{"p2": None}, ("p4",)],
+            [{"p4": None, "p3": None, "p6": None}, ("p2", "p1", "p5")],
+        ],
+        path=tmp_path,
+        local_inputs={0: {"p5": 101}},
+        loops=[
+            hf.Loop(name="inner", tasks=[2, 3], num_iterations=3),
+            hf.Loop(name="middle", tasks=[1, 2, 3], num_iterations=3),
+            hf.Loop(name="outer", tasks=[0, 1, 2, 3], num_iterations=3),
+        ],
+        overwrite=True,
+    )
+    pathway = wk.get_iteration_task_pathway(ret_data_idx=True)
+    assert len(pathway) == 66
+
+    p6_out_idx = [i[2][0]["outputs.p6"] for i in pathway if i[0] == 0]
+    p6_inp_idx = [i[2][0]["inputs.p6"] for i in pathway if i[0] == 3]
+    assert len(p6_out_idx) == 3  # 2 outer iterations
+    assert len(p6_inp_idx) == 27  # 3 * 3 * 3 iterations
+    assert p6_inp_idx == [p6_out_idx[0]] * 9 + [p6_out_idx[1]] * 9 + [p6_out_idx[2]] * 9
+
+
+def test_loop_non_input_task_input_from_element_group(null_config, tmp_path):
+    """Test correct sourcing of an element group input within a loop, for a task that is
+    not that loop's "input task" with respect to that parameter."""
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput("p1")],
+        outputs=[hf.SchemaOutput("p2"), hf.SchemaOutput("p3")],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        command="echo $((<<parameter:p1>> + 1))",
+                        stdout="<<parameter:p2>>",
+                        stderr="<<parameter:p3>>",
+                    )
+                ]
+            )
+        ],
+    )
+    s2 = hf.TaskSchema(
+        objective="t2",
+        inputs=[hf.SchemaInput("p2", group="my_group")],
+        outputs=[hf.SchemaOutput("p4")],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        command="echo $((<<sum(parameter:p2)>> + 1))",
+                        stdout="<<parameter:p4>>",
+                    )
+                ]
+            )
+        ],
+    )
+    s3 = hf.TaskSchema(
+        objective="t3",
+        inputs=[hf.SchemaInput("p3", group="my_group"), hf.SchemaInput("p4")],
+        outputs=[hf.SchemaOutput("p2")],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        command="echo $((<<sum(parameter:p3)>> + <<parameter:p4>>))",
+                        stdout="<<parameter:p2>>",
+                    )
+                ]
+            )
+        ],
+    )
+    wk = hf.Workflow.from_template_data(
+        template_name="test_loop",
+        path=tmp_path,
+        tasks=[
+            hf.Task(
+                schema=s1,
+                sequences=[hf.ValueSequence("inputs.p1", values=[1, 2, 3])],
+                groups=[hf.ElementGroup("my_group")],
+            ),
+            hf.Task(schema=s2),
+            hf.Task(schema=s3),  # test source of p3 (should be group from t1)
+        ],
+        loops=[hf.Loop(name="inner", tasks=[1, 2], num_iterations=2)],
+    )
+    pathway = wk.get_iteration_task_pathway(ret_data_idx=True)
+    assert len(pathway) == 5
+
+    expected = [i["outputs.p3"] for i in pathway[0][2]]
+    for i in pathway:
+        if i[0] == 2:  # task 3
+            assert i[2][0]["inputs.p3"] == expected

{hpcflow_new2-0.2.0a175.dist-info → hpcflow_new2-0.2.0a176.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hpcflow-new2
-Version: 0.2.0a175
+Version: 0.2.0a176
 Summary: Computational workflow management
 License: MIT
 Author: aplowman

{hpcflow_new2-0.2.0a175.dist-info → hpcflow_new2-0.2.0a176.dist-info}/RECORD

@@ -1,7 +1,7 @@
 hpcflow/__init__.py,sha256=WIETuRHeOp2SqUqHUzpjQ-lk9acbYv-6aWOhZPRdlhs,64
 hpcflow/__pyinstaller/__init__.py,sha256=YOzBlPSck6slucv6lJM9K80JtsJWxXRL00cv6tRj3oc,98
 hpcflow/__pyinstaller/hook-hpcflow.py,sha256=SeMopsPkhCyd9gqIrzwFNRj3ZlkUlUYl-74QYz61mo4,1089
-hpcflow/_version.py,sha256=qS9M42Fmpevc84bybtkqyQeAZhWacKKmZ1vAOYpl2WE,26
+hpcflow/_version.py,sha256=SYxreQlS-GrGdoMoqVR3gpikj6-V4k7yVrXGBKl-Xxg,26
 hpcflow/app.py,sha256=d-kgfnZNlqlCi2H8bK26714brD_u3ibN3FaEZgjF9aA,1332
 hpcflow/cli.py,sha256=G2J3D9v6MnMWOWMMWK6UEKLn_6wnV9lT_qygEBBxg-I,66
 hpcflow/data/demo_data_manifest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -52,7 +52,7 @@ hpcflow/sdk/core/element.py,sha256=hTAR2kxfGSRf4vFgWwrnyuP5z5RnKYOd2X6c6Xd70zo,4
 hpcflow/sdk/core/environment.py,sha256=DGUz1NvliKh6opP0IueGHD69rn_8wFLhDsq6kAmEgM4,4849
 hpcflow/sdk/core/errors.py,sha256=ku4wwsrmxBpJBFflUeZD6vrmAqgC7H02VdlRG4aAGqQ,9292
 hpcflow/sdk/core/json_like.py,sha256=LRZsUd1tn8zXC8fESeiXs7Eko-VdnB8zcXiqixKVcZM,18874
-hpcflow/sdk/core/loop.py,sha256=7WHif9U3KenNqICN5Ibq76WCwEn9xzBo0jnROwv4z-4,30051
+hpcflow/sdk/core/loop.py,sha256=SYlkmjvxifBfykhjgXCbxnCvn37BfXGMRIbyOholrEE,30936
 hpcflow/sdk/core/object_list.py,sha256=HASx7AMniX82bTlROIgIvrjE_DupmwDgxfkfROmI3GA,20168
 hpcflow/sdk/core/parallel.py,sha256=LI-g-qOuOR1oaEUWVT0qW0hmiP9hsJyUP8_IfSTKYYo,95
 hpcflow/sdk/core/parameters.py,sha256=0h1M-fXqOVgruyM0Au7Fo38cUbHgDNEPd1Alb1FULxE,65588
@@ -129,7 +129,7 @@ hpcflow/tests/unit/test_group.py,sha256=J7Gx6BdzD2uoRUnBow7L_OfdfLUZi7sv5AZd1yss
 hpcflow/tests/unit/test_input_source.py,sha256=pYm1V4kBsBIFstKDcrbwUiDLb82rr7ITcUEFJES2dI0,39194
 hpcflow/tests/unit/test_input_value.py,sha256=H6GX1ee7fuq5f-OsfkfiSW8eye_pWwVLUCYUSnj1-Gg,5731
 hpcflow/tests/unit/test_json_like.py,sha256=aGCiGfT-tNiFu3yzW6d_T-oDc5QLwSUgq3pN3jFhyF0,29939
-hpcflow/tests/unit/test_loop.py,sha256=6SPRzqxk0X3aWPiUJJTu6lKODsPGNUOdDgzdN1dFs0k,40438
+hpcflow/tests/unit/test_loop.py,sha256=GNKMGAXP0AaB7fl1TShluyh-PoG7O96X7SGUg6vnK6Y,45149
 hpcflow/tests/unit/test_object_list.py,sha256=nDpbRpCu4XqoYxKMr1_QmDS1s2_6nQOpIEBRHSAXoVg,3049
 hpcflow/tests/unit/test_parameter.py,sha256=39CVido_NJGX-Xj9NDSlazpGzWqMG4zp0GmIKwzO7lI,6659
 hpcflow/tests/unit/test_persistence.py,sha256=DPXFiuY2v8vj0zZ7299nf-KtgYT8LhHI52fq7UPoa6Y,8128
@@ -149,7 +149,7 @@ hpcflow/tests/unit/test_workflow_template.py,sha256=fF7LNveMwCledgncNCRfD9Nd9dL9
 hpcflow/tests/workflows/test_jobscript.py,sha256=9sp1o0g72JZbv2QlOl5v7wCZEFjotxiIKGNUxVaFgaA,724
 hpcflow/tests/workflows/test_workflows.py,sha256=xai6FRtGqG4lStJk6KmsqPUSuvqs9FrsBOxMVALshIs,13400
 hpcflow/viz_demo.ipynb,sha256=1QdnVsk72vihv2L6hOGyk318uEa22ZSgGxQCa7hW2oo,6238
-hpcflow_new2-0.2.0a175.dist-info/METADATA,sha256=43yDTsgpidfr8FPVgUgIDwGX4GJz9qZ9Owf5POXxBQ0,2466
-hpcflow_new2-0.2.0a175.dist-info/WHEEL,sha256=kLuE8m1WYU0Ig0_YEGrXyTtiJvKPpLpDEiChiNyei5Y,88
-hpcflow_new2-0.2.0a175.dist-info/entry_points.txt,sha256=aoGtCnFdfPcXfBdu2zZyMOJoz6fPgdR0elqsgrE-USU,106
-hpcflow_new2-0.2.0a175.dist-info/RECORD,,
+hpcflow_new2-0.2.0a176.dist-info/METADATA,sha256=Xg97HSWIATWbQGGHQ-ORIDzX1dRhvQ2O0mmtxDluQto,2466
+hpcflow_new2-0.2.0a176.dist-info/WHEEL,sha256=kLuE8m1WYU0Ig0_YEGrXyTtiJvKPpLpDEiChiNyei5Y,88
+hpcflow_new2-0.2.0a176.dist-info/entry_points.txt,sha256=aoGtCnFdfPcXfBdu2zZyMOJoz6fPgdR0elqsgrE-USU,106
+hpcflow_new2-0.2.0a176.dist-info/RECORD,,