dv-flow-mgr 1.0.0.14420805951a1__tar.gz → 1.0.0.14458222178a1__tar.gz

dv_flow_mgr-1.0.0.14458222178a1/.env

@@ -0,0 +1 @@
+PYTHONPATH=./src

{dv_flow_mgr-1.0.0.14420805951a1 → dv_flow_mgr-1.0.0.14458222178a1}/.gitignore

@@ -122,7 +122,6 @@ celerybeat.pid
 *.sage.py
 
 # Environments
-.env
 .venv
 env/
 venv/

{dv_flow_mgr-1.0.0.14420805951a1 → dv_flow_mgr-1.0.0.14458222178a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dv-flow-mgr
-Version: 1.0.0.14420805951a1
+Version: 1.0.0.14458222178a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-1.0.0.14420805951a1 → dv_flow_mgr-1.0.0.14458222178a1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dv-flow-mgr"
-version = "1.0.0.14420805951a1"
+version = "1.0.0.14458222178a1"
 dependencies = [
     'jq',
     'pydantic',

{dv_flow_mgr-1.0.0.14420805951a1 → dv_flow_mgr-1.0.0.14458222178a1}/src/dv_flow/mgr/task_graph_builder.py

@@ -235,7 +235,7 @@ class TaskGraphBuilder(object):
             else:
                 raise Exception("Task %s parameters do not include %s" % (task.name, k))
 
-        self._log.debug("<-- mkTaskNode: %s" % task_t)
+        self._log.debug("<-- mkTaskNode: %s (%d needs)" % (task_t, len(ret.needs)))
         return ret
     
     def _findTask(self, pkg, name):
@@ -386,13 +386,14 @@ class TaskGraphBuilder(object):
         node.input.rundir = self.get_rundir()
         self.leave_rundir()
 
-        self._log.debug("--> processing needs (%s)" % task.name)
+        self._log.debug("--> processing needs (%s) (%d)" % (task.name, len(task.needs)))
         for need in task.needs:
             need_n = self._getTaskNode(need.name)
-            self._log.debug("Add need %s" % need_n.name)
             if need_n is None:
                 raise Exception("Failed to find need %s" % need.name)
+            self._log.debug("Add need %s with %d dependencies" % (need_n.name, len(need_n.needs)))
             node.input.needs.append((need_n, False))
+#            node.needs.append((need_n, False))
         self._log.debug("<-- processing needs")
 
         # TODO: handle strategy
@@ -414,10 +415,13 @@ class TaskGraphBuilder(object):
             self._log.debug("Process node %s" % t.name)
 
             referenced = None
-            for tt in task.subtasks:
-                if tt in t.needs:
-                    referenced = tt
-                    break
+            for tt in node.tasks:
+                self._log.debug("  Checking task %s" % tt.name)
+                for tnn,_ in tt.needs:
+                    self._log.debug("    Check against need %s" % tnn.name)
+                    if tn == tnn:
+                        referenced = tnn
+                        break
 
             refs_internal = None
             # Assess how this task is connected to others in the compound node
@@ -439,12 +443,12 @@ class TaskGraphBuilder(object):
                 self._log.debug("Node %s references internal node %s" % (t.name, refs_internal.name))
 
             if referenced is not None:
+                self._log.debug("Node %s has internal needs: %s" % (tn.name, referenced.name))
+            else:
                 # Add this task as a dependency of the output
                 # node (the root one)
                 self._log.debug("Add node %s as a top-level dependency" % tn.name)
                 node.needs.append((tn, False))
-            else:
-                self._log.debug("Node %s has internal needs" % tn.name)
 
         if task.rundir == RundirE.Unique:
             self.leave_rundir()
@@ -452,7 +456,7 @@ class TaskGraphBuilder(object):
         return node
 
     def _gatherNeeds(self, task_t, node):
-        self._log.debug("--> _gatherNeeds %s" % task_t.name)
+        self._log.debug("--> _gatherNeeds %s (%d)" % (task_t.name, len(task_t.needs)))
         if task_t.uses is not None:
             self._gatherNeeds(task_t.uses, node)
 
@@ -461,7 +465,7 @@ class TaskGraphBuilder(object):
             if need_n is None:
                 raise Exception("Failed to find need %s" % need.name)
             node.needs.append((need_n, False))
-        self._log.debug("<-- _gatherNeeds %s" % task_t.name)
+        self._log.debug("<-- _gatherNeeds %s (%d)" % (task_t.name, len(node.needs)))
         
     def error(self, msg, loc=None):
         if loc is not None:

{dv_flow_mgr-1.0.0.14420805951a1 → dv_flow_mgr-1.0.0.14458222178a1}/src/dv_flow/mgr/task_node_compound.py

@@ -21,7 +21,7 @@
 #****************************************************************************
 import dataclasses as dc
 from pydantic import BaseModel
-from .task_def import ConsumesE
+from .task_def import ConsumesE, PassthroughE, PassthroughE, PassthroughE, PassthroughE
 from .task_node import TaskNode
 from .task_node_leaf import TaskNodeLeaf
 from .task_data import TaskDataResult, TaskDataInput, TaskDataOutput
@@ -44,7 +44,9 @@ class TaskNodeCompound(TaskNode):
         self.input = TaskNodeLeaf(
             name=self.name + ".in",
             srcdir=self.srcdir,
-            params=NullParams())
+            params=NullParams(),
+            consumes=ConsumesE.No,
+            passthrough=PassthroughE.All)
         self.input.task = null_run
         self.tasks.append(self.input)
 
@@ -55,7 +57,7 @@ class TaskNodeCompound(TaskNode):
         return self.input
     
     async def do_run(self, 
-                     runner : TaskRunner, 
+                     ctxt : TaskRunner, 
                      rundir, 
                      memento : Any=None) -> TaskDataResult:
         self._log.debug("Compound task %s (%d)" % (self.name, len(self.needs)))
@@ -90,6 +92,9 @@ class TaskNodeCompound(TaskNode):
             changed=changed,
             output=output,
             dep_m={})
+        
+#        if self.save_exec_data:
+#            self._save_exec_data(rundir, ctxt, self.input.input)
 
         return 0
 

{dv_flow_mgr-1.0.0.14420805951a1 → dv_flow_mgr-1.0.0.14458222178a1}/src/dv_flow_mgr.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dv-flow-mgr
-Version: 1.0.0.14420805951a1
+Version: 1.0.0.14458222178a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-1.0.0.14420805951a1 → dv_flow_mgr-1.0.0.14458222178a1}/src/dv_flow_mgr.egg-info/SOURCES.txt

@@ -1,3 +1,4 @@
+.env
 .gitignore
 LICENSE
 README.md

{dv_flow_mgr-1.0.0.14420805951a1 → dv_flow_mgr-1.0.0.14458222178a1}/tests/unit/test_compound_task.py

@@ -275,3 +275,161 @@ package:
     assert output.output[0].src == 'foo.entry.glob_txt'
     assert output.output[0].type == 'std.FileSet'
     assert len(output.output[0].files) == 1
+
+def test_compound_input_auto_bind(tmpdir):
+    flow_dv = """
+package:
+    name: foo
+
+    tasks:
+    - name: TopLevelTask
+      uses: std.CreateFile
+      with:
+        filename: TopLevelTask.txt
+        content: "TopLevelTask.txt"
+
+    - name: entry
+      needs: [TopLevelTask]
+      body:
+      - name: mytask
+        passthrough: all
+"""
+
+    rundir = os.path.join(tmpdir)
+    with open(os.path.join(rundir, "flow.dv"), "w") as fp:
+        fp.write(flow_dv)
+
+    marker_collector = MarkerCollector()
+    pkg_def = PackageLoader(
+        marker_listeners=[marker_collector]).load(
+            os.path.join(rundir, "flow.dv"))
+    assert len(marker_collector.markers) == 0
+
+    builder = TaskGraphBuilder(
+        root_pkg=pkg_def,
+        rundir=os.path.join(rundir, "rundir"))
+    runner = TaskSetRunner(rundir=os.path.join(rundir, "rundir"))
+
+    t1 = builder.mkTaskNode("foo.entry")
+
+    TaskGraphDotWriter().write(
+        t1, 
+        os.path.join(rundir, "graph.dot"))
+
+    output = asyncio.run(runner.run(t1))
+
+    assert runner.status == 0
+    assert len(output.output) == 1
+    assert output.output[0].src == 'foo.TopLevelTask'
+    assert output.output[0].type == 'std.FileSet'
+    assert len(output.output[0].files) == 1
+
+def test_compound_input_auto_bind_consumes_all(tmpdir):
+    flow_dv = """
+package:
+    name: foo
+
+    tasks:
+    - name: TopLevelTask
+      uses: std.CreateFile
+      with:
+        filename: TopLevelTask.txt
+        content: "TopLevelTask.txt"
+
+    - name: entry
+      needs: [TopLevelTask]
+      body:
+      - name: mytask
+        shell: pytask
+        run: |
+          with open(os.path.join(input.rundir, "mytask.txt"), "w") as fp:
+            fp.write("inputs: %d" % len(input.inputs))
+"""
+
+    rundir = os.path.join(tmpdir)
+    with open(os.path.join(rundir, "flow.dv"), "w") as fp:
+        fp.write(flow_dv)
+
+    marker_collector = MarkerCollector()
+    pkg_def = PackageLoader(
+        marker_listeners=[marker_collector]).load(
+            os.path.join(rundir, "flow.dv"))
+    assert len(marker_collector.markers) == 0
+
+    builder = TaskGraphBuilder(
+        root_pkg=pkg_def,
+        rundir=os.path.join(rundir, "rundir"))
+    runner = TaskSetRunner(rundir=os.path.join(rundir, "rundir"))
+
+    t1 = builder.mkTaskNode("foo.entry")
+
+    TaskGraphDotWriter().write(
+        t1, 
+        os.path.join(rundir, "graph.dot"))
+
+    output = asyncio.run(runner.run(t1))
+
+    assert runner.status == 0
+    # No output, since the inputting task consumes all
+    assert len(output.output) == 0
+
+    assert os.path.isfile(os.path.join(rundir, "rundir/foo.entry/foo.entry.mytask/mytask.txt"))
+    content = open(os.path.join(rundir, "rundir/foo.entry/foo.entry.mytask/mytask.txt"), "r").read().strip()
+    assert content == "inputs: 1"
+
+def test_compound_input_auto_bind_chain(tmpdir):
+    flow_dv = """
+package:
+    name: foo
+
+    tasks:
+    - name: TopLevelTask
+      uses: std.CreateFile
+      with:
+        filename: TopLevelTask.txt
+        content: "TopLevelTask.txt"
+
+    - name: entry
+      needs: [TopLevelTask]
+      body:
+      - name: mytask1
+        passthrough: all
+      - name: mytask2
+        passthrough: all
+        needs: [mytask1]
+      - name: mytask3
+        passthrough: all
+        needs: [mytask2]
+      - name: mytask4
+        passthrough: all
+        needs: [mytask3]
+"""
+
+    rundir = os.path.join(tmpdir)
+    with open(os.path.join(rundir, "flow.dv"), "w") as fp:
+        fp.write(flow_dv)
+
+    marker_collector = MarkerCollector()
+    pkg_def = PackageLoader(
+        marker_listeners=[marker_collector]).load(
+            os.path.join(rundir, "flow.dv"))
+    assert len(marker_collector.markers) == 0
+
+    builder = TaskGraphBuilder(
+        root_pkg=pkg_def,
+        rundir=os.path.join(rundir, "rundir"))
+    runner = TaskSetRunner(rundir=os.path.join(rundir, "rundir"))
+
+    t1 = builder.mkTaskNode("foo.entry")
+
+    TaskGraphDotWriter().write(
+        t1, 
+        os.path.join(rundir, "graph.dot"))
+
+    output = asyncio.run(runner.run(t1))
+
+    assert runner.status == 0
+    assert len(output.output) == 1
+    assert output.output[0].src == 'foo.TopLevelTask'
+    assert output.output[0].type == 'std.FileSet'
+    assert len(output.output[0].files) == 1