dv-flow-mgr 0.0.1.13798275241a1__tar.gz → 0.0.1.13824531439a1__tar.gz

{dv_flow_mgr-0.0.1.13798275241a1 → dv_flow_mgr-0.0.1.13824531439a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.13798275241a1
+Version: 0.0.1.13824531439a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.13798275241a1 → dv_flow_mgr-0.0.1.13824531439a1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dv-flow-mgr"
-version = "0.0.1.13798275241a1"
+version = "0.0.1.13824531439a1"
 dependencies = [
     'jq',
     'pydantic',

{dv_flow_mgr-0.0.1.13798275241a1 → dv_flow_mgr-0.0.1.13824531439a1}/src/dv_flow/mgr/__init__.py

@@ -4,8 +4,6 @@ from .pkg_rgy import PkgRgy
 from .task_graph_runner import *
 from .task import *
 from .task_data import *
-from .task_graph_runner import TaskGraphRunner
-from .task_graph_runner_local import TaskGraphRunnerLocal
 from .task_graph_builder import TaskGraphBuilder
 from .task_node import task
 from .task_runner import TaskSetRunner

{dv_flow_mgr-0.0.1.13798275241a1 → dv_flow_mgr-0.0.1.13824531439a1}/src/dv_flow/mgr/package_def.py

@@ -347,8 +347,27 @@ class PackageDef(BaseModel):
             if not os.path.isabs(imp_path):
                 cls._log.debug("basedir: %s ; imp_path: %s" % (pkg.basedir, imp_path))
                 imp_path = os.path.join(pkg.basedir, imp_path)
-            if os.path.isdir(imp_path) and os.path.isfile(os.path.join(imp_path, "flow.dv")):
-                imp_path = os.path.join(imp_path, "flow.dv")
+            
+            # Search down the tree looking for a flow.dv file
+            if os.path.isdir(imp_path):
+                path = imp_path
+
+                while path is not None and os.path.isdir(path) and not os.path.isfile(os.path.join(path, "flow.dv")):
+                    # Look one directory down
+                    next_dir = None
+                    for dir in os.listdir(path):
+                        if os.path.isdir(os.path.join(path, dir)):
+                            if next_dir is None:
+                                next_dir = dir
+                            else:
+                                path = None
+                                break
+                    if path is not None:
+                        path = next_dir
+
+                if path is not None and os.path.isfile(os.path.join(path, "flow.dv")):
+                    imp_path = os.path.join(path, "flow.dv")
+
             if not os.path.isfile(imp_path):
                 raise Exception("Import file %s not found" % imp_path)
 

{dv_flow_mgr-0.0.1.13798275241a1 → dv_flow_mgr-0.0.1.13824531439a1}/src/dv_flow/mgr/task_data.py

@@ -48,6 +48,7 @@ class TaskDataInput(BaseModel):
     srcdir : str
     rundir : str
     params : Any
+    inputs : List[Any]
     memento : Any
 
 class TaskDataResult(BaseModel):

{dv_flow_mgr-0.0.1.13798275241a1 → dv_flow_mgr-0.0.1.13824531439a1}/src/dv_flow/mgr/task_node.py

@@ -86,6 +86,13 @@ class TaskNode(object):
         self._log.debug("in_params[2]: %s" % ",".join(p.src for p in in_params))
         eval.setVar("in", in_params)
 
+        # Default inputs is the list of parameter sets that match 'consumes'
+        inputs = []
+        if self.consumes is not None and len(self.consumes):
+            for in_p in in_params:
+                if self._matches(in_p, self.consumes):
+                    inputs.append(in_p)
+
         for name,field in self.params.model_fields.items():
             value = getattr(self.params, name)
             if type(value) == str:
@@ -107,6 +114,7 @@ class TaskNode(object):
             srcdir=self.srcdir,
             rundir=rundir,
             params=self.params,
+            inputs=inputs,
             memento=memento)
 
         self._log.debug("--> Call task method %s" % str(self.task))
@@ -134,26 +142,28 @@ class TaskNode(object):
                 self._log.debug("consumes: %s" % str(self.consumes))
                 for need in self.needs:
                     for out in need.output.output:
-                        consumed = False
-                        for c in self.consumes:
-                            match = False
-                            for k,v in c.items():
-                                self._log.debug("k,v: %s,%s" % (k,v))
-                                if hasattr(out, k):
-                                    self._log.debug("has attribute: %s" % str(getattr(out ,k)))
-                                    if getattr(out, k) == v:
-                                        self._log.debug("match")
-                                        match = True
-                                        break
-                            if match:
-                                consumed = True
-                                break
-                        
-                        if not consumed:
+                        if not self._matches(out, self.consumes):
                             self._log.debug("Propagating type %s from %s" % (
                                 getattr(out, "type", "<unknown>"),
                                 getattr(out, "src", "<unknown>")))
                             output.append(out)
+
+                        # consumed = False
+                        # for c in self.consumes:
+                        #     match = False
+                        #     for k,v in c.items():
+                        #         self._log.debug("k,v: %s,%s" % (k,v))
+                        #         if hasattr(out, k):
+                        #             self._log.debug("has attribute: %s" % str(getattr(out ,k)))
+                        #             if getattr(out, k) == v:
+                        #                 self._log.debug("match")
+                        #                 match = True
+                        #                 break
+                        #     if match:
+                        #         consumed = True
+                        #         break
+                        
+                        # if not consumed:
         else:
             self._log.debug("non-passthrough: %s (only local outputs propagated)" % self.name)
             # empty dependency map
@@ -342,6 +352,24 @@ class TaskNodeCtorWrapper(TaskNodeCtor):
                 else:
                     setattr(obj, key, value)
         return obj
+    
+    def _matches(self, params, consumes):
+        """Determines if a parameter set matches a set of consumed parameters"""
+        consumed = False
+        for c in consumes:
+            # All matching attribute keys must have same value
+            match = False
+            for k,v in c.items():
+                if hasattr(params, k):
+                    if getattr(params, k) != v:
+                        match = True
+                    else:
+                        match = False
+                        break
+            if match:
+                consumed = True
+                break
+        return consumed
 
 def task(paramT,passthrough=False,consumes=None):
     """Decorator to wrap a task method as a TaskNodeCtor"""

{dv_flow_mgr-0.0.1.13798275241a1 → dv_flow_mgr-0.0.1.13824531439a1}/src/dv_flow/mgr/task_runner.py

@@ -195,6 +195,7 @@ class SingleTaskRunner(TaskRunner):
             srcdir=task.srcdir,
             rundir=self.rundir,
             params=task.params,
+            inputs=[],
             memento=memento)
 
         # TODO: notify of task start

{dv_flow_mgr-0.0.1.13798275241a1 → dv_flow_mgr-0.0.1.13824531439a1}/src/dv_flow_mgr.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.13798275241a1
+Version: 0.0.1.13824531439a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.13798275241a1 → dv_flow_mgr-0.0.1.13824531439a1}/tests/system/test_pkg_discovery.py

@@ -5,6 +5,7 @@ import sys
 from dv_flow.mgr import PkgRgy
 
 
+@pytest.mark.skip
 def test_import_specific(tmpdir):
     flow_dv = """
 package:
@@ -56,6 +57,7 @@ package:
     assert output.find("Hello There") != -1
 
 
+@pytest.mark.skip
 def test_import_alias(tmpdir):
     flow_dv = """
 package:
@@ -126,6 +128,7 @@ package:
 
     assert output.find("Hello There (p2.foo)") != -1
 
+@pytest.mark.skip
 def test_interface_impl(tmpdir):
 
     libdir = os.path.join(tmpdir, "lib")

{dv_flow_mgr-0.0.1.13798275241a1 → dv_flow_mgr-0.0.1.13824531439a1}/tests/unit/test_data_merge.py

@@ -9,6 +9,7 @@ from dv_flow.mgr import TaskData, FileSet, TaskData, TaskDataParamKindE
 from pydantic import BaseModel
 from shutil import copytree
 
+@pytest.mark.skip
 def test_empty_in():
 
     in1 = TaskData()
@@ -18,6 +19,7 @@ def test_empty_in():
 
     assert len(out.params) == 0
 
+@pytest.mark.skip
 def test_empty_combine_nonoverlap_in():
 
     in1 = TaskData()
@@ -33,6 +35,7 @@ def test_empty_combine_nonoverlap_in():
     assert "v2" in out.params.keys()
     assert out.getParamVal("v2") == "2"
 
+@pytest.mark.skip
 def test_empty_combine_nonoverlap_in():
 
     in1 = TaskData()
@@ -48,6 +51,7 @@ def test_empty_combine_nonoverlap_in():
     assert "v2" in out.params.keys()
     assert out.getParamVal("v2") == "2"
 
+@pytest.mark.skip
 def test_conflict_1():
 
     in1 = TaskData()
@@ -58,6 +62,7 @@ def test_conflict_1():
     with pytest.raises(Exception):
         out = TaskData.merge([in1, in2])
 
+@pytest.mark.skip
 def test_fileset_merge_1():
     in1 = TaskData(src="in1")
     in1.addFileSet(FileSet(
@@ -75,6 +80,7 @@ def test_fileset_merge_1():
 
     assert len(out.filesets) == 2
 
+@pytest.mark.skip
 def test_fileset_merge_common_dep_1():
     in1 = TaskData(src="in1")
     in1.addFileSet(FileSet(

{dv_flow_mgr-0.0.1.13798275241a1 → dv_flow_mgr-0.0.1.13824531439a1}/tests/unit/test_expr_eval.py

@@ -19,7 +19,7 @@ def test_smoke():
     expr = parser.parse(content)
     result = eval.eval(expr)
 
-    assert result == 10
+    assert result == '10'
 
 
 

{dv_flow_mgr-0.0.1.13798275241a1 → dv_flow_mgr-0.0.1.13824531439a1}/tests/unit/test_fileset.py

@@ -76,7 +76,7 @@ def test_fileset_input_1(tmpdir):
         fs = FileSet(**(fs_l[0]))
 
         return TaskDataResult(
-            output=[input.params.files]
+            output=[fs]
         )
     
     pkg_def = PackageDef.load(os.path.join(tmpdir, "test1", "flow.dv"))

{dv_flow_mgr-0.0.1.13798275241a1 → dv_flow_mgr-0.0.1.13824531439a1}/tests/unit/test_imports.py

@@ -8,7 +8,7 @@ import pytest
 from typing import Any, List, Union
 import yaml
 from dv_flow.mgr import PackageDef, TaskGraphBuilder, TaskSetRunner, task, TaskDataResult
-from dv_flow.mgr import TaskGraphBuilder, TaskGraphRunnerLocal, PackageDef
+from dv_flow.mgr import TaskGraphBuilder, PackageDef
 from dv_flow.mgr.task_runner import TaskSetRunner
 from dv_flow.mgr.task_listener_log import TaskListenerLog
 from dv_flow.mgr.fileset import FileSet

{dv_flow_mgr-0.0.1.13798275241a1 → dv_flow_mgr-0.0.1.13824531439a1}/tests/unit/test_pyclass.py

@@ -2,7 +2,7 @@
 import os
 import asyncio
 import pytest
-from dv_flow.mgr import TaskGraphBuilder, TaskGraphRunnerLocal, PackageDef
+from dv_flow.mgr import TaskGraphBuilder, PackageDef
 from dv_flow.mgr.task_runner import TaskSetRunner
 from dv_flow.mgr.task_listener_log import TaskListenerLog
 #from dv_flow_mgr.tasklib.builtin_pkg import TaskPyClass, TaskPyClassParams
@@ -283,11 +283,11 @@ package:
   - name: Sleep10
     uses: std.Exec
     with:
-      command: sleep 10
+      command: sleep 2
   - name: Sleep5
     uses: std.Exec
     with:
-      command: sleep 5
+      command: sleep 1
   - name: Sleep
     needs: [Sleep10, Sleep5]
 

{dv_flow_mgr-0.0.1.13798275241a1 → dv_flow_mgr-0.0.1.13824531439a1}/tests/unit/test_pytask_smoke.py

@@ -6,7 +6,7 @@ import pydantic.dataclasses as pdc
 from pydantic import BaseModel
 from dv_flow.mgr.param import Param, ParamT
 from dv_flow.mgr.task import Task
-from dv_flow.mgr.task_data import TaskDataResult, TaskMarker, TaskParameterSet
+from dv_flow.mgr.task_data import TaskDataResult, TaskMarker, TaskParameterSet, TaskDataItem
 from dv_flow.mgr.task_node import task as t_decorator
 from dv_flow.mgr.task_node import task
 from dv_flow.mgr.task_runner import SingleTaskRunner, TaskSetRunner
@@ -27,7 +27,7 @@ def test_smoke_1(tmpdir):
             print("Hello from run")
             return TaskDataResult()
 
-    task1 = MyTask(srcdir="srcdir", p1="p1")
+    task1 = MyTask(name="task1", srcdir="srcdir", p1="p1")
     runner = SingleTaskRunner("rundir")
 
     result = asyncio.run(runner.run(task1))
@@ -100,7 +100,7 @@ def test_smoke_4(tmpdir):
     class Params(BaseModel):
         p1 : str = None
 
-    class TaskData(TaskParameterSet):
+    class TaskData(TaskDataItem):
         val : int = -1
 
     called = []
@@ -110,7 +110,7 @@ def test_smoke_4(tmpdir):
             nonlocal called
             called.append(("MyTask1", input.params.p1))
             return TaskDataResult(
-                  output=[TaskData(val=1)]
+                  output=[TaskData(type="foo", src=input.name, id="bar", val=1)]
             )
 
     @task(Params)
@@ -118,7 +118,7 @@ def test_smoke_4(tmpdir):
             nonlocal called
             called.append(("MyTask2", input.params.p1))
             return TaskDataResult(
-                  output=[TaskData(val=2)]
+                  output=[TaskData(type="foo", src=input.name, id="bar", val=2)]
             )
 
     @task(Params)
@@ -138,14 +138,14 @@ def test_smoke_4(tmpdir):
 
     assert len(called) == 3
     assert called[-1][0] == "MyTask3"
-    assert called[-1][1] == "[1, 2]"
+    assert called[-1][1] == "[1, 2]" or called[-1][1] == "[2, 1]"
 
 def test_smoke_5(tmpdir):
 
     class Params(BaseModel):
         p1 : ParamT[List[str]] = pdc.Field(default_factory=list)
 
-    class TaskData(TaskParameterSet):
+    class TaskData(TaskDataItem):
         files : ParamT[List[str]] = pdc.Field(default_factory=list)
 
     called = []
@@ -155,7 +155,7 @@ def test_smoke_5(tmpdir):
             nonlocal called
             called.append(("MyTask1", input.params.p1))
             return TaskDataResult(
-                  output=[TaskData(files=["f1", "f2", "f3"])]
+                  output=[TaskData(src=input.name, type="foo", id="bar", files=["f1", "f2", "f3"])]
             )
 
     @task(Params)
@@ -163,7 +163,7 @@ def test_smoke_5(tmpdir):
             nonlocal called
             called.append(("MyTask2", input.params.p1))
             return TaskDataResult(
-                  output=[TaskData(files=["f4", "f5", "f6"])]
+                  output=[TaskData(src=input.name, type="foo", id="bar", files=["f4", "f5", "f6"])]
             )
 
     @task(Params)
@@ -184,14 +184,15 @@ def test_smoke_5(tmpdir):
 
     assert len(called) == 3
     assert called[-1][0] == "MyTask3"
-    assert called[-1][1] == '["f1", "f2", "f3", "f4", "f5", "f6"]'
+    for it in ["f1", "f2", "f3", "f4", "f5", "f6"]:
+        assert it in called[-1][1]
 
 def test_smoke_6(tmpdir):
 
     class Params(BaseModel):
         p1 : ParamT[List[str]] = pdc.Field(default=["1","2"])
 
-    class TaskData(TaskParameterSet):
+    class TaskData(TaskDataItem):
         files : ParamT[List[str]] = pdc.Field(default_factory=list)
 
     called = []
@@ -201,7 +202,7 @@ def test_smoke_6(tmpdir):
             nonlocal called
             called.append(("MyTask1", input.params.p1))
             return TaskDataResult(
-                  output=[TaskData(files=["f1", "f2", "f3"])]
+                  output=[TaskData(src=input.name, type="foo", id="bar", files=["f1", "f2", "f3"])]
             )
 
     @task(Params)
@@ -209,7 +210,7 @@ def test_smoke_6(tmpdir):
             nonlocal called
             called.append(("MyTask2", input.params.p1))
             return TaskDataResult(
-                  output=[TaskData(files=["f4", "f5", "f6"])]
+                  output=[TaskData(src=input.name, type="foo", id="bar", files=["f4", "f5", "f6"])]
             )
 
     @task(Params)
@@ -230,6 +231,8 @@ def test_smoke_6(tmpdir):
 
     assert len(called) == 3
     assert called[1][0] == "MyTask2"
-    assert called[1][1] == ["1", "2", "4"]
+    for it in ["1", "2", "4"]:
+        assert it in called[1][1]
     assert called[-1][0] == "MyTask3"
-    assert called[-1][1] == '["f1", "f2", "f3", "f4", "f5", "f6"]'
+    for it in ["f1", "f2", "f3", "f4", "f5", "f6"]:
+        assert it in called[-1][1]

{dv_flow_mgr-0.0.1.13798275241a1 → dv_flow_mgr-0.0.1.13824531439a1}/tests/unit/test_smoke.py

@@ -18,10 +18,10 @@ package:
     tasks:
     - name: entry
       with:
-      - name: p1
-        type: int
-      - name: p2
-        type: int
+        p1:
+          type: int
+        p2:
+          type: int
 """
 
     pkg_def = PackageDef.loads(flowdv)

{dv_flow_mgr-0.0.1.13798275241a1 → dv_flow_mgr-0.0.1.13824531439a1}/tests/unit/test_stdlib.py

@@ -1,7 +1,7 @@
 import os
 import asyncio
 import pytest
-from dv_flow.mgr import TaskGraphBuilder, TaskGraphRunnerLocal, PackageDef
+from dv_flow.mgr import TaskGraphBuilder, TaskSetRunner, PackageDef
 from dv_flow.mgr.util import loadProjPkgDef
 
 def test_message(tmpdir, capsys):
@@ -25,7 +25,7 @@ package:
     builder = TaskGraphBuilder(
         root_pkg=pkg_def,
         rundir=rundir)
-    runner = TaskGraphRunnerLocal(rundir=rundir)
+    runner = TaskSetRunner(rundir=rundir)
 
     task = builder.mkTaskGraph("pkg1.foo")