dv-flow-mgr 0.0.1.13478128278a1__tar.gz → 0.0.1.13525773676a1__tar.gz

{dv_flow_mgr-0.0.1.13478128278a1 → dv_flow_mgr-0.0.1.13525773676a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.13478128278a1
+Version: 0.0.1.13525773676a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.13478128278a1 → dv_flow_mgr-0.0.1.13525773676a1}/docs/Steps.md

@@ -104,6 +104,11 @@ runner.run(simImg)
 
 factory.addTaskType(MyTask.ctor)
 
+# Proper values
+- Show that can pass individual values
+- Need to test append/prepend/path-append/path-prepend
+- Need to process and expand expressions
+- 
 
 
 

dv_flow_mgr-0.0.1.13525773676a1/docs/Tasks.md

@@ -0,0 +1,19 @@
+
+A task graph is composed of task nodes. A task is defined as the combination
+of:
+- behavior
+- parameters data structure
+- rules/process for specifying parameter values
+
+Task behavior must:
+- accept input data composed of:
+  - parameter data structure with values populated
+  - change notification
+  - (optional) meta-data
+- produce result data composed of:
+  - list of parameter sets
+  - change notification
+  - (optional) meta-data
+  - (optional) markers
+  - execution status
+

{dv_flow_mgr-0.0.1.13478128278a1 → dv_flow_mgr-0.0.1.13525773676a1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dv-flow-mgr"
-version = "0.0.1.13478128278a1"
+version = "0.0.1.13525773676a1"
 dependencies = [
     'pydantic',
     'pyyaml',

{dv_flow_mgr-0.0.1.13478128278a1 → dv_flow_mgr-0.0.1.13525773676a1}/src/dv_flow/mgr/expr_eval.py

@@ -9,16 +9,26 @@ class ExprEval(ExprVisitor):
     variables : Dict[str, object] = dc.field(default_factory=dict)
     value : Any = None
 
-    def eval(self, e : Expr):
+    def eval(self, e : Expr) -> str:
         self.value = None
         e.accept(self)
-        return self.value
+
+        val = self._toString(self.value)
+
+        return val
+    
+    def _toString(self, val):
+        if isinstance(val, list):
+            val = '[' + ",".join(self._toString(v) for v in val) + ']'
+        elif hasattr(val, "model_dump_json"):
+            val = val.model_dump_json()
+        return val
 
     def visitExprId(self, e : ExprId):
         if e.id in self.variables:
             self.value = self.variables[e.id]
         else:
-            raise Exception("Variable %s not found" % e.id)
+            raise Exception("Variable '%s' not found" % e.id)
 
     def visitExprString(self, e : ExprString):
         self.value = e.value

dv_flow_mgr-0.0.1.13525773676a1/src/dv_flow/mgr/param_ref_eval.py

@@ -0,0 +1,44 @@
+import dataclasses as dc
+import json
+from .expr_eval import ExprEval
+from .expr_parser import ExprParser
+
+@dc.dataclass
+class ParamRefEval(object):
+
+    parser : ExprParser = ExprParser()
+    expr_eval : ExprEval = ExprEval()
+
+    def eval(self, val : str) -> str:
+        idx = 0
+
+        while True:
+            idx = val.find("${{", idx)
+
+            if idx != -1:
+                eidx = val.find("}}", idx+1)
+
+                if eidx == -1:
+                    raise Exception("unterminated variable ref")
+                
+                ref = val[idx+3:eidx].strip()
+                print("ref: %s" % ref)
+
+                expr_ast = self.parser.parse(ref)
+                print("expr_ast: %s" % str(expr_ast))
+                exp_val = self.expr_eval.eval(expr_ast)
+                print("exp_val: %s" % str(exp_val))
+
+                # Replacing [idx..eidx+2] with len(exp_val)
+                val = val[:idx] + exp_val + val[eidx+2:]
+                idx += len(exp_val)
+
+
+
+            else:
+                break
+
+        return val
+    
+    def setVar(self, name, value):
+        self.expr_eval.variables[name] = value

{dv_flow_mgr-0.0.1.13478128278a1 → dv_flow_mgr-0.0.1.13525773676a1}/src/dv_flow/mgr/task_data.py

@@ -37,6 +37,10 @@ class TaskMarker(BaseModel):
     severity : str
     loc : TaskMarkerLoc = dc.Field(default=None)
 
+class TaskParameterSet(BaseModel):
+    task : str = None # Name of the task that produced this param set
+    seq : int  = -1 # Order in which the param-set must appear
+
 class TaskDataInput(BaseModel):
     changed : bool
     srcdir : str
@@ -46,14 +50,15 @@ class TaskDataInput(BaseModel):
 
 class TaskDataResult(BaseModel):
     changed : bool = dc.Field(default=True)
-    output : List[Any] = dc.Field(default_factory=list)
+    output : List[TaskParameterSet] = dc.Field(default_factory=list)
     memento : Any = dc.Field(default=None)
     markers : List[TaskMarker] = dc.Field(default_factory=list)
     status : int = dc.Field(default=0)
 
 class TaskDataOutput(BaseModel):
-    changed : bool
-    output : List[Any]
+    changed : bool = True
+    output : List[TaskParameterSet] = dc.Field(default_factory=list)
+    dep_m : Dict[str,List[str]] = dc.Field(default_factory=dict)
 
 class TaskDataParamOpE(enum.Enum):
     Set = enum.auto()

dv_flow_mgr-0.0.1.13525773676a1/src/dv_flow/mgr/task_node.py

@@ -0,0 +1,142 @@
+
+import dataclasses as dc
+import pydantic.dataclasses as pdc
+import logging
+from typing import Any, Callable, ClassVar, Dict, List
+from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
+from .task_params_ctor import TaskParamsCtor
+from .param_ref_eval import ParamRefEval
+
+@dc.dataclass
+class TaskNode(object):
+    """Executable view of a task"""
+    # Ctor fields -- must specify on construction
+    name : str
+    srcdir : str
+    # This can be the resolved parameters
+    params : TaskParamsCtor 
+
+    task : Callable[['TaskRunner','TaskDataInput'],'TaskDataResult']
+
+    # Runtime fields -- these get populated during execution
+    changed : bool = False
+    needs : List['TaskNode'] = dc.field(default_factory=list)
+    rundir : str = dc.field(default=None)
+    output : TaskDataOutput = dc.field(default=None)
+
+    _log : ClassVar = logging.getLogger("TaskNode")
+
+    async def do_run(self, 
+                  runner,
+                  rundir,
+                  memento : Any = None) -> 'TaskDataResult':
+        changed = False
+        for dep in self.needs:
+            changed |= dep.changed
+
+        # TODO: Form dep-map from inputs
+        # TODO: Order param sets according to dep-map
+        in_params = []
+        for need in self.needs:
+            in_params.extend(need.output.output)
+
+        # TODO: create an evaluator for substituting param values
+        eval = ParamRefEval()
+
+        eval.setVar("in", in_params)
+
+        for name,field in self.params.model_fields.items():
+            value = getattr(self.params, name)
+            if value.find("${{") != -1:
+                new_val = eval.eval(value)
+                setattr(self.params, name, new_val)
+                print("TODO: expand")
+            print("Field: %s %s" % (name, str(value)))
+            pass
+
+        input = TaskDataInput(
+            changed=changed,
+            srcdir=self.srcdir,
+            rundir=rundir,
+            params=self.params,
+            memento=memento)
+
+        # TODO: notify of task start
+        ret : TaskDataResult = await self.task(self, input)
+        # TODO: notify of task complete
+
+        # TODO: form a dep map from the outgoing param sets
+        dep_m = {}
+
+        # Store the result
+        self.output = TaskDataOutput(
+            changed=ret.changed,
+            dep_m=dep_m,
+            output=ret.output.copy())
+
+        # TODO: 
+
+        return ret
+
+    def __hash__(self):
+        return id(self)
+    
+@staticmethod
+def task(paramT):
+    def wrapper(T):
+        ctor = TaskNodeCtorWrapper(T.__name__, T, paramT)
+        return ctor
+    return wrapper
+
+@dc.dataclass
+class TaskNodeCtor(object):
+    """
+    Factory for a specific task type
+    - Produces a task parameters object, applying value-setting instructions
+    - Produces a TaskNode
+    """
+    name : str
+
+
+    def mkTaskNode(self, srcdir, params, name=None) -> TaskNode:
+        raise NotImplementedError("mkTaskNode in type %s" % str(type(self)))
+
+    def mkTaskParams(self, params : Dict) -> Any:
+        raise NotImplementedError("mkTaskParams in type %s" % str(type(self)))
+
+@dc.dataclass
+class TaskNodeCtorWrapper(TaskNodeCtor):
+    T : Any
+    paramT : Any
+
+    def __call__(self, 
+                 srcdir, 
+                 name=None, 
+                 params=None, 
+                 needs=None,
+                 **kwargs):
+        """Convenience method for direct creation of tasks"""
+        if params is None:
+            params = self.mkTaskParams(kwargs)
+        
+        node = self.mkTaskNode(srcdir, params, name)
+        if needs is not None:
+            node.needs.extend(needs)
+        return node
+
+    def mkTaskNode(self, srcdir, params, name=None) -> TaskNode:
+        node = TaskNode(name, srcdir, params, self.T)
+        return node
+
+    def mkTaskParams(self, params : Dict) -> Any:
+        obj = self.paramT()
+
+        # Apply user-specified params
+        for key,value in params.items():
+            if not hasattr(obj, key):
+                raise Exception("Parameters class %s does not contain field %s" % (
+                    str(type(obj)),
+                    key))
+            else:
+                setattr(obj, key, value)
+        return obj

dv_flow_mgr-0.0.1.13525773676a1/src/dv_flow/mgr/task_runner.py

@@ -0,0 +1,121 @@
+import asyncio
+import dataclasses as dc
+from toposort import toposort
+from typing import Any, Callable, List, Tuple, Union
+from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
+from .task_node import TaskNode
+
+@dc.dataclass
+class TaskRunner(object):
+    rundir : str
+
+    # List of [Listener:Callable[Task],Recurisve:bool]
+    listeners : List[Tuple[Callable['Task','Reason'], bool]] = dc.field(default_factory=list)
+
+    async def do_run(self, 
+                  task : 'Task',
+                  memento : Any = None) -> 'TaskDataResult':
+        return await self.run(task, memento)
+
+    async def run(self, 
+                  task : 'Task',
+                  memento : Any = None) -> 'TaskDataResult':
+        pass
+
+@dc.dataclass
+class TaskSetRunner(TaskRunner):
+    nproc : int = 8
+
+    async def run(self, task : Union[TaskNode,List[TaskNode]]):
+        # First, build a depedency map
+        tasks = task if isinstance(task, list) else [task]
+        dep_m = {}
+        for t in tasks:
+            self._buildDepMap(dep_m, t)
+
+        print("dep_m: %s" % str(dep_m))
+
+        order = list(toposort(dep_m))
+
+        active_task_l = []
+        done_task_s = set()
+        for active_s in order:
+            done = True
+            for t in active_s:
+                while len(active_task_l) >= self.nproc and t not in done_task_s:
+                    # Wait for at least one job to complete
+                    done, pending = await asyncio.wait(at[1] for at in active_task_l)
+                    for d in done:
+                        for i in range(len(active_task_l)):
+                            if active_task_l[i][1] == d:
+                                tt = active_task_l[i][0]
+                                done_task_s.add(tt)
+                                active_task_l.pop(i)
+                                break
+                if t not in done_task_s:
+                    coro = asyncio.Task(t.do_run(
+                        self,
+                        self.rundir, # TODO
+                        None)) # TODO: memento
+                    active_task_l.append((t, coro))
+               
+            # Now, wait for tasks to complete
+            if len(active_task_l):
+                coros = list(at[1] for at in active_task_l)
+                res = await asyncio.gather(*coros)
+
+
+        pass
+
+    def _buildDepMap(self, dep_m, task : TaskNode):
+        if task not in dep_m.keys():
+            dep_m[task] = set(task.needs)
+            for need in task.needs:
+                self._buildDepMap(dep_m, need)
+
+@dc.dataclass
+class SingleTaskRunner(TaskRunner):
+
+    async def run(self, 
+                  task : 'Task',
+                  memento : Any = None) -> 'TaskDataResult':
+        changed = False
+        for dep in task.needs:
+            changed |= dep.changed
+
+        # TODO: create an evaluator for substituting param values
+        eval = None
+
+        for field in dc.fields(task.params):
+            print("Field: %s" % field.name)
+
+        input = TaskDataInput(
+            changed=changed,
+            srcdir=task.srcdir,
+            rundir=self.rundir,
+            params=task.params,
+            memento=memento)
+
+        # TODO: notify of task start
+        ret : TaskDataResult = await task.task(self, input)
+        # TODO: notify of task complete
+
+        # Store the result
+        task.output = TaskDataOutput(
+            changed=ret.changed,
+            output=ret.output.copy())
+
+        # # By definition, none of this have run, since we just ran        
+        # for dep in task.dependents:
+        #     is_sat = True
+        #     for need in dep.needs:
+        #         if need.output is None:
+        #             is_sat = False
+        #             break
+            
+        #     if is_sat:
+        #         # TODO: queue task for evaluation
+        #     pass
+        # TODO: 
+
+        return ret

{dv_flow_mgr-0.0.1.13478128278a1 → dv_flow_mgr-0.0.1.13525773676a1}/src/dv_flow_mgr.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.13478128278a1
+Version: 0.0.1.13525773676a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.13478128278a1 → dv_flow_mgr-0.0.1.13525773676a1}/src/dv_flow_mgr.egg-info/SOURCES.txt

@@ -12,6 +12,7 @@ docs/Roadmap.md
 docs/RundirLayout.md
 docs/Stages.md
 docs/Steps.md
+docs/Tasks.md
 docs/TypesAndDefs.md
 docs/Usecases.md
 docs/conf.py
@@ -30,6 +31,7 @@ src/dv_flow/mgr/package.py
 src/dv_flow/mgr/package_def.py
 src/dv_flow/mgr/package_import_spec.py
 src/dv_flow/mgr/param_def.py
+src/dv_flow/mgr/param_ref_eval.py
 src/dv_flow/mgr/parsetab.py
 src/dv_flow/mgr/pkg_rgy.py
 src/dv_flow/mgr/task.py
@@ -42,6 +44,7 @@ src/dv_flow/mgr/task_graph_runner.py
 src/dv_flow/mgr/task_graph_runner_local.py
 src/dv_flow/mgr/task_impl_data.py
 src/dv_flow/mgr/task_memento.py
+src/dv_flow/mgr/task_node.py
 src/dv_flow/mgr/task_output.py
 src/dv_flow/mgr/task_params_ctor.py
 src/dv_flow/mgr/task_runner.py

dv_flow_mgr-0.0.1.13525773676a1/tests/unit/test_pytask_smoke.py

@@ -0,0 +1,141 @@
+import asyncio
+import pytest
+import dataclasses as dc
+from pydantic import BaseModel
+from dv_flow.mgr.task import Task
+from dv_flow.mgr.task_data import TaskDataResult, TaskMarker, TaskParameterSet
+from dv_flow.mgr.task_node import task as t_decorator
+from dv_flow.mgr.task_runner import SingleTaskRunner, TaskSetRunner
+
+
+def test_smoke_1(tmpdir):
+
+    @dc.dataclass
+    class Params(object):
+        p1 : str = None
+
+    called = False
+
+    @t_decorator(Params)
+    async def MyTask(runner, input):
+            nonlocal called
+            called = True
+            print("Hello from run")
+            return TaskDataResult()
+
+    task = MyTask("srcdir", p1="p1")
+    runner = SingleTaskRunner("rundir")
+
+    result = asyncio.run(runner.run(task))
+
+    assert called
+
+def test_smoke_2(tmpdir):
+
+    @dc.dataclass
+    class Params(object):
+        p1 : str = None
+
+    called = False
+    @Task.ctor(Params)
+    class MyTask(Task):
+        async def run(self, runner, input):
+            nonlocal called
+            called = True
+            print("Hello from run")
+            return TaskDataResult(
+                markers=[TaskMarker(msg="testing", severity="info")]
+            )
+
+    task = MyTask.mkTask("task1", "srcdir", MyTask.mkParams(
+        p1="p1"
+    ))
+    runner = SingleTaskRunner("rundir")
+
+    result = asyncio.run(runner.run(task))
+
+    assert called
+    assert result is not None
+    assert len(result.markers) == 1
+
+def test_smoke_3(tmpdir):
+
+    @dc.dataclass
+    class Params(object):
+        p1 : str = None
+
+    called = []
+
+    @t_decorator(Params)
+    async def MyTask1(runner, input):
+            nonlocal called
+            called.append(("MyTask1", input.params.p1))
+            return TaskDataResult()
+
+    @t_decorator(Params)
+    async def MyTask2(runner, input):
+            nonlocal called
+            called.append(("MyTask2", input.params.p1))
+            return TaskDataResult()
+
+    @t_decorator(Params)
+    async def MyTask3(runner, input):
+            nonlocal called
+            called.append(("MyTask3", input.params.p1))
+            return TaskDataResult()
+
+    task1 = MyTask1("srcdir", p1="1")
+    task2 = MyTask2("srcdir", p1="2")
+    task3 = MyTask3("srcdir", p1="3", needs=[task1, task2])
+    runner = TaskSetRunner("rundir")
+
+    result = asyncio.run(runner.run(task3))
+
+    assert len(called) == 3
+    assert called[-1][0] == "MyTask3"
+    assert called[-1][1] == "3"
+
+def test_smoke_4(tmpdir):
+
+    class Params(BaseModel):
+        p1 : str = None
+
+    class TaskData(TaskParameterSet):
+        val : int = -1
+
+    called = []
+
+    @t_decorator(Params)
+    async def MyTask1(runner, input):
+            nonlocal called
+            called.append(("MyTask1", input.params.p1))
+            return TaskDataResult(
+                  output=[TaskData(val=1)]
+            )
+
+    @t_decorator(Params)
+    async def MyTask2(runner, input):
+            nonlocal called
+            called.append(("MyTask2", input.params.p1))
+            return TaskDataResult(
+                  output=[TaskData(val=2)]
+            )
+
+    @t_decorator(Params)
+    async def MyTask3(runner, input):
+            nonlocal called
+            called.append(("MyTask3", input.params.p1))
+            return TaskDataResult()
+
+    task1 = MyTask1("srcdir", p1="1")
+    task2 = MyTask2("srcdir", p1="2")
+    task3 = MyTask3("srcdir", 
+                    p1="${{ in }}", 
+                    needs=[task1, task2])
+    runner = TaskSetRunner("rundir")
+
+    result = asyncio.run(runner.run(task3))
+
+    assert len(called) == 3
+    assert called[-1][0] == "MyTask3"
+    assert called[-1][1] == "3"

dv_flow_mgr-0.0.1.13478128278a1/src/dv_flow/mgr/task_runner.py

@@ -1,46 +0,0 @@
-import dataclasses as dc
-from .task_data import TaskDataInput, TaskDataResult
-from typing import Any, Callable, List, Tuple
-
-@dc.dataclass
-class TaskRunner(object):
-    rundir : str
-
-    # List of [Listener:Callable[Task],Recurisve:bool]
-    listeners : List[Tuple[Callable['Task','Reason'], bool]] = dc.field(default_factory=list)
-
-    async def do_run(self, 
-                  task : 'Task',
-                  memento : Any = None) -> 'TaskDataResult':
-        return await self.run(task, memento)
-
-    async def run(self, 
-                  task : 'Task',
-                  memento : Any = None) -> 'TaskDataResult':
-        pass
-
-@dc.dataclass
-class SingleTaskRunner(TaskRunner):
-
-    async def run(self, 
-                  task : 'Task',
-                  memento : Any = None) -> 'TaskDataResult':
-        changed = False
-        for dep in task.needs:
-            changed |= dep.changed
-
-        # TODO: create an evaluator for substituting param values
-        eval = None
-
-        params = task.params.mk(eval)
-
-        input = TaskDataInput(
-            changed=changed,
-            srcdir=task.srcdir,
-            rundir=self.rundir,
-            params=params,
-            memento=memento)
-        
-        ret = await task.run(self, input)
-
-        return ret

dv_flow_mgr-0.0.1.13478128278a1/tests/unit/test_pytask_smoke.py

@@ -1,59 +0,0 @@
-import asyncio
-import pytest
-import dataclasses as dc
-from dv_flow.mgr.task import Task
-from dv_flow.mgr.task_data import TaskDataResult, TaskMarker
-from dv_flow.mgr.task_runner import SingleTaskRunner
-
-
-def test_smoke_1(tmpdir):
-
-    @dc.dataclass
-    class Params(object):
-        p1 : str
-
-    called = False
-    @Task.ctor(Params)
-    class MyTask(Task):
-        async def run(self, runner, input):
-            nonlocal called
-            called = True
-            print("Hello from run")
-            pass
-
-    task = MyTask("task1", "srcdir", MyTask.mkParams("p1"))
-    runner = SingleTaskRunner("rundir")
-
-    result = asyncio.run(runner.run(task))
-
-    assert called
-
-def test_smoke_2(tmpdir):
-
-    @dc.dataclass
-    class Params(object):
-        p1 : str = None
-
-    called = False
-    @Task.ctor(Params)
-    class MyTask(Task):
-        async def run(self, runner, input):
-            nonlocal called
-            called = True
-            print("Hello from run")
-            return TaskDataResult(
-                markers=[TaskMarker(msg="testing", severity="info")]
-            )
-
-    task = MyTask.mkTask("task1", "srcdir", MyTask.mkParams(
-        p1="p1"
-    ))
-    runner = SingleTaskRunner("rundir")
-
-    result = asyncio.run(runner.run(task))
-
-    assert called
-    assert result is not None
-    assert len(result.markers) == 1
-
-