dv-flow-mgr 0.0.1.14147547123a1__tar.gz → 0.0.2.14148837747a1__tar.gz

{dv_flow_mgr-0.0.1.14147547123a1 → dv_flow_mgr-0.0.2.14148837747a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dv-flow-mgr
-Version: 0.0.1.14147547123a1
+Version: 0.0.2.14148837747a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

dv_flow_mgr-0.0.2.14148837747a1/docs/ChangeLog.md

@@ -0,0 +1,8 @@
+
+# 0.0.2
+- Add support for compound tasks
+- Add support for simple run-directory control
+- Add control over input/output data-item passthrough
+
+# 0.0.1
+- Basic leaf-task functionality

{dv_flow_mgr-0.0.1.14147547123a1 → dv_flow_mgr-0.0.2.14148837747a1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dv-flow-mgr"
-version = "0.0.1.14147547123a1"
+version = "0.0.2.14148837747a1"
 dependencies = [
     'jq',
     'pydantic',

{dv_flow_mgr-0.0.1.14147547123a1 → dv_flow_mgr-0.0.2.14148837747a1}/src/dv_flow/mgr/cmds/cmd_run.py

@@ -90,7 +90,7 @@ class CmdRun(object):
         for spec in args.tasks:
             if spec.find('.') == -1:
                 spec = pkg.name + "." + spec
-            task = builder.mkTaskGraph(spec)
+            task = builder.mkTaskGraph(spec, rundir=spec)
             tasks.append(task)
 
         asyncio.run(runner.run(tasks))

{dv_flow_mgr-0.0.1.14147547123a1 → dv_flow_mgr-0.0.2.14148837747a1}/src/dv_flow/mgr/package_def.py

@@ -317,7 +317,7 @@ class PackageDef(BaseModel):
         consumes = task.consumes.copy() if isinstance(task.consumes, list) else task.consumes
         needs = [] if task.needs is None else task.needs.copy()
 
-        if self.uses is not None:
+        if base_ctor_t is not None:
             if passthrough is None:
                 passthrough = base_ctor_t.passthrough
             if consumes is None:
@@ -328,7 +328,6 @@ class PackageDef(BaseModel):
         if consumes is None:
             consumes = ConsumesE.All
 
-
         return (passthrough, consumes, needs)
 
     def _getParamT(self, session, task, base_t : BaseModel):

{dv_flow_mgr-0.0.1.14147547123a1 → dv_flow_mgr-0.0.2.14148837747a1}/src/dv_flow/mgr/std/create_file.py

@@ -66,6 +66,10 @@ async def CreateFile(runner, input) -> TaskDataResult:
 
         with open(filename, "w") as fp:
             fp.write(input.params.content)
+        memento.name = input.params.filename
+        memento.hash = hashlib.md5(input.params.content.encode()).hexdigest()
+    else:
+        memento = ex_memento
 
     fs = _FileSet(
                 filetype=input.params.type,

{dv_flow_mgr-0.0.1.14147547123a1 → dv_flow_mgr-0.0.2.14148837747a1}/src/dv_flow/mgr/task_def.py

@@ -50,8 +50,9 @@ class PassthroughE(enum.Enum):
 
 
 class StrategyDef(BaseModel):
-    matrix : Dict[str,List[Any]] = dc.Field(
-        default_factory=dict,
+    chain: Union[bool, None] = dc.Field(default=None)
+    matrix : Union[Dict[str,List[Any]],None] = dc.Field(
+        default=None,
         description="Matrix of parameter values to explore")
 
 class TaskDef(BaseModel):
@@ -107,5 +108,7 @@ class TaskDef(BaseModel):
             name=self.name,
             type=self.type,
             depends=self.depends.copy())
-        return ret  
+        return ret 
+
+TaskDef.model_rebuild()
 

{dv_flow_mgr-0.0.1.14147547123a1 → dv_flow_mgr-0.0.2.14148837747a1}/src/dv_flow/mgr/task_graph_builder.py

@@ -203,13 +203,21 @@ class TaskGraphBuilder(object):
         if ctxt.rundir is None or ctxt.rundir == RundirE.Unique:
             self._rundir_s.pop()
 
-    def mkTaskGraph(self, task : str) -> TaskNode:
+    def mkTaskGraph(self, task : str, rundir=None) -> TaskNode:
         self._pkg_s.clear()
         self._task_m.clear()
 
-        return self._mkTaskGraph(task, self.rundir)
+        if rundir is not None:
+            self._rundir_s.append(rundir)
+
+        ret = self._mkTaskGraph(task)
+
+        if rundir is not None:
+            self._rundir_s.pop()
+
+        return ret
         
-    def _mkTaskGraph(self, task : str, parent_rundir : str) -> TaskNode:
+    def _mkTaskGraph(self, task : str) -> TaskNode:
 
         elems = task.split(".")
 
@@ -224,8 +232,6 @@ class TaskGraphBuilder(object):
         else:
             pkg_spec = PackageSpec(pkg_name)
 
-        rundir = os.path.join(parent_rundir, pkg_name, task_name)
-
         self._logger.debug("pkg_spec: %s" % str(pkg_spec))
         self._pkg_spec_s.append(pkg_spec)
         pkg = self.getPackage(pkg_spec)
@@ -243,7 +249,7 @@ class TaskGraphBuilder(object):
             need_fullname = self._resolveNeedRef(need_def)
             self._logger.debug("Searching for qualifed-name task %s" % need_fullname)
             if not need_fullname in self._task_m.keys():
-                need_t = self._mkTaskGraph(need_fullname, rundir)
+                need_t = self._mkTaskGraph(need_fullname)
                 self._task_m[need_fullname] = need_t
             needs.append(self._task_m[need_fullname])
 
@@ -258,7 +264,7 @@ class TaskGraphBuilder(object):
             params=params,
             name=task,
             needs=needs)
-        task.rundir = rundir
+#        task.rundir = rundir
         
         self._task_m[task.name] = task
 
@@ -359,7 +365,7 @@ class TaskGraphBuilder(object):
                 need_fullname = self._resolveNeedRef(need_def)
                 self._logger.debug("Searching for qualifed-name task %s" % need_fullname)
                 if not need_fullname in self._task_m.keys():
-                    need_t = self._mkTaskGraph(need_fullname, self.rundir)
+                    need_t = self._mkTaskGraph(need_fullname)
                     self._task_m[need_fullname] = need_t
                 needs.append(self._task_m[need_fullname])
 
@@ -371,13 +377,14 @@ class TaskGraphBuilder(object):
                 name=name, 
                 srcdir=srcdir, 
                 needs=needs)
+            ret.rundir = self.get_rundir(name)
         else:
             raise Exception("Failed to find ctor for task %s" % task_t)
         self._pkg_s.pop()
         self._logger.debug("<-- mkTaskNode: %s" % task_t)
         return ret
         
-    def getTaskCtor(self, spec : Union[str,'TaskSpec'], pkg : PackageDef = None) -> 'TaskCtor':
+    def getTaskCtor(self, spec : Union[str,'TaskSpec'], pkg : PackageDef = None) -> 'TaskNodeCtor':
         from .task_def import TaskSpec
         if type(spec) == str:
             spec = TaskSpec(spec)

{dv_flow_mgr-0.0.1.14147547123a1 → dv_flow_mgr-0.0.2.14148837747a1}/src/dv_flow/mgr/task_node.py

@@ -20,6 +20,7 @@
 #*
 #****************************************************************************
 import enum
+import json
 import os
 import sys
 import dataclasses as dc
@@ -27,7 +28,7 @@ import pydantic.dataclasses as pdc
 import logging
 import toposort
 from typing import Any, Callable, ClassVar, Dict, List, Tuple
-from .task_data import TaskDataOutput, TaskDataResult
+from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
 from .param import Param
 
 class RundirE(enum.Enum):
@@ -49,11 +50,11 @@ class TaskNode(object):
     consumes : List[Any] = dc.field(default_factory=list)
     needs : List[Tuple['TaskNode',bool]] = dc.field(default_factory=list)
     rundir : List[str] = dc.field(default=None)
-#    rundir_t : RundirE = dc.field(default=RundirE.Unique)
     output : TaskDataOutput = dc.field(default=None)
     result : TaskDataResult = dc.field(default=None)
     start : float = dc.field(default=None)
     end : float = dc.field(default=None)
+    save_exec_data : bool = dc.field(default=True)
 
     _log : ClassVar = logging.getLogger("TaskNode")
 
@@ -101,6 +102,36 @@ class TaskNode(object):
         self._log.debug("<-- _matches: %s %s" % (self.name, consumed))
         return consumed
     
+    def _save_exec_data(self, rundir, input : TaskDataInput):
+        """Saves execution data to the rundir"""
+        data = {
+            "name": self.name,
+            "srcdir": self.srcdir,
+            "rundir": rundir,
+            "input": input.model_dump(warnings=False),
+            "needs": [need[0].name for need in self.needs],
+            "result": {
+                "status": self.result.status,
+                "changed": self.result.changed,
+                "memento": self.result.memento.model_dump() if self.result.memento else None,
+            },
+            "output": self.output.model_dump(),
+        }
+
+        if isinstance(self.consumes, list):
+            data["consumes"] = self.consumes
+        else:
+            data["consumes"] = str(self.consumes)
+
+        if isinstance(self.passthrough, list):
+            data["passthrough"] = self.passthrough
+        else:
+            data["passthrough"] = str(self.passthrough)
+
+        with open(os.path.join(rundir, "%s.exec_data.json" % self.name), "w") as f:
+            json.dump(data, f, indent=2)
+            f.write("\n")
+    
 
 
 

{dv_flow_mgr-0.0.1.14147547123a1 → dv_flow_mgr-0.0.2.14148837747a1}/src/dv_flow/mgr/task_node_ctor_compound.py

@@ -53,9 +53,10 @@ class TaskNodeCtorCompound(TaskNodeCtor):
             params=params,
             needs=needs)
         # Use the compound task's rundir
-        node.input.rundir = builder.get_rundir()
 
         builder.enter_compound(node, self.task_def.rundir)
+        node.input.rundir = builder.get_rundir()
+        print("input rundir: %s" % str(node.input.rundir))
         builder.addTask("in", node.input)
 
         self._buildSubGraph(builder, node)

{dv_flow_mgr-0.0.1.14147547123a1 → dv_flow_mgr-0.0.2.14148837747a1}/src/dv_flow/mgr/task_node_leaf.py

@@ -172,6 +172,9 @@ class TaskNodeLeaf(TaskNode):
             changed=self.result.changed,
             dep_m=dep_m,
             output=output)
+        
+        if self.save_exec_data:
+            self._save_exec_data(rundir, input)
 
         # TODO: 
         self._log.debug("<-- do_run: %s" % self.name)

{dv_flow_mgr-0.0.1.14147547123a1 → dv_flow_mgr-0.0.2.14148837747a1}/src/dv_flow_mgr.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dv-flow-mgr
-Version: 0.0.1.14147547123a1
+Version: 0.0.2.14148837747a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.14147547123a1 → dv_flow_mgr-0.0.2.14148837747a1}/src/dv_flow_mgr.egg-info/SOURCES.txt

@@ -6,6 +6,7 @@ ivpm.yaml
 pyproject.toml
 .github/workflows/ci.yml
 .vscode/settings.json
+docs/ChangeLog.md
 docs/KeyArchitecture.md
 docs/Makefile
 docs/Notes.md

{dv_flow_mgr-0.0.1.14147547123a1 → dv_flow_mgr-0.0.2.14148837747a1}/tests/unit/test_compound_task.py

@@ -22,6 +22,7 @@ package:
         rundir: inherit
         uses: std.FileSet
         needs: [create_file]
+        passthrough: none
         with:
           base: ${{ rundir }}
           include: "*.txt"
@@ -69,6 +70,7 @@ package:
       - name: glob_txt
         rundir: inherit
         uses: std.FileSet
+        passthrough: none
         needs: [create_file]
         with:
           base: ${{ rundir }}
@@ -81,13 +83,15 @@ package:
       uses: TaskType1
 
     - name: entry
+      passthrough: all
+      consumes: none
       needs: [Task1, Task2]
 """
 
     rundir = os.path.join(tmpdir)
     with open(os.path.join(rundir, "flow.dv"), "w") as fp:
         fp.write(flow_dv)
-
+    
     pkg_def = PackageDef.load(os.path.join(rundir, "flow.dv"))
     builder = TaskGraphBuilder(
         root_pkg=pkg_def,

{dv_flow_mgr-0.0.1.14147547123a1 → dv_flow_mgr-0.0.2.14148837747a1}/tests/unit/test_smoke.py

@@ -6,7 +6,7 @@ import pytest
 import jq
 from typing import List
 import yaml
-from dv_flow.mgr import FileSet, PackageDef, TaskData, TaskGraphBuilder
+from dv_flow.mgr import TaskSetRunner, PackageDef, TaskData, TaskGraphBuilder
 from pydantic import BaseModel
 from shutil import copytree
 
@@ -26,7 +26,9 @@ package:
 
     pkg_def = PackageDef.loads(flowdv)
     builder = TaskGraphBuilder(pkg_def, os.getcwd())
-    task = builder.mkTaskGraph("my_pkg.entry")
+    task = builder.mkTaskGraph("my_pkg.entry", rundir="mk_pkg.entry")
+
+    assert len(task.rundir) == 1
 
 def test_jq():
     data = [