dv-flow-mgr 1.0.0.14528489065a1__py3-none-any.whl → 1.5.0__py3-none-any.whl

dv_flow/mgr/task_graph_dot_writer.py

@@ -48,9 +48,31 @@ class TaskGraphDotWriter(object):
             self._node_id += 1
             node_name = "n%d" % node_id
             self._node_id_m[node] = node_name
-            self.println("%s[label=\"%s\"];" % (node_name, node.name))
+            self.println("%s[label=\"%s\",tooltip=\"%s\"];" % (
+                node_name, 
+                node.name,
+                self._genLeafTooltip(node)))
         self._log.debug("<-- build_node %s (%d)" % (node.name, len(node.needs),))
 
+    def _genLeafTooltip(self, node):
+        params = type(node.params).model_fields
+        ret = ""
+        if len(params):
+            ret += "Parameters:\\n"
+            for k in type(node.params).model_fields.keys():
+                ret += "- %s: " % k
+                v = getattr(node.params, k)
+                if isinstance(v, str):
+                    ret += "%s" % v
+                elif isinstance(v, list):
+                    ret += "[%s]" % ", ".join([str(x) for x in v])
+                elif isinstance(v, dict):
+                    ret += "{%s}" % ", ".join(["%s: %s" % (str(k), str(v)) for k,v in v.items()])
+                else:
+                    ret += "%s" % str(v)
+                ret += "\\n"
+        return ret
+
     def process_needs(self, node):
         self._log.debug("--> process_needs %s (%d)" % (node.name, len(node.needs),))
 
@@ -99,13 +121,17 @@ class TaskGraphDotWriter(object):
         self.println("subgraph cluster_%d {" % id)
         self.inc_ind()
         self.println("label=\"%s\";" % node.name)
+        self.println("tooltip=\"%s\";" % self._genLeafTooltip(node))
         self.println("color=blue;")
         self.println("style=dashed;")
 
         task_node_id = self._node_id
         self._node_id += 1
         task_node_name = "n%d" % task_node_id
-        self.println("%s[label=\"%s\"];" % (task_node_name, node.name))
+        self.println("%s[label=\"%s\", tooltip=\"%s\"];" % (
+            task_node_name, 
+            node.name,
+            self._genLeafTooltip(node)))
         self._node_id_m[node] = task_node_name
 
         for n in node.tasks:
@@ -119,7 +145,10 @@ class TaskGraphDotWriter(object):
                 node_name = "n%d" % node_id
                 self._node_id_m[n] = node_name
                 leaf_name = n.name[n.name.rfind(".") + 1:]
-                self.println("%s[label=\"%s\"];" % (node_name, leaf_name))
+                self.println("%s[label=\"%s\",tooltip=\"%s\"];" % (
+                    node_name, 
+                    leaf_name,
+                    self._genLeafTooltip(n)))
         self.dec_ind()
         self.println("}")
 

dv_flow/mgr/task_listener_log.py

@@ -22,6 +22,7 @@
 import dataclasses as dc
 from datetime import datetime
 from rich.console import Console
+from typing import ClassVar, Dict
 from .task_data import SeverityE
 
 @dc.dataclass
@@ -29,12 +30,26 @@ class TaskListenerLog(object):
     console : Console = dc.field(default=None)
     level : int = 0
     quiet : bool = False
+    has_severity : Dict[SeverityE, int] = dc.field(default_factory=dict)
+
+    sev_pref_m : ClassVar = {
+        "info": "[blue]I[/blue]",
+        SeverityE.Info: "[blue]I[/blue]",
+        "warn": "[yellow]W[/yellow]",
+        SeverityE.Warning: "[yellow]W[/yellow]",
+        "error": "[red]E[/red]",
+        SeverityE.Error: "[red]E[/red]",
+    }
 
     def __post_init__(self):
         self.console = Console(highlight=False)
+        for sev in SeverityE:
+            self.has_severity[sev] = 0
 
     def marker(self, marker):
         """Receives markers during loading"""
+        self.show_marker(marker)
+        self.has_severity[marker.severity] += 1
         pass
 
     def event(self, task : 'Task', reason : 'Reason'):
@@ -55,39 +70,9 @@ class TaskListenerLog(object):
                     else:
                         delta_s = " %0.2fmS" % (1000*delta.total_seconds())
 
-                sev_pref_m = {
-                    "info": "[blue]I[/blue]",
-                    SeverityE.Info: "[blue]I[/blue]",
-                    "warn": "[yellow]W[/yellow]",
-                    SeverityE.Warning: "[yellow]W[/yellow]",
-                    "error": "[red]E[/red]",
-                    SeverityE.Error: "[red]E[/red]",
-                }
                 for m in task.result.markers:
-                    severity_s = str(m.severity)
-
-                    if m.severity in sev_pref_m.keys():
-                        sev_pref = sev_pref_m[m.severity]
-                    elif severity_s in sev_pref_m.keys():
-                        sev_pref = sev_pref_m[severity_s]
-                    else:
-                        sev_pref = ""
-
-                    msg = "  %s %s: %s" % (
-                        sev_pref,
-                        task.name,
-                        m.msg)
+                    self.show_marker(m, task.name, task.rundir)
 
-                    if m.loc is not None:
-                        self.console.print("%s" % msg)
-                        if m.loc.line != -1 and m.loc.pos != -1:
-                            self.console.print("    %s:%d:%d" % (m.loc.path, m.loc.line, m.loc.pos))
-                        elif m.loc.line != -1:
-                            self.console.print("    %s:%d" % (m.loc.path, m.loc.line))
-                        else:
-                            self.console.print("    %s" % m.loc.path)
-                    else:
-                        self.console.print("%s (%s)" % (msg, task.rundir))
                 if task.result.status == 0:
                     self.console.print("[green]<< [%d][/green] Task %s%s%s" % (
                         self.level, 
@@ -101,3 +86,33 @@ class TaskListenerLog(object):
             self.console.print("[red]-[/red] Task %s" % task.name)
         pass
 
+    def show_marker(self, m, name=None, rundir=None):
+        severity_s = str(m.severity)
+
+        if m.severity in self.sev_pref_m.keys():
+            sev_pref = self.sev_pref_m[m.severity]
+        elif severity_s in self.sev_pref_m.keys():
+            sev_pref = self.sev_pref_m[severity_s]
+        else:
+            sev_pref = ""
+
+        msg = "  %s%s: %s" % (
+            sev_pref,
+            (" " + name) if name is not None and name != "" else "",
+            m.msg)
+
+        if m.loc is not None:
+            self.console.print("%s" % msg)
+            if m.loc.line != -1 and m.loc.pos != -1:
+                self.console.print("    %s:%d:%d" % (m.loc.path, m.loc.line, m.loc.pos))
+            elif m.loc.line != -1:
+                self.console.print("    %s:%d" % (m.loc.path, m.loc.line))
+            else:
+                self.console.print("    %s" % m.loc.path)
+        else:
+            self.console.print("%s%s" % (
+                msg, 
+                ("(%s)" % rundir) if rundir is not None else ""))
+
+        pass
+

dv_flow/mgr/task_node.py

@@ -29,6 +29,7 @@ import logging
 import toposort
 from typing import Any, Callable, ClassVar, Dict, List, Tuple
 from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
+from .task_node_ctxt import TaskNodeCtxt
 from .task_run_ctxt import TaskRunCtxt
 from .param import Param
 
@@ -44,6 +45,7 @@ class TaskNode(object):
     srcdir : str
     # This can be the resolved parameters
     params : Any
+    ctxt : TaskNodeCtxt
 
     # Runtime fields -- these get populated during execution
     changed : bool = False
@@ -56,6 +58,7 @@ class TaskNode(object):
     start : float = dc.field(default=None)
     end : float = dc.field(default=None)
     save_exec_data : bool = dc.field(default=True)
+    iff : bool = dc.field(default=True)
     parent : 'TaskNode' = dc.field(default=None)
 
     _log : ClassVar = logging.getLogger("TaskNode")

dv_flow/mgr/task_node_compound.py

@@ -45,6 +45,7 @@ class TaskNodeCompound(TaskNode):
             name=self.name + ".in",
             srcdir=self.srcdir,
             params=NullParams(),
+            ctxt=self.ctxt,
             consumes=ConsumesE.No,
             passthrough=PassthroughE.All)
         self.input.task = null_run

dv_flow/mgr/task_node_ctor_wrapper.py

@@ -28,6 +28,7 @@ import logging
 import toposort
 from typing import Any, Callable, ClassVar, Dict, List, Tuple
 from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
+from .task_def import PassthroughE, ConsumesE
 from .param import Param
 from .task_node import TaskNode
 from .task_node_leaf import TaskNodeLeaf
@@ -47,6 +48,7 @@ class TaskNodeCtorWrapper(TaskNodeCtor):
             srcdir=srcdir, 
             params=params, 
             task=self.T, 
+            ctxt=None,
             needs=needs)
         node.passthrough = self.passthrough
         node.consumes = self.consumes
@@ -87,7 +89,7 @@ class TaskNodeCtorWrapper(TaskNodeCtor):
                     setattr(obj, key, value)
         return obj
     
-def task(paramT,passthrough=False,consumes=None):
+def task(paramT,passthrough=PassthroughE.Unused,consumes=ConsumesE.All):
     """Decorator to wrap a task method as a TaskNodeCtor"""
     def wrapper(T):
         task_mname = T.__module__

dv_flow/mgr/task_node_ctxt.py

@@ -0,0 +1,7 @@
+import dataclasses as dc
+
+@dc.dataclass
+class TaskNodeCtxt(object):
+    """Holds data shared with all task-graph nodes"""
+    root_pkgdir : str
+    root_rundir : str

dv_flow/mgr/task_node_leaf.py

@@ -10,7 +10,6 @@ from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult, TaskMarker
 from .task_def import ConsumesE, PassthroughE
 from .task_node import TaskNode
 from .task_run_ctxt import TaskRunCtxt
-from .param_ref_eval import ParamRefEval
 from .param import Param
 
 @dc.dataclass
@@ -21,6 +20,19 @@ class TaskNodeLeaf(TaskNode):
                   runner,
                   rundir,
                   memento : Any = None) -> 'TaskDataResult':
+        try:
+            ret = await self._do_run(runner, rundir, memento)
+        except Exception as e:
+            print("Exception: %s" % str(e))
+            ret = TaskDataResult()
+            raise e
+
+        return ret
+
+    async def _do_run(self, 
+                  runner,
+                  rundir,
+                  memento : Any = None) -> 'TaskDataResult':
         self._log.debug("--> do_run: %s" % self.name)
         changed = False
         for dep,_ in self.needs:
@@ -34,7 +46,7 @@ class TaskNodeLeaf(TaskNode):
         # TODO: Form dep-map from inputs
 
         dep_m = {}
-        for need,block in self.needs:
+        for i,(need,block) in enumerate(self.needs):
             self._log.debug("dep %s dep_m: %s" % (need.name, str(dep_m)))
             if not block:
                 for subdep in need.output.dep_m.keys():
@@ -45,45 +57,51 @@ class TaskNodeLeaf(TaskNode):
                             dep_m[subdep].append(dep)
         self._log.debug("input dep_m: %s %s" % (self.name, str(dep_m)))
 
-        sorted = toposort.toposort(dep_m)
-
-        in_params_m = {}
-        added_srcs = set()
-        for need,block in self.needs:
-            self._log.debug("Process need=%s block=%s" % (need.name, block))
-            if not block:
-                for p in need.output.output:
-
-                    # Avoid adding parameters from a single task more than once
-                    key = (p.src, p.seq)
-                    if key not in added_srcs:
-                        added_srcs.add(key)
-                        if p.src not in in_params_m.keys():
-                            in_params_m[p.src] = []
-                        in_params_m[p.src].append(p)
+        # This gets the dependencies in topological order
+#        sorted = toposort.toposort(dep_m)
 
-        # in_params holds parameter sets ordered by dependency
+        # Now, process the 'needs' in the order that they're listed
         in_params = []
-        for sorted_s in sorted:
-            self._log.debug("sorted_s: %s" % str(sorted_s))
-            for dep in sorted_s:
-                if dep in in_params_m.keys():
-                    self._log.debug("(%s) Extend with: %s" % (dep, str(in_params_m[dep])))
-                    in_params.extend(in_params_m[dep])
+        in_params_s = set()
+        in_task_s = set()
+
+        for need, _ in self.needs:
+            if need not in in_task_s:
+                in_task_s.add(need)
+                for item in need.output.output:
+                    key = (item.src, item.seq)
+                    if key not in in_params_s:
+                        in_params_s.add(key)
+                        in_params.append(item)
+
+        # 
+        # in_params_m = {}
+        # added_srcs = set()
+        # for need,block in self.needs:
+        #     self._log.debug("Process need=%s block=%s" % (need.name, block))
+        #     if not block:
+        #         for p in need.output.output:
+
+        #             # Avoid adding parameters from a single task more than once
+        #             key = (p.src, p.seq)
+        #             if key not in added_srcs:
+        #                 added_srcs.add(key)
+        #                 if p.src not in in_params_m.keys():
+        #                     in_params_m[p.src] = []
+        #                 in_params_m[p.src].append(p)
+
+        # # in_params holds parameter sets ordered by dependency
+        # in_params = []
+        # for sorted_s in sorted:
+        #     self._log.debug("sorted_s: %s" % str(sorted_s))
+        #     for dep in sorted_s:
+        #         if dep in in_params_m.keys():
+        #             self._log.debug("(%s) Extend with: %s" % (dep, str(in_params_m[dep])))
+        #             in_params.extend(in_params_m[dep])
 
         self._log.debug("in_params[1]: %s" % ",".join(p.src for p in in_params))
 
-        # Create an evaluator for substituting param values
-        eval = ParamRefEval()
 
-        self._log.debug("in_params[2]: %s" % ",".join(p.src for p in in_params))
-        eval.setVar("in", in_params)
-        eval.setVar("rundir", rundir)
-
-        # Set variables from the inputs
-        for need in self.needs:
-            for name,value in {"rundir" : need[0].rundir}.items():
-                eval.setVar("%s.%s" % (need[0].name, name), value)
 
         # Default inputs is the list of parameter sets that match 'consumes'
         inputs = []
@@ -98,19 +116,6 @@ class TaskNodeLeaf(TaskNode):
         else:
             self._log.debug("consumes(unknown): %s" % str(self.consumes))
 
-        for name,field in type(self.params).model_fields.items():
-            value = getattr(self.params, name)
-            if type(value) == str:
-                if value.find("${{") != -1:
-                    new_val = eval.eval(value)
-                    self._log.debug("Param %s: Evaluate expression \"%s\" => \"%s\"" % (name, value, new_val))
-                    setattr(self.params, name, new_val)
-            elif isinstance(value, list):
-                for i,elem in enumerate(value):
-                    if elem.find("${{") != -1:
-                        new_val = eval.eval(elem)
-                        value[i] = new_val
-
         input = TaskDataInput(
             name=self.name,
             changed=changed,
@@ -120,7 +125,10 @@ class TaskNodeLeaf(TaskNode):
             inputs=inputs,
             memento=memento)
         
-        ctxt = TaskRunCtxt(runner=runner, rundir=input.rundir)
+        ctxt = TaskRunCtxt(
+            runner=runner, 
+            ctxt=self.ctxt,
+            rundir=input.rundir)
 
         self._log.debug("--> Call task method %s" % str(self.task))
         try:
@@ -144,10 +152,7 @@ class TaskNodeLeaf(TaskNode):
 
         self.result.markers.extend(ctxt._markers)
 
-        output=self.result.output.copy()
-        for i,out in enumerate(output):
-            out.src = self.name
-            out.seq = i
+        output = []
 
         self._log.debug("output[1]: %s" % str(output))
 
@@ -193,6 +198,13 @@ class TaskNodeLeaf(TaskNode):
 #                self.name : []
 #            }
 
+        # Add our own output
+        local_out = self.result.output.copy()
+        for i,out in enumerate(local_out):
+            out.src = self.name
+            out.seq = i
+            output.append(out)
+
         self._log.debug("output dep_m: %s %s" % (self.name, str(dep_m)))
         self._log.debug("output[2]: %s" % str(output))
 
@@ -214,6 +226,17 @@ class TaskNodeLeaf(TaskNode):
         if self.output is None:
             raise Exception("Task %s did not produce a result" % self.name)
         return self.result
+    
+    def _processNeed(self, need, in_params, in_task_s):
+        # Go depth-first
+        for nn, _ in need.needs:
+            self._processNeed(nn, in_params, in_task_s)
+
+        if need not in in_task_s:
+            in_params.extend(need.output.output)
+        
+
+
 
     def __hash__(self):
-        return id(self)
+        return id(self)

dv_flow/mgr/task_run_ctxt.py

@@ -5,6 +5,7 @@ import pydantic.dataclasses as pdc
 import os
 from typing import List
 from .task_data import TaskMarker, SeverityE, TaskMarkerLoc
+from .task_node_ctxt import TaskNodeCtxt
 
 class ExecInfo(BaseModel):
     cmd : List[str] = pdc.Field(default_factory=list)
@@ -13,10 +14,20 @@ class ExecInfo(BaseModel):
 @dc.dataclass
 class TaskRunCtxt(object):
     runner : 'TaskRunner'
+    ctxt : TaskNodeCtxt
     rundir : str
+
     _markers : List[TaskMarker] = dc.field(default_factory=list)
     _exec_info : List[ExecInfo] = dc.field(default_factory=list)
 
+    @property
+    def root_pkgdir(self):
+        return self.ctxt.root_pkgdir
+    
+    @property
+    def root_rundir(self):
+        return self.ctxt.root_rundir
+
     async def exec(self, 
                    cmd : List[str],
                    logfile=None,
@@ -55,6 +66,13 @@ class TaskRunCtxt(object):
         if status != 0:
             self.error("Command failed: %s" % " ".join(cmd))
 
+        if logfilter is not None:
+            with open(os.path.join(self.rundir, logfile), "r") as fp:
+                for line in fp.readlines():
+                    if logfilter(line):
+                        self.info(line.strip())
+                logfilter("")
+
         return status
 
     def create(self, path, content):

dv_flow/mgr/task_runner.py

@@ -126,11 +126,14 @@ class TaskSetRunner(TaskRunner):
 
                     # TaskNode rundir is a list of path elements relative
                     # to the root rundir
-                    rundir = self.rundir
-
-                    for rundir_e in t.rundir:
-                        rundir_e = re.sub(invalid_chars_pattern, '_', rundir_e)
-                        rundir = os.path.join(rundir, rundir_e)
+                    rundir = ""
+
+                    for i, rundir_e in enumerate(t.rundir):
+                        if i:
+                            rundir_e = re.sub(invalid_chars_pattern, '_', rundir_e)
+                            rundir = os.path.join(rundir, rundir_e)
+                        else:
+                            rundir = rundir_e
 
                     # if t.rundir_t == RundirE.Unique:
                     #     # Replace invalid characters with the replacement string.

dv_flow/mgr/type.py

@@ -0,0 +1,33 @@
+import dataclasses as dc
+from typing import Any, Dict, List
+from .srcinfo import SrcInfo
+
+@dc.dataclass
+class TypeField(object):
+    name : str
+    type : Any
+    doc : str = None
+    value : str = None
+    append : List[Any] = None
+    srcinfo : SrcInfo = None
+
+@dc.dataclass
+class Type(object):
+    name : str
+    doc : str = None
+    params : Dict[str, TypeField] = dc.field(default_factory=dict)
+    paramT : Any = None
+    uses : 'Type' = None
+    srcinfo : SrcInfo = None
+
+    def dump(self):
+        ret = {}
+        ret["name"] = self.name
+        ret["doc"] = self.doc
+        ret["params"] = {}
+        ret['srcinfo'] = self.srcinfo.dump()
+
+        return ret
+    
+    def __hash__(self):
+        return id(self)

dv_flow/mgr/type_def.py

@@ -19,12 +19,14 @@
 #*     Author: 
 #*
 #****************************************************************************
-from typing import List, Union
+from typing import Dict, List, Union
 from pydantic import BaseModel, Field
 from .param_def import ParamDef
+from .srcinfo import SrcInfo
 
 class TypeDef(BaseModel):
     name : str
     uses : str = None
     doc : str = None
-    fields : List[ParamDef] = Field(alias="with", default_factory=list)
+    params : Dict[str, Union[str, ParamDef]] = Field(alias="with", default_factory=list)
+    srcinfo : SrcInfo = None

dv_flow/mgr/util/util.py

@@ -22,20 +22,29 @@
 import os
 import yaml
 from ..package_loader import PackageLoader
+from ..task_data import TaskMarker, TaskMarkerLoc, SeverityE
 
 def loadProjPkgDef(path, listener=None):
     """Locates the project's flow spec and returns the PackageDef"""
 
     dir = path
     ret = None
+    found = False
     while dir != "/" and dir != "" and os.path.isdir(dir):
         if os.path.exists(os.path.join(dir, "flow.dv")):
             with open(os.path.join(dir, "flow.dv")) as f:
                 data = yaml.load(f, Loader=yaml.FullLoader)
                 if "package" in data.keys():
+                    found = True
                     listeners = [listener] if listener is not None else []
                     ret = PackageLoader(marker_listeners=listeners).load(os.path.join(dir, "flow.dv"))
                     break
         dir = os.path.dirname(dir)
+    
+    if not found:
+        if listener:
+            listener.marker(TaskMarker(
+                msg="Failed to find a 'flow.dv' file that defines a package in %s or its parent directories" % path))
+    
     return ret
 

dv_flow/mgr/yaml_srcinfo_loader.py

@@ -4,6 +4,7 @@ from yaml.loader import SafeLoader
 class YamlSrcInfoLoader(SafeLoader):
     scopes = {
         "tasks",
+        "types",
         "body",
         "package",
         "fragment"

{dv_flow_mgr-1.0.0.14528489065a1.dist-info → dv_flow_mgr-1.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dv-flow-mgr
-Version: 1.0.0.14528489065a1
+Version: 1.5.0
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License