dv-flow-mgr 0.0.1.13577785562a1__tar.gz → 0.0.1.13657597614a1__tar.gz

{dv_flow_mgr-0.0.1.13577785562a1 → dv_flow_mgr-0.0.1.13657597614a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.13577785562a1
+Version: 0.0.1.13657597614a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.13577785562a1 → dv_flow_mgr-0.0.1.13657597614a1}/docs/Steps.md

@@ -115,4 +115,31 @@ with:
   # List-of 
   - append-list: 
 
+# Why PSS?
+- Key usecases
+
+# Open Source and PSS
+
+# YAML basics
+- Load packages and create TaskNodeCtor objects from a YAML file
+- Define types in YAML
+- Implement a 'task factory'
+  - Support package overrides (config setting?)
+  - 
+
+# TaskNode, TaskNodeCtor
+- TaskNodeCtor exists for each node type
+  - Every node declaration in a YAML file
+  - Node types accessed via the Python API
+- A new node -- likely with different parameter values -- can be created via the API
+- A node created via the YAML spec has a dedicated node ctor
+
+When created via the 
+
+# Early Differentiators
+- Library of tools with aggressive work avoidance (faster turnaround)
+- Cross-tool support (strategy for category support)
+- Extract and display markers (easier identification of failures)
+
+
 

{dv_flow_mgr-0.0.1.13577785562a1 → dv_flow_mgr-0.0.1.13657597614a1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dv-flow-mgr"
-version = "0.0.1.13577785562a1"
+version = "0.0.1.13657597614a1"
 dependencies = [
     'pydantic',
     'pyyaml',

{dv_flow_mgr-0.0.1.13577785562a1 → dv_flow_mgr-0.0.1.13657597614a1}/src/dv_flow/mgr/__init__.py

@@ -7,4 +7,5 @@ from .task_data import *
 from .task_graph_runner import TaskGraphRunner
 from .task_graph_runner_local import TaskGraphRunnerLocal
 from .task_graph_builder import TaskGraphBuilder
+from .task_node import task
 

{dv_flow_mgr-0.0.1.13577785562a1 → dv_flow_mgr-0.0.1.13657597614a1}/src/dv_flow/mgr/package_def.py

@@ -33,10 +33,10 @@ from typing import Any, Dict, List, Callable, Tuple, ClassVar
 from .fragment_def import FragmentDef
 from .package import Package
 from .package_import_spec import PackageImportSpec, PackageSpec
-#from .task import TaskCtorCls, TaskCtorParam, TaskCtorParamCls
+from .task_node import TaskNodeCtor, TaskNodeCtorProxy, TaskNodeCtorTask
 from .task_ctor import TaskCtor
 from .task_def import TaskDef, TaskSpec
-from .std.task_null import TaskNull
+from .std.task_null import TaskNull, TaskNullParams
 from .type_def import TypeDef
 
 
@@ -122,115 +122,27 @@ class PackageDef(BaseModel):
             ctor_t = tasks_m[task_name][2]
         return ctor_t
 
-    def handleParams(self, task, ctor_t):
-        self._log.debug("--> handleParams %s params=%s" % (task.name, str(task.params)))
-
-        if task.params is not None and len(task.params) > 0:
-            decl_params = False
-
-            # First, add in a parameter-setting stage 
-            ctor_t = TaskCtorParam(
-                name=ctor_t.name,
-                uses=ctor_t,
-                srcdir=ctor_t.srcdir)
-#            ctor_t.params.update(task.params)
-
-            for value in task.params.values():
-                self._log.debug("value: %s" % str(value))
-                if type(value) == dict and "type" in value.keys():
-                    decl_params = True
-                    break
-
-            field_m = {}
-            # First, add parameters from the base class
-            base_o = ctor_t.mkParams()
-            for fname,info in base_o.model_fields.items():
-                self._log.debug("Field: %s (%s)" % (fname, info.default))
-                field_m[fname] = (info.annotation, info.default)
-
-            if decl_params:
-                self._log.debug("Type declares new parameters")
-                # We need to combine base parameters with new parameters
-                ptype_m = {
-                    "str" : str,
-                    "int" : int,
-                    "float" : float,
-                    "bool" : bool,
-                    "list" : List
-                }
-                pdflt_m = {
-                    "str" : "",
-                    "int" : 0,
-                    "float" : 0.0,
-                    "bool" : False,
-                    "list" : []
-                }
-                for p in task.params.keys():
-                    param = task.params[p]
-                    self._log.debug("param: %s" % str(param))
-                    if type(param) == dict and "type" in param.keys():
-                        ptype_s = param["type"]
-                        if ptype_s not in ptype_m.keys():
-                            raise Exception("Unknown type %s" % ptype_s)
-                        ptype = ptype_m[ptype_s]
-
-                        if p in field_m.keys():
-                            raise Exception("Duplicate field %s" % p)
-                        if "value" in param.keys():
-                            field_m[p] = (ptype, param["value"])
-                        else:
-                            field_m[p] = (ptype, pdflt_m[ptype_s])
-                    else:
-                        if p not in field_m.keys():
-                            raise Exception("Field %s not found" % p)
-                        if type(param) != dict:
-                            value = param
-                        elif "value" in param.keys():
-                            value = param["value"]
-                        else:
-                            raise Exception("No value specified for param %s: %s" % (
-                                p, str(param)))
-                        field_m[p] = (field_m[p][0], value)
-                self._log.debug("field_m: %s" % str(field_m))
-                param_t = pydantic.create_model(
-                    "Task%sParams" % task.name, **field_m)
-                ctor_t = TaskCtorParamCls(
-                    name=ctor_t.name,
-                    uses=ctor_t,
-                    params_ctor=param_t)
-            else: # no new parameters declared
-                self._log.debug("Type only overrides existing parameters")
-                for p in task.params.keys():
-                    param = task.params[p]
-                    if p not in field_m.keys():
-                        raise Exception("Field %s not found" % p)
-                    if type(param) != dict:
-                        value = param
-                    elif "value" in param.keys():
-                        value = param["value"]
-                    else:
-                        raise Exception("No value specified for param %s: %s" % (
-                            p, str(param)))
-                    field_m[p] = (field_m[p][0], value)
-                    self._log.debug("Set param=%s to %s" % (p, str(value)))
-                    ctor_t.params[p] = value
-
-        self._log.debug("<-- handleParams %s" % task.name)
-
-        return ctor_t
-
     def mkTaskCtor(self, session, task, srcdir, tasks_m) -> TaskCtor:
         self._log.debug("--> %s::mkTaskCtor %s (srcdir: %s)" % (self.name, task.name, srcdir))
+        base_ctor_t : TaskCtor = None
         ctor_t : TaskCtor = None
+        base_params : BaseModel = None
+        callable = None
+        passthrough = False
+        needs = [] if task.needs is None else task.needs.copy()
+
+        if task.uses is not None:
+            base_ctor_t = self.getTaskCtor(session, task.uses, tasks_m)
+            base_params = base_ctor_t.mkTaskParams()
 
         # Determine the implementation constructor first
-        if task.pyclass is not None:
+        if task.pytask is not None:
             # Built-in impl
             # Now, lookup the class
-            self._log.debug("Use PyClass implementation")
-            last_dot = task.pyclass.rfind('.')
-            clsname = task.pyclass[last_dot+1:]
-            modname = task.pyclass[:last_dot]
+            self._log.debug("Use PyTask implementation")
+            last_dot = task.pytask.rfind('.')
+            clsname = task.pytask[last_dot+1:]
+            modname = task.pytask[:last_dot]
 
             try:
                 if modname not in sys.modules:
@@ -244,39 +156,106 @@ class PackageDef(BaseModel):
                     modname, self.basedir, str(e)))
                 
             if not hasattr(mod, clsname):
-                raise Exception("Class %s not found in module %s" % (clsname, modname))
-            task_ctor = getattr(mod, clsname)
-
-            # Determine if we need to use a new 
-
-            if task.uses is not None:
-                uses = self.getTaskCtor(session, task.uses, tasks_m)
-            else:
-                uses = None
-
-            ctor_t = TaskCtorCls(
+                raise Exception("Method %s not found in module %s" % (clsname, modname))
+            callable = getattr(mod, clsname)
+
+        # Determine if we need to use a new 
+        paramT = self._getParamT(task, base_params)
+        
+        if callable is not None:
+            ctor_t = TaskNodeCtorTask(
                 name=task.name,
-                uses=uses,
-                task_ctor=task_ctor,
-                srcdir=srcdir)
-        elif task.uses is not None:
+                srcdir=srcdir,
+                paramT=paramT, # TODO: need to determine the parameter type
+                passthrough=passthrough,
+                needs=needs, # TODO: need to determine the needs
+                task=callable)
+        elif base_ctor_t is not None:
             # Use the existing (base) to create the implementation
-            ctor_t = TaskCtor(
+            ctor_t = TaskNodeCtorProxy(
                 name=task.name,
-                uses=self.getTaskCtor(session, task.uses, tasks_m),
-                srcdir=srcdir)
+                srcdir=srcdir,
+                paramT=paramT, # TODO: need to determine the parameter type
+                passthrough=passthrough,
+                needs=needs,
+                uses=base_ctor_t)
         else:
             self._log.debug("Use 'Null' as the class implementation")
-            ctor_t = TaskCtorCls(
+            ctor_t = TaskNodeCtorTask(
                 name=task.name,
-                task_ctor=TaskNull,
-                srcdir=srcdir)
-
-        ctor_t = self.handleParams(task, ctor_t)
-        ctor_t.depends.extend(task.depends)
+                srcdir=srcdir,
+                paramT=TaskNullParams,
+                passthrough=passthrough,
+                needs=needs,
+                task=TaskNull)
 
         self._log.debug("<-- %s::mkTaskCtor %s" % (self.name, task.name))
         return ctor_t
+    
+    def _getParamT(self, task, base_t : BaseModel):
+        # Get the base parameter type (if available)
+        # We will build a new type with updated fields
+
+        ptype_m = {
+            "str" : str,
+            "int" : int,
+            "float" : float,
+            "bool" : bool,
+            "list" : List
+        }
+        pdflt_m = {
+            "str" : "",
+            "int" : 0,
+            "float" : 0.0,
+            "bool" : False,
+            "list" : []
+        }
+
+        fields = []
+        field_m : Dict[str,int] = {}
+
+        # First, pull out existing fields (if there's a base type)
+        if base_t is not None:
+            self._log.debug("Base type: %s" % str(base_t))
+            for name,f in base_t.model_fields.items():
+                ff : dc.Field = f
+                fields.append(f)
+                field_m[name] = (f.annotation, getattr(base_t, name))
+        else:
+            self._log.debug("No base type")
+
+        for p in task.params.keys():
+            param = task.params[p]
+            self._log.debug("param: %s %s (%s)" % (p, str(param), str(type(param))))
+            if hasattr(param, "type") and param.type is not None:
+                ptype_s = param.type
+                if ptype_s not in ptype_m.keys():
+                    raise Exception("Unknown type %s" % ptype_s)
+                ptype = ptype_m[ptype_s]
+
+                if p in field_m.keys():
+                    raise Exception("Duplicate field %s" % p)
+                if param.value is not None:
+                    field_m[p] = (ptype, param.value)
+                else:
+                    field_m[p] = (ptype, pdflt_m[ptype_s])
+                self._log.debug("Set param=%s to %s" % (p, str(field_m[p][1])))
+            else:
+                if p not in field_m.keys():
+                    raise Exception("Field %s not found" % p)
+                if type(param) != dict:
+                    value = param
+                elif "value" in param.keys():
+                    value = param["value"]
+                else:
+                    raise Exception("No value specified for param %s: %s" % (
+                        p, str(param)))
+                field_m[p] = (field_m[p][0], value)
+                self._log.debug("Set param=%s to %s" % (p, str(field_m[p][1])))
+
+        params_t = pydantic.create_model("Task%sParams" % task.name, **field_m)
+
+        return params_t
 
     @staticmethod
     def load(path, exp_pkg_name=None):

{dv_flow_mgr-0.0.1.13577785562a1 → dv_flow_mgr-0.0.1.13657597614a1}/src/dv_flow/mgr/param_def.py

@@ -14,7 +14,6 @@ class ComplexType(BaseModel):
     map : Union[MapType, None] = None
 
 class ParamDef(BaseModel):
-    name : str
     doc : str = None
     type : Union[str, 'ComplexType'] = None
     value : Union[Any, None] = None

{dv_flow_mgr-0.0.1.13577785562a1 → dv_flow_mgr-0.0.1.13657597614a1}/src/dv_flow/mgr/param_ref_eval.py

@@ -7,8 +7,8 @@ from .eval_jq import eval_jq
 @dc.dataclass
 class ParamRefEval(object):
 
-    parser : ExprParser = ExprParser()
-    expr_eval : ExprEval = ExprEval()
+    parser : ExprParser = dc.field(default_factory=ExprParser)
+    expr_eval : ExprEval = dc.field(default_factory=ExprEval)
 
     def __post_init__(self):
         self.expr_eval.methods["jq"] = eval_jq

dv_flow_mgr-0.0.1.13657597614a1/src/dv_flow/mgr/std/exec.py

@@ -0,0 +1,19 @@
+import asyncio
+import logging
+from dv_flow.mgr import TaskDataResult
+
+_log = logging.getLogger("Exec")
+
+async def Exec(runner, input) -> TaskDataResult:
+    _log.debug("TaskExec run: %s: cmd=%s" % (input.name, input.params.command))
+
+
+    proc = await asyncio.create_subprocess_shell(
+        input.params.command,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE)
+    
+    await proc.wait()
+
+    return TaskDataResult()
+

dv_flow_mgr-0.0.1.13657597614a1/src/dv_flow/mgr/std/fileset.py

@@ -0,0 +1,75 @@
+
+import os
+import fnmatch
+import glob
+import logging
+import pydantic.dataclasses as dc
+from pydantic import BaseModel
+from typing import ClassVar, List, Tuple
+from dv_flow.mgr import TaskDataResult
+from dv_flow.mgr import FileSet as _FileSet
+
+class TaskFileSetMemento(BaseModel):
+    files : List[Tuple[str,float]] = dc.Field(default_factory=list)
+
+_log = logging.getLogger("FileSet")
+
+async def FileSet(runner, input) -> TaskDataResult:
+    _log.debug("TaskFileSet run: %s: basedir=%s, base=%s type=%s include=%s" % (
+        input.name,
+        input.srcdir,
+        input.params.base, input.params.type, str(input.params.include)
+    ))
+
+
+    changed = False
+    ex_memento = input.memento
+    memento = TaskFileSetMemento()
+
+    _log.debug("ex_memento: %s" % str(ex_memento))
+    _log.debug("params: %s" % str(input.params))
+
+    if input.params is not None:
+        glob_root = os.path.join(input.srcdir, input.params.base)
+        glob_root = glob_root.strip()
+
+        if glob_root[-1] == '/' or glob_root == '\\':
+            glob_root = glob_root[:-1]
+
+        _log.debug("glob_root: %s" % glob_root)
+
+        fs = _FileSet(
+                src=input.name, 
+                type=input.params.type,
+                basedir=glob_root)
+
+        if not isinstance(input.params.include, list):
+            input.params.include = [input.params.include]
+
+        included_files = []
+        for pattern in input.params.include:
+            included_files.extend(glob.glob(os.path.join(glob_root, pattern), recursive=False))
+
+        _log.debug("included_files: %s" % str(included_files))
+
+        for file in included_files:
+            if not any(glob.fnmatch.fnmatch(file, os.path.join(glob_root, pattern)) for pattern in input.params.exclude):
+                memento.files.append((file, os.path.getmtime(os.path.join(glob_root, file))))
+                fs.files.append(file[len(glob_root)+1:])
+
+    # Check to see if the filelist or fileset have changed
+    # Only bother doing this if the upstream task data has not changed
+    if ex_memento is not None and not input.changed:
+        ex_memento.files.sort(key=lambda x: x[0])
+        memento.files.sort(key=lambda x: x[0])
+        _log.debug("ex_memento.files: %s" % str(ex_memento.files))
+        _log.debug("memento.files: %s" % str(memento.files))
+        changed = ex_memento != memento
+    else:
+        changed = True
+
+    return TaskDataResult(
+        memento=memento,
+        changed=changed,
+        output=[fs]
+    )

{dv_flow_mgr-0.0.1.13577785562a1 → dv_flow_mgr-0.0.1.13657597614a1}/src/dv_flow/mgr/std/flow.dv

@@ -1,16 +1,19 @@
 
+# yaml-language-server: $schema=https://dv-flow.github.io/dv-flow.schema.json
+
 package:
   name: std
 
   tasks:
   - name: Message
-    pyclass: dv_flow.mgr.std.message.Message
+    pytask: dv_flow.mgr.std.message.Message
     with:
       msg:
         type: str
         value: ""
   - name: FileSet
-    pyclass: dv_flow.mgr.std.fileset.FileSet
+    pytask: dv_flow.mgr.std.fileset.FileSet
+    passthrough: true
     with:
       base:
         type: str
@@ -24,9 +27,12 @@ package:
       exclude:
         type: str
         value: ""
-#  - name: Exec
-#    pyclass: dv_flow.mgr.std.exec.Exec
-#    with: {}
+  - name: Exec
+    pytask: dv_flow.mgr.std.exec.Exec
+    with: 
+      command:
+        type: str
+        value: ""
   types:
   # - name: TaskDataItem
   #   doc: |

dv_flow_mgr-0.0.1.13657597614a1/src/dv_flow/mgr/std/message.py

@@ -0,0 +1,6 @@
+
+from dv_flow.mgr import Task, TaskDataResult
+
+async def Message(runner, input) -> TaskDataResult:
+        print("%s: %s" % (input.name, input.params.msg))
+        return TaskDataResult()

dv_flow_mgr-0.0.1.13657597614a1/src/dv_flow/mgr/std/task_null.py

@@ -0,0 +1,9 @@
+from pydantic import BaseModel
+from ..task_data import TaskDataResult
+
+class TaskNullParams(BaseModel):
+    pass
+
+async def TaskNull(runner, input) -> TaskDataResult:
+    return TaskDataResult()
+

{dv_flow_mgr-0.0.1.13577785562a1 → dv_flow_mgr-0.0.1.13657597614a1}/src/dv_flow/mgr/task_data.py

@@ -42,6 +42,7 @@ class TaskParameterSet(BaseModel):
     seq : int  = -1 # Order in which the param-set must appear
 
 class TaskDataInput(BaseModel):
+    name : str
     changed : bool
     srcdir : str
     rundir : str

{dv_flow_mgr-0.0.1.13577785562a1 → dv_flow_mgr-0.0.1.13657597614a1}/src/dv_flow/mgr/task_def.py

@@ -35,11 +35,12 @@ class TaskDef(BaseModel):
     name : str
 #    type : Union[str,TaskSpec] = dc.Field(default_factory=list)
     uses : str = dc.Field(default=None)
-    pyclass : str = dc.Field(default=None)
+    pytask : str = dc.Field(default=None)
     desc : str = dc.Field(default="")
     doc : str = dc.Field(default="")
-    depends : List[Union[str,TaskSpec]] = dc.Field(default_factory=list, alias="needs")
-    params: List[ParamDef] = dc.Field(default_factory=list, alias="with")
+    needs : List[Union[str,TaskSpec]] = dc.Field(default_factory=list, alias="needs")
+    params: Dict[str,Union[str,ParamDef]] = dc.Field(default_factory=dict, alias="with")
+    passthrough: bool = dc.Field(default=False)
 #    out: List[TaskOutput] = dc.Field(default_factory=list)
 
     def copy(self) -> 'TaskDef':

{dv_flow_mgr-0.0.1.13577785562a1 → dv_flow_mgr-0.0.1.13657597614a1}/src/dv_flow/mgr/task_graph_builder.py

@@ -26,6 +26,7 @@ from .package import Package
 from .package_def import PackageDef, PackageSpec
 from .pkg_rgy import PkgRgy
 from .task import Task
+from .task_node import TaskNodeCtor
 from typing import Dict, List
 
 @dc.dataclass
@@ -92,24 +93,29 @@ class TaskGraphBuilder(object):
         
         self._pkg_s.append(pkg)
 
-        ctor_t : TaskCtor = pkg.getTaskCtor(task_name)
+        ctor_t : TaskNodeCtor = pkg.getTaskCtor(task_name)
 
         self._logger.debug("ctor_t: %s" % ctor_t.name)
 
-        depends = []
+        needs = []
 
-        for dep in ctor_t.depends:
-            if not dep in self._task_m.keys():
-                task = self._mkTaskGraph(dep, rundir)
-                self._task_m[dep] = task
-                pass
-            depends.append(self._task_m[dep])
+        for need_def in ctor_t.getNeeds():
+            if not need_def in self._task_m.keys():
+                task = self._mkTaskGraph(need_def, rundir)
+                self._task_m[need_def] = task
+            needs.append(self._task_m[need_def])
 
         # The returned task should have all param references resolved
-        task = ctor_t.mkTask(
+        params = ctor_t.mkTaskParams()
+
+        if params is None:
+            raise Exception("ctor %s returned None for params" % str(ctor_t))
+
+        task = ctor_t.mkTaskNode(
+            params=params,
             name=task_name,
-            depends=depends,
-            rundir=rundir)
+            needs=needs)
+        task.rundir = rundir
         
         self._task_m[task.name] = task
 

dv_flow_mgr-0.0.1.13657597614a1/src/dv_flow/mgr/task_listener_log.py

@@ -0,0 +1,15 @@
+import dataclasses as dc
+
+class TaskListenerLog(object):
+
+    def event(self, task : 'Task', reason : 'Reason'):
+        if reason == 'enter':
+            print("> Task %s" % task.name, flush=True)
+        elif reason == 'leave':
+            for m in task.result.markers:
+                print("  %s" % m)
+            print("< Task %s" % task.name, flush=True)
+        else:
+            print("- Task %s" % task.name, flush=True)
+        pass
+