dv-flow-mgr 0.0.1.12992722023a1__py3-none-any.whl → 0.0.1.13468776985a1__py3-none-any.whl

dv_flow/mgr/expr_eval.py

@@ -0,0 +1,50 @@
+
+import dataclasses as dc
+from typing import Any, Callable, Dict, List
+from .expr_parser import ExprVisitor, Expr, ExprBin, ExprBinOp, ExprCall, ExprId, ExprString, ExprInt
+
+@dc.dataclass
+class ExprEval(ExprVisitor):
+    methods : Dict[str, Callable] = dc.field(default_factory=dict)
+    variables : Dict[str, object] = dc.field(default_factory=dict)
+    value : Any = None
+
+    def eval(self, e : Expr):
+        self.value = None
+        e.accept(self)
+        return self.value
+
+    def visitExprId(self, e : ExprId):
+        if e.id in self.variables:
+            self.value = self.variables[e.id]
+        else:
+            raise Exception("Variable %s not found" % e.id)
+
+    def visitExprString(self, e : ExprString):
+        self.value = e.value
+    
+    def visitExprBin(self, e):
+        e.lhs.accept(self)
+
+        if e.op == ExprBinOp.Pipe:
+            # Value just goes over to the rhs
+            e.rhs.accept(self)
+        elif e.op == ExprBinOp.Plus:
+            pass
+    
+    def visitExprCall(self, e : ExprCall):
+        if e.id in self.methods:
+            # Need to gather up argument values
+            in_value = self.value
+            args = []
+            for arg in e.args:
+                self.value = None
+                arg.accept(self)
+                args.append(self.value)
+
+            self.value = self.methods[e.id](in_value, args)
+        else:
+            raise Exception("Method %s not found" % e.id)
+        
+    def visitExprInt(self, e : ExprInt):
+        self.value = e.value

dv_flow/mgr/expr_parser.py

@@ -0,0 +1,233 @@
+import dataclasses as dc
+import enum
+import ply.lex as lex
+import ply.yacc as yacc
+from typing import ClassVar, List
+
+@dc.dataclass
+class Expr(object):
+    def accept(self, v):
+        raise NotImplementedError("Expr.accept for %s" % str(type(sepf)))
+
+@dc.dataclass
+class ExprId(Expr):
+    id : str
+
+    def accept(self, v):
+        v.visitExprId(self)
+
+class ExprBinOp(enum.Enum):
+    Pipe = enum.auto()
+    Plus = enum.auto()
+    Minus = enum.auto()
+    Times = enum.auto()
+    Divide = enum.auto()
+
+@dc.dataclass
+class ExprBin(Expr):
+    op : ExprBinOp
+    lhs : Expr
+    rhs : Expr
+
+    def accept(self, v):
+        v.visitExprBin(self)
+
+@dc.dataclass
+class ExprCall(Expr):
+    id : str
+    args : List[Expr]
+
+    def accept(self, v):
+        v.visitExprCall(self)
+
+@dc.dataclass
+class ExprString(Expr):
+    value : str
+
+    def accept(self, v):
+        v.visitExprString(self)
+
+@dc.dataclass
+class ExprInt(Expr):
+    value : int
+
+    def accept(self, v):
+        v.visitExprInt(self)
+
+class ExprVisitor(object):
+    def visitExprId(self, e : ExprId):
+        pass
+
+    def visitExprBin(self, e : ExprBin):
+        e.lhs.accept(self)
+        e.rhs.accept(self)
+
+    def visitExprCall(self, e : ExprCall):
+        for arg in e.args:
+            arg.accept(self)
+
+    def visitExprString(self, e : ExprString):
+        pass
+
+    def visitExprInt(self, e : ExprInt):
+        pass
+
+@dc.dataclass
+class ExprVisitor2String(ExprVisitor):
+    _ret : str = ""
+    _idx : int = 0
+
+    @staticmethod
+    def toString(e : Expr):
+        v = ExprVisitor2String()
+        e.accept(v)
+        return v._ret
+
+    def visitExprId(self, e : ExprId):
+        self._ret += e.id
+    
+    def visitExprBin(self, e):
+        e.lhs.accept(self)
+        self._ret += " "
+        self._ret += "op%d" % self._idx
+        self._idx += 1
+        self._ret += " "
+        e.rhs.accept(self)
+
+    def visitExprCall(self, e):
+        self._ret += e.id
+        self._ret += "("
+        for i, arg in enumerate(e.args):
+            if i > 0:
+                self._ret += ", "
+            arg.accept(self)
+        self._ret += ")"
+    
+    def visitExprString(self, e):
+        self._ret += e.value
+
+    def visitExprInt(self, e):
+        self._ret += str(e.value)
+
+
+class ExprParser(object):
+
+    _inst : ClassVar['ExprParser'] = None
+
+    def __init__(self):
+
+        # Build the lexer
+        self.lexer = lex.lex(module=self)
+        self.parser = yacc.yacc(module=self)
+
+    @classmethod
+    def inst(cls):
+        if cls._inst is None:
+            cls._inst = ExprParser()
+        return cls._inst
+
+    tokens = (
+        'ID','NUMBER','COMMA',
+        'PLUS','MINUS','TIMES','DIVIDE',
+        'LPAREN','RPAREN','PIPE','STRING1','STRING2'
+        )
+    
+    # Tokens
+
+    t_COMMA   = r',' 
+    t_PLUS    = r'\+'
+    t_MINUS   = r'-'
+    t_TIMES   = r'\*'
+    t_DIVIDE  = r'/'
+    t_LPAREN  = r'\('
+    t_RPAREN  = r'\)'
+    t_ID      = r'[a-zA-Z_][a-zA-Z0-9_]*'
+    t_PIPE    = r'\|'
+    t_STRING2 = r'\'.*\''
+    
+    def t_NUMBER(self, t):
+        r'\d+'
+        try:
+            t.value = ExprInt(int(t.value)) 
+        except ValueError:
+            print("Integer value too large %d", t.value)
+            t.value = ExprInt(0)
+        return t
+    
+    def t_STRING1(self, t):
+        r'(\'|\")([^\\\n]|(\\.))*?(\'|\")'
+        t.value = t.value[1:-1].replace('\\"', '"').replace("\\'", "'").replace('\\n', '\n').replace('\\t', '\t').replace('\\\\', '\\')
+        return t
+    
+    # Ignored characters
+    t_ignore = " \t"
+    
+    def t_newline(self, t):
+        r'\n+'
+        t.lexer.lineno += t.value.count("\n")
+        
+    def t_error(self, t):
+        print("Illegal character '%s'" % t.value[0])
+        t.lexer.skip(1)
+        
+    precedence = (
+        ('left','PLUS','MINUS','PIPE'),
+        ('left','TIMES','DIVIDE'),
+#        ('right','UMINUS'),
+        )
+    
+    def p_call(self, t):
+        """expression : ID LPAREN RPAREN 
+                      | ID LPAREN args RPAREN"""
+        t[0] = ExprCall(t[1], t[3] if len(t) == 5 else [])
+
+    def p_args(self, t):
+        """args : expression 
+                | args COMMA expression"""
+        if len(t) == 2:
+            t[0] = [t[1]]
+        else:
+            t[0] = t[1]
+            t[0].append(t[3])
+    
+    def p_expression_binop(self, t):
+        '''expression : expression PLUS expression
+                      | expression MINUS expression
+                      | expression TIMES expression
+                      | expression PIPE expression
+                      | expression DIVIDE expression'''
+        op_m = {
+            '+' : ExprBinOp.Plus,
+            '-' : ExprBinOp.Minus,
+            '*' : ExprBinOp.Times,
+            '|' : ExprBinOp.Pipe,
+            '/' : ExprBinOp.Divide
+        }
+        t[0] = ExprBin(op_m[t[2]], t[1], t[3])
+    
+    def p_expression_group(self, t):
+        'expression : LPAREN expression RPAREN'
+        t[0] = t[2]
+    
+    def p_expression_number(self, t):
+        'expression : NUMBER'
+        t[0] = t[1]
+    
+    def p_expression_name(self, t):
+        'expression : ID'
+        t[0] = ExprId(t[1])
+
+    def p_expression_string1(self, t):
+        'expression : STRING1'
+        t[0] = ExprString(t[1])
+
+    def p_expression_string2(self, t):
+        'expression : STRING2'
+        t[0] = ExprString(t[1])
+    
+    def p_error(self, t):
+        print("Syntax error at '%s'" % t.value)
+
+    def parse(self, input):
+        return self.parser.parse(input, lexer=self.lexer)
+    

dv_flow/mgr/fragment_def.py

@@ -26,11 +26,13 @@ from typing import Any, Dict, List, Union
 from .package import Package
 from .package_import_spec import PackageImportSpec
 from .task_def import TaskDef
+from .type_def import TypeDef
 
 class FragmentDef(BaseModel):
     tasks : List[TaskDef] = dc.Field(default_factory=list)
     imports : List[Union[str,PackageImportSpec]] = dc.Field(default_factory=list, alias="imports")
     fragments: List[str] = dc.Field(default_factory=list)
+    types : List[TypeDef] = dc.Field(default_factory=list)
 
     basedir : str = None
 

dv_flow/mgr/out

@@ -0,0 +1,12 @@
+	   __main__.py
+	    flow.py
+	   package.py
+	   package_def.py
+	   package_import_spec.py
+	    parameters.py
+	   pkg_rgy.py
+	   task.py
+	   task_graph_builder.py
+	   task_graph_runner.py
+	   util.py
+

dv_flow/mgr/package.py

@@ -31,6 +31,7 @@ class Package(object):
     # Package holds constructors for tasks
     # - Dict holds the default parameters for the task
     tasks : Dict[str,TaskCtor] = dc.field(default_factory=dict)
+    types : Dict[str,Any] = dc.field(default_factory=dict)
     _log : ClassVar = logging.getLogger("Package")
 
     def getTaskCtor(self, name : str) -> TaskCtor:

dv_flow/mgr/package_def.py

@@ -19,6 +19,7 @@
 #*     Author: 
 #*
 #****************************************************************************
+import io
 import os
 import json
 import yaml
@@ -35,6 +36,7 @@ from .package_import_spec import PackageImportSpec, PackageSpec
 from .task import TaskCtor, TaskCtorProxy, TaskCtorCls, TaskCtorParam, TaskCtorParamCls
 from .task_def import TaskDef, TaskSpec
 from .std.task_null import TaskNull
+from .type_def import TypeDef
 
 
 class PackageDef(BaseModel):
@@ -44,6 +46,7 @@ class PackageDef(BaseModel):
     tasks : List[TaskDef] = dc.Field(default_factory=list)
     imports : List[PackageImportSpec] = dc.Field(default_factory=list)
     fragments: List[str] = dc.Field(default_factory=list)
+    types : List[TypeDef] = dc.Field(default_factory=list)
 
     fragment_l : List['FragmentDef'] = dc.Field(default_factory=list, exclude=True)
 
@@ -314,6 +317,32 @@ class PackageDef(BaseModel):
         file_s.pop()
 
         return pkg
+
+    @staticmethod
+    def loads(data, exp_pkg_name=None):
+        return PackageDef._loadPkgDefS(data, exp_pkg_name)
+        pass
+
+    @staticmethod
+    def _loadPkgDefS(data, exp_pkg_name):
+        ret = None
+        doc = yaml.load(io.StringIO(data), Loader=yaml.FullLoader)
+        if "package" not in doc.keys():
+            raise Exception("Missing 'package' key in %s" % root)
+        pkg = PackageDef(**(doc["package"]))
+        pkg.basedir = None
+
+#            for t in pkg.tasks:
+#                t.basedir = os.path.dirname(root)
+
+        if exp_pkg_name is not None:
+            if exp_pkg_name != pkg.name:
+                raise Exception("Package name mismatch: %s != %s" % (exp_pkg_name, pkg.name))
+
+        if len(pkg.fragments) > 0:
+            raise Exception("Cannot load a package-def with fragments from a string")
+
+        return pkg
     
     @staticmethod
     def _loadFragmentSpec(pkg, spec, file_s):

dv_flow/mgr/param_def.py

@@ -0,0 +1,25 @@
+
+from typing import Any, List, Union
+from pydantic import BaseModel, Field
+
+class ListType(BaseModel):
+    item : Union[str, 'ComplexType']
+
+class MapType(BaseModel):
+    key : Union[str, 'ComplexType']
+    item : Union[str, 'ComplexType']
+
+class ComplexType(BaseModel):
+    list : Union[ListType, None] = None
+    map : Union[MapType, None] = None
+
+class ParamDef(BaseModel):
+    name : str
+    doc : str = None
+    type : Union[str, 'ComplexType'] = None
+    value : Union[Any, None] = None
+    append : Union[Any, None] = None
+    prepend : Union[Any, None] = None
+    path_append : Union[Any, None] = Field(alias="path-append", default=None)
+    path_prepend : Union[Any, None] = Field(alias="path-prepend", default=None)
+

dv_flow/mgr/parsetab.py

@@ -0,0 +1,44 @@
+
+# parsetab.py
+# This file is automatically generated. Do not edit.
+# pylint: disable=W,C,R
+_tabversion = '3.10'
+
+_lr_method = 'LALR'
+
+_lr_signature = 'leftPLUSMINUSPIPEleftTIMESDIVIDECOMMA DIVIDE ID LPAREN MINUS NUMBER PIPE PLUS RPAREN STRING1 STRING2 TIMESexpression : ID LPAREN RPAREN \n                      | ID LPAREN args RPARENargs : expression \n                | args COMMA expressionexpression : expression PLUS expression\n                      | expression MINUS expression\n                      | expression TIMES expression\n                      | expression PIPE expression\n                      | expression DIVIDE expressionexpression : LPAREN expression RPARENexpression : NUMBERexpression : IDexpression : STRING1expression : STRING2'
+    
+_lr_action_items = {'ID':([0,3,7,8,9,10,11,12,24,],[2,2,2,2,2,2,2,2,2,]),'LPAREN':([0,2,3,7,8,9,10,11,12,24,],[3,12,3,3,3,3,3,3,3,3,]),'NUMBER':([0,3,7,8,9,10,11,12,24,],[4,4,4,4,4,4,4,4,4,]),'STRING1':([0,3,7,8,9,10,11,12,24,],[5,5,5,5,5,5,5,5,5,]),'STRING2':([0,3,7,8,9,10,11,12,24,],[6,6,6,6,6,6,6,6,6,]),'$end':([1,2,4,5,6,14,15,16,17,18,19,22,23,],[0,-12,-11,-13,-14,-5,-6,-7,-8,-9,-1,-10,-2,]),'PLUS':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[7,-12,-11,-13,-14,7,-5,-6,-7,-8,-9,-1,7,-10,-2,7,]),'MINUS':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[8,-12,-11,-13,-14,8,-5,-6,-7,-8,-9,-1,8,-10,-2,8,]),'TIMES':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[9,-12,-11,-13,-14,9,9,9,-7,9,-9,-1,9,-10,-2,9,]),'PIPE':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[10,-12,-11,-13,-14,10,-5,-6,-7,-8,-9,-1,10,-10,-2,10,]),'DIVIDE':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[11,-12,-11,-13,-14,11,11,11,-7,11,-9,-1,11,-10,-2,11,]),'RPAREN':([2,4,5,6,12,13,14,15,16,17,18,19,20,21,22,23,25,],[-12,-11,-13,-14,19,22,-5,-6,-7,-8,-9,-1,23,-3,-10,-2,-4,]),'COMMA':([2,4,5,6,14,15,16,17,18,19,20,21,22,23,25,],[-12,-11,-13,-14,-5,-6,-7,-8,-9,-1,24,-3,-10,-2,-4,]),}
+
+_lr_action = {}
+for _k, _v in _lr_action_items.items():
+   for _x,_y in zip(_v[0],_v[1]):
+      if not _x in _lr_action:  _lr_action[_x] = {}
+      _lr_action[_x][_k] = _y
+del _lr_action_items
+
+_lr_goto_items = {'expression':([0,3,7,8,9,10,11,12,24,],[1,13,14,15,16,17,18,21,25,]),'args':([12,],[20,]),}
+
+_lr_goto = {}
+for _k, _v in _lr_goto_items.items():
+   for _x, _y in zip(_v[0], _v[1]):
+       if not _x in _lr_goto: _lr_goto[_x] = {}
+       _lr_goto[_x][_k] = _y
+del _lr_goto_items
+_lr_productions = [
+  ("S' -> expression","S'",1,None,None,None),
+  ('expression -> ID LPAREN RPAREN','expression',3,'p_call','expr_parser.py',160),
+  ('expression -> ID LPAREN args RPAREN','expression',4,'p_call','expr_parser.py',161),
+  ('args -> expression','args',1,'p_args','expr_parser.py',165),
+  ('args -> args COMMA expression','args',3,'p_args','expr_parser.py',166),
+  ('expression -> expression PLUS expression','expression',3,'p_expression_binop','expr_parser.py',174),
+  ('expression -> expression MINUS expression','expression',3,'p_expression_binop','expr_parser.py',175),
+  ('expression -> expression TIMES expression','expression',3,'p_expression_binop','expr_parser.py',176),
+  ('expression -> expression PIPE expression','expression',3,'p_expression_binop','expr_parser.py',177),
+  ('expression -> expression DIVIDE expression','expression',3,'p_expression_binop','expr_parser.py',178),
+  ('expression -> LPAREN expression RPAREN','expression',3,'p_expression_group','expr_parser.py',189),
+  ('expression -> NUMBER','expression',1,'p_expression_number','expr_parser.py',193),
+  ('expression -> ID','expression',1,'p_expression_name','expr_parser.py',197),
+  ('expression -> STRING1','expression',1,'p_expression_string1','expr_parser.py',201),
+  ('expression -> STRING2','expression',1,'p_expression_string2','expr_parser.py',205),
+]

dv_flow/mgr/std/flow.dv

@@ -27,4 +27,52 @@ package:
 #  - name: Exec
 #    pyclass: dv_flow.mgr.std.exec.Exec
 #    with: {}
-      
+  types:
+  # - name: TaskDataItem
+  #   doc: |
+  #     Base type for a data element produced by a task.
+  #     TaskDataItem is never used directly
+  #   with:
+  #   - name: kind
+  #     type: str
+  #     kind: metadata
+  #     doc: |
+  #       Kind of data item
+  #   - name: src
+  #     type: str
+  #     kind: metadata
+  #     doc: |
+  #       Name of the task that produced this item
+  #   - name: id
+  #     type: str
+  #     kind: metadata
+  #     doc: |
+  #       Unique identiifer 
+
+  - name: Env
+    doc: |
+      Environment variables
+    with:
+    - name: doc
+
+  - name: FileSet
+    uses: TaskDataItem
+    with:
+    - name: kind
+      value: "FileSet"
+    - name: base
+      type: str
+    - name: fileType
+      type: str
+    - name: files
+      type: 
+        list:
+          item:
+            type: str
+
+    # type:
+    #   list: (str,int)
+    #     - complex type
+    # type:
+    #   object:
+    #     

dv_flow/mgr/task.py

@@ -25,7 +25,7 @@ import dataclasses as dc
 import logging
 from pydantic import BaseModel
 from typing import Any, Callable, ClassVar, Dict, List, Tuple
-from .task_data import TaskData
+from .task_data import TaskData, TaskDataInput, TaskDataOutput, TaskDataResult
 from .task_memento import TaskMemento
 
 @dc.dataclass
@@ -180,112 +180,21 @@ class TaskCtorProxy(TaskCtor):
 class Task(object):
     """Executable view of a task"""
     name : str
-    params : TaskParams
-    srcdir : str = None
-    session : 'TaskGraphRunner' = None
-    basedir : str = None
-    memento : TaskMemento = None
-    depends : List['Task'] = dc.field(default_factory=list)
-    output : Any = None
-
-    # Implementation data below
-    basedir : str = dc.field(default=None)
-    rundir : str = dc.field(default=None)
-    impl : str = None
-    body: Dict[str,Any] = dc.field(default_factory=dict)
-    impl_t : Any = None
-
-    _log : ClassVar = logging.getLogger("Task")
-
-    def init(self, runner, basedir):
-        self.session = runner
-        self.basedir = basedir
-
-    def getMemento(self, T) -> TaskMemento:
-        if os.path.isfile(os.path.join(self.rundir, "memento.json")):
-            with open(os.path.join(self.rundir, "memento.json"), "r") as fp:
-                try:
-                    data = json.load(fp)
-                    self.memento = T(**data)
-                except Exception as e:
-                    self._log.critical("Failed to load memento %s: %s" % (
-                        os.path.join(self.rundir, "memento.json"), str(e)))
-                    os.unlink(os.path.join(self.rundir, "memento.json"))
-        return self.memento
-
-    def setMemento(self, memento : TaskMemento):
-        self.memento = memento
-
-    async def isUpToDate(self, memento) -> bool:
-        return False
-
-    async def do_run(self, session) -> TaskData:
-        self._log.info("--> %s (%s) do_run - %d depends" % (
-            self.name, 
-            str(type(self)),
-            len(self.depends)))
-
-        self.session = session
-
-        if len(self.depends) > 0:
-            deps_o = []
-            for d in self.depends:
-                dep_o = d.getOutput()
-                if dep_o is None:
-                    raise Exception("Null output for %s" % d.name)
-                deps_o.append(dep_o)
-
-            input = TaskData.merge(deps_o)
-            input.src = self.name
-            input.deps[self.name] = list(inp.name for inp in self.depends)
-        else:
-            input = TaskData()
-
-
-        # Mark the source of this data as being this task
-        input.src = self.name
-
-        self.init_rundir()
 
-        self.output = await self.run(input)
+    # Need TaskDef 
 
-        if self.output is None:
-            raise Exception("No output produced by %s" % self.name)
-            result = TaskData()
-
-        # Write-back the memento, if specified
-        self.save_memento()
+    params : TaskParams # Might need to be set of instructions for building...
+    # This needs to be 'inside out' -- listed bottom to top
+    srcdir : str = dc.field(default=None)
+    rundir : str = dc.field(default=None)
+    depends    : List['Task'] = dc.field(default_factory=list)
+    dependents : List['Task'] = dc.field(default_factory=list)
+    output : TaskDataOutput = dc.field(default=None)
 
-        # Combine data from the deps to produce a result
-        self._log.info("<-- %s (%s) do_run - %d depends" % (
-            self.name, 
-            str(type(self)),
-            len(self.depends)))
-        return self.output
+    _log : ClassVar = logging.getLogger("Task")
 
-    async def run(self, input : TaskData) -> TaskData:
+    async def run(self, 
+                  input : TaskDataInput,
+                  runner : 'TaskGraphRunner') -> TaskDataResult:
         raise NotImplementedError("TaskImpl.run() not implemented")
-    
-    def init_rundir(self):
-        if not os.path.isdir(self.rundir):
-            os.makedirs(self.rundir)
-
-    def save_memento(self):
-        if self.memento is not None:
-            with open(os.path.join(self.rundir, "memento.json"), "w") as fp:
-                fp.write(self.memento.model_dump_json(indent=2))
-    
-    def getOutput(self) -> TaskData:
-        return self.output
-
-    def getField(self, name : str) -> Any:
-        if name in self.__dict__.keys():
-            return self.__dict__[name]
-        elif name in self.__pydantic_extra__.keys():
-            return self.__pydantic_extra__[name]
-        else:
-            raise Exception("No such field %s" % name)
-        
-
-
 

dv_flow/mgr/task_data.py

@@ -27,6 +27,24 @@ from typing import Any, ClassVar, Dict, Set, List, Tuple
 from .fileset import FileSet
 from toposort import toposort
 
+class TaskDataInput(BaseModel):
+    changed : bool
+    srcdir : str
+    rundir : str
+    params : Any
+    memento : Any
+
+class TaskDataResult(BaseModel):
+    changed : bool
+    output : List[Any]
+    memento : Any
+    markers : List[Any]
+    status : int
+
+class TaskDataOutput(BaseModel):
+    changed : bool
+    output : List[Any]
+
 class TaskDataParamOpE(enum.Enum):
     Set = enum.auto()
     Append = enum.auto()
@@ -48,26 +66,22 @@ class TaskDataParam(BaseModel):
     kind : TaskDataParamKindE
     ops : List[TaskDataParamOp] = dc.Field(default_factory=list)
 
+class TaskDataItem(BaseModel):
+    type : str
+    src : str
+    id : str
+
 class TaskData(BaseModel):
     src : str = None
-    params : Dict[str,TaskDataParam] = dc.Field(default_factory=dict)
     deps : Dict[str,Set[str]] = dc.Field(default_factory=dict)
-    filesets : List[FileSet] = dc.Field(default_factory=list)
     changed : bool = False
+    data : List[TaskDataItem] = dc.Field(default_factory=list)
 
     _log : ClassVar = logging.getLogger("TaskData")
 
-    def hasParam(self, name: str) -> bool:
-        return name in self.params
-    
-    def getParam(self, name: str) -> Any:
-        return self.params[name]
-    
-    def setParam(self, name: str, value: Any):
-        self.params[name] = value
-
-    def addFileSet(self, fs : FileSet):
-        self.filesets.append(fs)
+    def addData(self, item : TaskDataItem):
+        item.src = self.src
+        self.data.append(item)
 
     def getFileSets(self, type=None, order=True) -> List[FileSet]:
         ret = []
@@ -77,8 +91,10 @@ class TaskData(BaseModel):
         if order:
             # The deps map specifies task dependencies
 
+            filesets = filter(lambda x: x.type == "std.FileSet", self.data)
+
             candidate_fs = []
-            for fs in self.filesets:
+            for fs in filesets:
                 self._log.debug("fs: %s" % str(fs))
                 if type is None or fs.type in type:
                     candidate_fs.append(fs)

dv_flow/mgr/task_def.py

@@ -22,7 +22,9 @@
 import pydantic.dataclasses as dc
 from pydantic import BaseModel
 from typing import Any, Dict, List, Union, Tuple
+from .param_def import ParamDef
 from .task import Task
+from .task_output import TaskOutput
 
 @dc.dataclass
 class TaskSpec(object):
@@ -37,7 +39,8 @@ class TaskDef(BaseModel):
     desc : str = dc.Field(default="")
     doc : str = dc.Field(default="")
     depends : List[Union[str,TaskSpec]] = dc.Field(default_factory=list, alias="needs")
-    params: Dict[str,Any] = dc.Field(default_factory=dict, alias="with")
+    params: List[ParamDef] = dc.Field(default_factory=list, alias="with")
+    out: List[TaskOutput] = dc.Field(default_factory=list)
 
     def copy(self) -> 'TaskDef':
         ret = TaskDef(

dv_flow/mgr/task_exec_data.py

@@ -0,0 +1,22 @@
+import pydantic.dataclasses as dc
+from pydantic import BaseModel
+from typing import Any, Dict, List
+
+
+class TaskExecData(BaseModel):
+    """Data from a single exection of a task"""
+    name : str
+    start : str
+    finish : str
+    status : int
+    memento : Any
+    markers : List[Any]
+
+class FlowExecData(BaseModel):
+    """
+    Data from multiple tasks executions. 'info' holds information
+    across multiple flow invocations. 'tasks' holds the names of
+    tasks executed in the most-recent invocation.
+    """
+    info : Dict[str, TaskExecData] = dc.Field(default_factory=dict)
+    tasks : List[str] = dc.Field(default_factory=list)

dv_flow/mgr/task_impl_data.py

@@ -0,0 +1,17 @@
+
+from pydantic import BaseModel
+from typing import Any, ClassVar, Dict, Set, List, Tuple
+
+class TaskImplParams(BaseModel):
+    pass
+
+class TaskImplSourceData(BaseModel):
+    params : Any
+    changed : bool
+    memento : Any
+
+class TaskImplResultData(BaseModel):
+    data : List[Any]
+    changed : bool
+    memento : Any
+

dv_flow/mgr/task_output.py

@@ -0,0 +1,6 @@
+from pydantic import BaseModel, Field
+from typing import Any, List
+
+class TaskOutput(BaseModel):
+    type : str
+    params : List[Any] = Field(default_factory=list, alias="with")

dv_flow/mgr/type_def.py

@@ -0,0 +1,10 @@
+
+from typing import List, Union
+from pydantic import BaseModel, Field
+from .param_def import ParamDef
+
+class TypeDef(BaseModel):
+    name : str
+    uses : str = None
+    doc : str = None
+    fields : List[ParamDef] = Field(alias="with", default_factory=list)

{dv_flow_mgr-0.0.1.12992722023a1.dist-info → dv_flow_mgr-0.0.1.13468776985a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12992722023a1
+Version: 0.0.1.13468776985a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

dv_flow_mgr-0.0.1.13468776985a1.dist-info/RECORD

@@ -0,0 +1,38 @@
+dv_flow/mgr/__init__.py,sha256=IZA7I1u7RH34DrJXSkETxWFpK5Jn_A2zXfnCAzJ8XxA,289
+dv_flow/mgr/__main__.py,sha256=uik6gDAvtZNG0yyDKYc4FTl6R1QHAR543jNG2lCAa6E,1771
+dv_flow/mgr/expr_eval.py,sha256=pfnTDurXPTURgpdzlno5XtXlkoPfnJmAAq45G9T2D3k,1517
+dv_flow/mgr/expr_parser.py,sha256=aR8n6hgk5xmFknIaSVC02K-So-eU4pkkFtwb7zWsSPs,5474
+dv_flow/mgr/fileset.py,sha256=FNvC5sU2ArxJ0OO3v8dXTv8zX-bZ5t0a0ljne0fQQ1o,1150
+dv_flow/mgr/fragment_def.py,sha256=cyzp1XeWtNOaagScmeS-BPsoXj9j2LTBbKq5ZUioz8I,1641
+dv_flow/mgr/out,sha256=d8GGBi3J43fhdLBlnsUbzBfRe0TD0QTP3nOTz54l2bI,200
+dv_flow/mgr/package.py,sha256=MDnt9q4pD_Aex_hSK_EUQ38IiaxOihgxbMmk5dH1deQ,1674
+dv_flow/mgr/package_def.py,sha256=Sv3KS5pDysUrtPBWElCvmWt2pnkTipZcZGYwKr5N7Mc,14967
+dv_flow/mgr/package_import_spec.py,sha256=ah3r15v5Jdub2poc3sgi6Uar1L3oGoYsCPPNiOHV-a4,1760
+dv_flow/mgr/param_def.py,sha256=e2WvRCMArbgcsKC4TKQqZTscZmCdo3WvVR6w3LN0VK8,727
+dv_flow/mgr/parsetab.py,sha256=enSOnMQ-woIsMEzHyeYiefvhAl8enxfX9Ct_o8-jkqs,3780
+dv_flow/mgr/pkg_rgy.py,sha256=2R_EaeBDJn5qUq9DzSnLc37wUP36MWSv-p0LgUjJRAg,4471
+dv_flow/mgr/task.py,sha256=jZrxTevSulUQ56RdgD0TQXrQ05859FgmXahOdxhdyT8,6349
+dv_flow/mgr/task_data.py,sha256=TCc5KYuKoQNYfemVOzIlNKkMKIw-cqVl5_EmDs03W7M,10718
+dv_flow/mgr/task_def.py,sha256=6qDDgge5RXssL5TAZR01dGb_Vjvbj72tJirg66O-Yt4,1818
+dv_flow/mgr/task_exec_data.py,sha256=aT__kmVmNxqnS_GbTRS1vrYgKiv8cH-HUSmRb6YYKsI,640
+dv_flow/mgr/task_graph_builder.py,sha256=4dfy_T26MYI7ls2xiofr4V0ItZHDehU2iw6UBPt-SZQ,7074
+dv_flow/mgr/task_graph_runner.py,sha256=NwNYcOJ952lPMLwIIlYE9CoDdedqvcw2fWHYUsKFXuU,2164
+dv_flow/mgr/task_graph_runner_local.py,sha256=66JYJcu-W60LjfuT9UkbcKslNqDcD5q4UR7ZAWrSaGM,4707
+dv_flow/mgr/task_impl_data.py,sha256=bFPijoKrh9x7fZN2DsvRJp0UHo-gGM0VjtDQISyfhFk,321
+dv_flow/mgr/task_memento.py,sha256=C7VTQpBhDEoYuDmE6YTM-6TLMLnqHp6Y0Vat1aTgtCs,1096
+dv_flow/mgr/task_output.py,sha256=l-W-FvVo6YDah1RQS-I9N0KUtB3vp-kl7lxIdmNz0l4,178
+dv_flow/mgr/type.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dv_flow/mgr/type_def.py,sha256=KdhuNlfw-NKU-4VZFCnMPyj775yEB7cpr5tz73a9yuQ,259
+dv_flow/mgr/util.py,sha256=06eVyURF4ga-s8C9Sd3ZSDebwO4QS0XXaB8xADVbWRc,1437
+dv_flow/mgr/cmds/cmd_run.py,sha256=eths8kT7mBmpZqwOuMtpKAaux4rg-f7hPBxxTHbpKT4,2903
+dv_flow/mgr/share/flow.json,sha256=lNmZex9NXkYbyb2aZseQfUOkV9CMyfH0iLODEI7EPBw,5096
+dv_flow/mgr/std/fileset.py,sha256=uP7bGntRq-Tn5_GEFnt0_J_OAmfvep3GlCwCuE8by4o,2710
+dv_flow/mgr/std/flow.dv,sha256=j9wLrF3Ghh1ZLbJxmk7WiNiRYUYEer-8CCUA5hsgtfk,1409
+dv_flow/mgr/std/message.py,sha256=BPTHnEMD4tBufQ9LvsS9Sa_0xjaJATbBpwqosWslvVA,193
+dv_flow/mgr/std/task_null.py,sha256=KObmjG_4D08GJ1k6neqKIQrFY72Sj0jLnwXxEkq5HA0,321
+dv_flow_mgr-0.0.1.13468776985a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-0.0.1.13468776985a1.dist-info/METADATA,sha256=C2BOmVhhEQfwS9QQ9reuEWFsP9uWupVItzxnL5lilJc,13276
+dv_flow_mgr-0.0.1.13468776985a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+dv_flow_mgr-0.0.1.13468776985a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
+dv_flow_mgr-0.0.1.13468776985a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
+dv_flow_mgr-0.0.1.13468776985a1.dist-info/RECORD,,

dv_flow_mgr-0.0.1.12992722023a1.dist-info/RECORD

@@ -1,28 +0,0 @@
-dv_flow/mgr/__init__.py,sha256=IZA7I1u7RH34DrJXSkETxWFpK5Jn_A2zXfnCAzJ8XxA,289
-dv_flow/mgr/__main__.py,sha256=uik6gDAvtZNG0yyDKYc4FTl6R1QHAR543jNG2lCAa6E,1771
-dv_flow/mgr/fileset.py,sha256=FNvC5sU2ArxJ0OO3v8dXTv8zX-bZ5t0a0ljne0fQQ1o,1150
-dv_flow/mgr/fragment_def.py,sha256=p5i6ONtBWlDHTBFsduu3Z36_76Bn8PCIylp_xoZ7jfQ,1552
-dv_flow/mgr/package.py,sha256=AOLEEotVQF9VNMMl13uNQ7Na2TuHPXxEvybBOX7XIps,1615
-dv_flow/mgr/package_def.py,sha256=iHBFV-CKyCypvrd31zm3Yjkcib9xZo1gFlFL7YMfJO8,14012
-dv_flow/mgr/package_import_spec.py,sha256=ah3r15v5Jdub2poc3sgi6Uar1L3oGoYsCPPNiOHV-a4,1760
-dv_flow/mgr/pkg_rgy.py,sha256=2R_EaeBDJn5qUq9DzSnLc37wUP36MWSv-p0LgUjJRAg,4471
-dv_flow/mgr/task.py,sha256=giy5elwIS_fab6wGnA8JCeZIEeX0nG5zpcq5ifGYSOs,9087
-dv_flow/mgr/task_data.py,sha256=XBPWwvuaQ3BZ94wknvv1bqLh98iTzAg5fskv7_kk6DQ,10524
-dv_flow/mgr/task_def.py,sha256=96hSwqJo0MazJ1VcLhovYRmNCplsNLt47AumtyjSddU,1690
-dv_flow/mgr/task_graph_builder.py,sha256=4dfy_T26MYI7ls2xiofr4V0ItZHDehU2iw6UBPt-SZQ,7074
-dv_flow/mgr/task_graph_runner.py,sha256=NwNYcOJ952lPMLwIIlYE9CoDdedqvcw2fWHYUsKFXuU,2164
-dv_flow/mgr/task_graph_runner_local.py,sha256=66JYJcu-W60LjfuT9UkbcKslNqDcD5q4UR7ZAWrSaGM,4707
-dv_flow/mgr/task_memento.py,sha256=C7VTQpBhDEoYuDmE6YTM-6TLMLnqHp6Y0Vat1aTgtCs,1096
-dv_flow/mgr/util.py,sha256=06eVyURF4ga-s8C9Sd3ZSDebwO4QS0XXaB8xADVbWRc,1437
-dv_flow/mgr/cmds/cmd_run.py,sha256=eths8kT7mBmpZqwOuMtpKAaux4rg-f7hPBxxTHbpKT4,2903
-dv_flow/mgr/share/flow.json,sha256=lNmZex9NXkYbyb2aZseQfUOkV9CMyfH0iLODEI7EPBw,5096
-dv_flow/mgr/std/fileset.py,sha256=uP7bGntRq-Tn5_GEFnt0_J_OAmfvep3GlCwCuE8by4o,2710
-dv_flow/mgr/std/flow.dv,sha256=pSpzrPPEu_L8DHccGfArxsKYgUfyQidShZc0ShgGtsY,500
-dv_flow/mgr/std/message.py,sha256=BPTHnEMD4tBufQ9LvsS9Sa_0xjaJATbBpwqosWslvVA,193
-dv_flow/mgr/std/task_null.py,sha256=KObmjG_4D08GJ1k6neqKIQrFY72Sj0jLnwXxEkq5HA0,321
-dv_flow_mgr-0.0.1.12992722023a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dv_flow_mgr-0.0.1.12992722023a1.dist-info/METADATA,sha256=kCTd3eqAjQmtiL2_rnfPvCERKOp4I3aRl3yusyz4xMw,13276
-dv_flow_mgr-0.0.1.12992722023a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-dv_flow_mgr-0.0.1.12992722023a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
-dv_flow_mgr-0.0.1.12992722023a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
-dv_flow_mgr-0.0.1.12992722023a1.dist-info/RECORD,,