dv-flow-mgr 0.0.1.13335470340a1__py3-none-any.whl → 0.0.1.13478128278a1__py3-none-any.whl

dv_flow/mgr/expr_eval.py

@@ -0,0 +1,50 @@
+
+import dataclasses as dc
+from typing import Any, Callable, Dict, List
+from .expr_parser import ExprVisitor, Expr, ExprBin, ExprBinOp, ExprCall, ExprId, ExprString, ExprInt
+
+@dc.dataclass
+class ExprEval(ExprVisitor):
+    methods : Dict[str, Callable] = dc.field(default_factory=dict)
+    variables : Dict[str, object] = dc.field(default_factory=dict)
+    value : Any = None
+
+    def eval(self, e : Expr):
+        self.value = None
+        e.accept(self)
+        return self.value
+
+    def visitExprId(self, e : ExprId):
+        if e.id in self.variables:
+            self.value = self.variables[e.id]
+        else:
+            raise Exception("Variable %s not found" % e.id)
+
+    def visitExprString(self, e : ExprString):
+        self.value = e.value
+    
+    def visitExprBin(self, e):
+        e.lhs.accept(self)
+
+        if e.op == ExprBinOp.Pipe:
+            # Value just goes over to the rhs
+            e.rhs.accept(self)
+        elif e.op == ExprBinOp.Plus:
+            pass
+    
+    def visitExprCall(self, e : ExprCall):
+        if e.id in self.methods:
+            # Need to gather up argument values
+            in_value = self.value
+            args = []
+            for arg in e.args:
+                self.value = None
+                arg.accept(self)
+                args.append(self.value)
+
+            self.value = self.methods[e.id](in_value, args)
+        else:
+            raise Exception("Method %s not found" % e.id)
+        
+    def visitExprInt(self, e : ExprInt):
+        self.value = e.value

dv_flow/mgr/expr_parser.py

@@ -1,38 +1,233 @@
 import dataclasses as dc
 import enum
-
-class TokenKind(enum.Enum):
-    ID = enum.auto()
+import ply.lex as lex
+import ply.yacc as yacc
+from typing import ClassVar, List
 
 @dc.dataclass
 class Expr(object):
-    kind : TokenKind = TokenKind.ID
+    def accept(self, v):
+        raise NotImplementedError("Expr.accept for %s" % str(type(sepf)))
 
+@dc.dataclass
 class ExprId(Expr):
     id : str
 
-    def __post_init__(self):
-        self.kind = TokenKind.ID
+    def accept(self, v):
+        v.visitExprId(self)
+
+class ExprBinOp(enum.Enum):
+    Pipe = enum.auto()
+    Plus = enum.auto()
+    Minus = enum.auto()
+    Times = enum.auto()
+    Divide = enum.auto()
 
-class ExprPipe(Expr):
+@dc.dataclass
+class ExprBin(Expr):
+    op : ExprBinOp
     lhs : Expr
     rhs : Expr
 
-    def __post_init__(self):
-        self.kind = TokenKind.ID
+    def accept(self, v):
+        v.visitExprBin(self)
+
+@dc.dataclass
+class ExprCall(Expr):
+    id : str
+    args : List[Expr]
+
+    def accept(self, v):
+        v.visitExprCall(self)
+
+@dc.dataclass
+class ExprString(Expr):
+    value : str
+
+    def accept(self, v):
+        v.visitExprString(self)
 
+@dc.dataclass
+class ExprInt(Expr):
+    value : int
+
+    def accept(self, v):
+        v.visitExprInt(self)
+
+class ExprVisitor(object):
+    def visitExprId(self, e : ExprId):
+        pass
+
+    def visitExprBin(self, e : ExprBin):
+        e.lhs.accept(self)
+        e.rhs.accept(self)
+
+    def visitExprCall(self, e : ExprCall):
+        for arg in e.args:
+            arg.accept(self)
+
+    def visitExprString(self, e : ExprString):
+        pass
+
+    def visitExprInt(self, e : ExprInt):
+        pass
+
+@dc.dataclass
+class ExprVisitor2String(ExprVisitor):
+    _ret : str = ""
+    _idx : int = 0
+
+    @staticmethod
+    def toString(e : Expr):
+        v = ExprVisitor2String()
+        e.accept(v)
+        return v._ret
+
+    def visitExprId(self, e : ExprId):
+        self._ret += e.id
+    
+    def visitExprBin(self, e):
+        e.lhs.accept(self)
+        self._ret += " "
+        self._ret += "op%d" % self._idx
+        self._idx += 1
+        self._ret += " "
+        e.rhs.accept(self)
+
+    def visitExprCall(self, e):
+        self._ret += e.id
+        self._ret += "("
+        for i, arg in enumerate(e.args):
+            if i > 0:
+                self._ret += ", "
+            arg.accept(self)
+        self._ret += ")"
+    
+    def visitExprString(self, e):
+        self._ret += e.value
+
+    def visitExprInt(self, e):
+        self._ret += str(e.value)
 
 
 class ExprParser(object):
 
-    def __init__(self, input):
-        self.input = input
+    _inst : ClassVar['ExprParser'] = None
+
+    def __init__(self):
+
+        # Build the lexer
+        self.lexer = lex.lex(module=self)
+        self.parser = yacc.yacc(module=self)
+
+    @classmethod
+    def inst(cls):
+        if cls._inst is None:
+            cls._inst = ExprParser()
+        return cls._inst
+
+    tokens = (
+        'ID','NUMBER','COMMA',
+        'PLUS','MINUS','TIMES','DIVIDE',
+        'LPAREN','RPAREN','PIPE','STRING1','STRING2'
+        )
+    
+    # Tokens
+
+    t_COMMA   = r',' 
+    t_PLUS    = r'\+'
+    t_MINUS   = r'-'
+    t_TIMES   = r'\*'
+    t_DIVIDE  = r'/'
+    t_LPAREN  = r'\('
+    t_RPAREN  = r'\)'
+    t_ID      = r'[a-zA-Z_][a-zA-Z0-9_]*'
+    t_PIPE    = r'\|'
+    t_STRING2 = r'\'.*\''
+    
+    def t_NUMBER(self, t):
+        r'\d+'
+        try:
+            t.value = ExprInt(int(t.value)) 
+        except ValueError:
+            print("Integer value too large %d", t.value)
+            t.value = ExprInt(0)
+        return t
+    
+    def t_STRING1(self, t):
+        r'(\'|\")([^\\\n]|(\\.))*?(\'|\")'
+        t.value = t.value[1:-1].replace('\\"', '"').replace("\\'", "'").replace('\\n', '\n').replace('\\t', '\t').replace('\\\\', '\\')
+        return t
+    
+    # Ignored characters
+    t_ignore = " \t"
+    
+    def t_newline(self, t):
+        r'\n+'
+        t.lexer.lineno += t.value.count("\n")
+        
+    def t_error(self, t):
+        print("Illegal character '%s'" % t.value[0])
+        t.lexer.skip(1)
+        
+    precedence = (
+        ('left','PLUS','MINUS','PIPE'),
+        ('left','TIMES','DIVIDE'),
+#        ('right','UMINUS'),
+        )
+    
+    def p_call(self, t):
+        """expression : ID LPAREN RPAREN 
+                      | ID LPAREN args RPAREN"""
+        t[0] = ExprCall(t[1], t[3] if len(t) == 5 else [])
+
+    def p_args(self, t):
+        """args : expression 
+                | args COMMA expression"""
+        if len(t) == 2:
+            t[0] = [t[1]]
+        else:
+            t[0] = t[1]
+            t[0].append(t[3])
+    
+    def p_expression_binop(self, t):
+        '''expression : expression PLUS expression
+                      | expression MINUS expression
+                      | expression TIMES expression
+                      | expression PIPE expression
+                      | expression DIVIDE expression'''
+        op_m = {
+            '+' : ExprBinOp.Plus,
+            '-' : ExprBinOp.Minus,
+            '*' : ExprBinOp.Times,
+            '|' : ExprBinOp.Pipe,
+            '/' : ExprBinOp.Divide
+        }
+        t[0] = ExprBin(op_m[t[2]], t[1], t[3])
+    
+    def p_expression_group(self, t):
+        'expression : LPAREN expression RPAREN'
+        t[0] = t[2]
+    
+    def p_expression_number(self, t):
+        'expression : NUMBER'
+        t[0] = t[1]
+    
+    def p_expression_name(self, t):
+        'expression : ID'
+        t[0] = ExprId(t[1])
+
+    def p_expression_string1(self, t):
+        'expression : STRING1'
+        t[0] = ExprString(t[1])
 
-    def __iter__(self):
-        return self
+    def p_expression_string2(self, t):
+        'expression : STRING2'
+        t[0] = ExprString(t[1])
     
-    def __next__(self):
+    def p_error(self, t):
+        print("Syntax error at '%s'" % t.value)
 
-        self.expr = expr
-        self.tokens = []
-        self.pos = 0
+    def parse(self, input):
+        return self.parser.parse(input, lexer=self.lexer)
+    

dv_flow/mgr/package.py

@@ -22,7 +22,7 @@
 import dataclasses as dc
 import logging
 from typing import Any, ClassVar, Dict
-from .task import TaskCtor
+from .task_ctor import TaskCtor
 
 @dc.dataclass
 class Package(object):

dv_flow/mgr/package_def.py

@@ -33,7 +33,8 @@ from typing import Any, Dict, List, Callable, Tuple, ClassVar
 from .fragment_def import FragmentDef
 from .package import Package
 from .package_import_spec import PackageImportSpec, PackageSpec
-from .task import TaskCtor, TaskCtorProxy, TaskCtorCls, TaskCtorParam, TaskCtorParamCls
+#from .task import TaskCtorCls, TaskCtorParam, TaskCtorParamCls
+from .task_ctor import TaskCtor
 from .task_def import TaskDef, TaskSpec
 from .std.task_null import TaskNull
 from .type_def import TypeDef

dv_flow/mgr/parsetab.py

@@ -0,0 +1,44 @@
+
+# parsetab.py
+# This file is automatically generated. Do not edit.
+# pylint: disable=W,C,R
+_tabversion = '3.10'
+
+_lr_method = 'LALR'
+
+_lr_signature = 'leftPLUSMINUSPIPEleftTIMESDIVIDECOMMA DIVIDE ID LPAREN MINUS NUMBER PIPE PLUS RPAREN STRING1 STRING2 TIMESexpression : ID LPAREN RPAREN \n                      | ID LPAREN args RPARENargs : expression \n                | args COMMA expressionexpression : expression PLUS expression\n                      | expression MINUS expression\n                      | expression TIMES expression\n                      | expression PIPE expression\n                      | expression DIVIDE expressionexpression : LPAREN expression RPARENexpression : NUMBERexpression : IDexpression : STRING1expression : STRING2'
+    
+_lr_action_items = {'ID':([0,3,7,8,9,10,11,12,24,],[2,2,2,2,2,2,2,2,2,]),'LPAREN':([0,2,3,7,8,9,10,11,12,24,],[3,12,3,3,3,3,3,3,3,3,]),'NUMBER':([0,3,7,8,9,10,11,12,24,],[4,4,4,4,4,4,4,4,4,]),'STRING1':([0,3,7,8,9,10,11,12,24,],[5,5,5,5,5,5,5,5,5,]),'STRING2':([0,3,7,8,9,10,11,12,24,],[6,6,6,6,6,6,6,6,6,]),'$end':([1,2,4,5,6,14,15,16,17,18,19,22,23,],[0,-12,-11,-13,-14,-5,-6,-7,-8,-9,-1,-10,-2,]),'PLUS':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[7,-12,-11,-13,-14,7,-5,-6,-7,-8,-9,-1,7,-10,-2,7,]),'MINUS':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[8,-12,-11,-13,-14,8,-5,-6,-7,-8,-9,-1,8,-10,-2,8,]),'TIMES':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[9,-12,-11,-13,-14,9,9,9,-7,9,-9,-1,9,-10,-2,9,]),'PIPE':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[10,-12,-11,-13,-14,10,-5,-6,-7,-8,-9,-1,10,-10,-2,10,]),'DIVIDE':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[11,-12,-11,-13,-14,11,11,11,-7,11,-9,-1,11,-10,-2,11,]),'RPAREN':([2,4,5,6,12,13,14,15,16,17,18,19,20,21,22,23,25,],[-12,-11,-13,-14,19,22,-5,-6,-7,-8,-9,-1,23,-3,-10,-2,-4,]),'COMMA':([2,4,5,6,14,15,16,17,18,19,20,21,22,23,25,],[-12,-11,-13,-14,-5,-6,-7,-8,-9,-1,24,-3,-10,-2,-4,]),}
+
+_lr_action = {}
+for _k, _v in _lr_action_items.items():
+   for _x,_y in zip(_v[0],_v[1]):
+      if not _x in _lr_action:  _lr_action[_x] = {}
+      _lr_action[_x][_k] = _y
+del _lr_action_items
+
+_lr_goto_items = {'expression':([0,3,7,8,9,10,11,12,24,],[1,13,14,15,16,17,18,21,25,]),'args':([12,],[20,]),}
+
+_lr_goto = {}
+for _k, _v in _lr_goto_items.items():
+   for _x, _y in zip(_v[0], _v[1]):
+       if not _x in _lr_goto: _lr_goto[_x] = {}
+       _lr_goto[_x][_k] = _y
+del _lr_goto_items
+_lr_productions = [
+  ("S' -> expression","S'",1,None,None,None),
+  ('expression -> ID LPAREN RPAREN','expression',3,'p_call','expr_parser.py',160),
+  ('expression -> ID LPAREN args RPAREN','expression',4,'p_call','expr_parser.py',161),
+  ('args -> expression','args',1,'p_args','expr_parser.py',165),
+  ('args -> args COMMA expression','args',3,'p_args','expr_parser.py',166),
+  ('expression -> expression PLUS expression','expression',3,'p_expression_binop','expr_parser.py',174),
+  ('expression -> expression MINUS expression','expression',3,'p_expression_binop','expr_parser.py',175),
+  ('expression -> expression TIMES expression','expression',3,'p_expression_binop','expr_parser.py',176),
+  ('expression -> expression PIPE expression','expression',3,'p_expression_binop','expr_parser.py',177),
+  ('expression -> expression DIVIDE expression','expression',3,'p_expression_binop','expr_parser.py',178),
+  ('expression -> LPAREN expression RPAREN','expression',3,'p_expression_group','expr_parser.py',189),
+  ('expression -> NUMBER','expression',1,'p_expression_number','expr_parser.py',193),
+  ('expression -> ID','expression',1,'p_expression_name','expr_parser.py',197),
+  ('expression -> STRING1','expression',1,'p_expression_string1','expr_parser.py',201),
+  ('expression -> STRING2','expression',1,'p_expression_string2','expr_parser.py',205),
+]

dv_flow/mgr/task.py

@@ -25,267 +25,158 @@ import dataclasses as dc
 import logging
 from pydantic import BaseModel
 from typing import Any, Callable, ClassVar, Dict, List, Tuple
-from .task_data import TaskData
+from .task_data import TaskData, TaskDataInput, TaskDataOutput, TaskDataResult
 from .task_memento import TaskMemento
+from .task_ctor import TaskParamsCtor
+from .task_params_ctor import TaskParamsCtor
+from .task_runner import TaskRunner
 
-@dc.dataclass
-class TaskSpec(object):
-    name : str
+# @dc.dataclass
+# class TaskSpec(object):
+#     name : str
 
-class TaskParams(BaseModel):
-    pass
+# class TaskParams(BaseModel):
+#     pass
 
 
-@dc.dataclass
-class TaskCtor(object):
-    name : str
-    uses : 'TaskCtor' = None
-    srcdir : str = None
-    depends : List[TaskSpec] = dc.field(default_factory=list)
-
-    _log : ClassVar = logging.getLogger("TaskCtor")
-
-    def mkTask(self, name : str, depends, rundir, srcdir=None, params=None):
-        if srcdir is None:
-            srcdir = self.srcdir
-        if params is None:
-            params = self.mkParams()
-
-        if self.uses is not None:
-            return self.uses.mkTask(name, depends, rundir, srcdir, params)
-        else:
-            raise NotImplementedError("TaskCtor.mkTask() not implemented for %s" % str(type(self)))
-    
-    def mkParams(self):
-        self._log.debug("--> %s::mkParams" % self.name)
-        if self.uses is not None:
-            params = self.uses.mkParams()
-        else:
-            params = TaskParams()
-        self._log.debug("<-- %s::mkParams: %s" % (self.name, str(params)))
+# @dc.dataclass
+# class TaskCtorParam(TaskCtor):
+#     params : Dict[str,Any] = dc.field(default_factory=dict)
 
-        return params
+#     _log : ClassVar = logging.getLogger("TaskCtorParam")
 
-    def applyParams(self, params):
-        if self.uses is not None:
-            self.uses.applyParams(params)
+#     def mkTask(self, name : str, depends, rundir, srcdir=None, params=None):
+#         self._log.debug("--> %s::mkTask" % self.name)
+#         if params is None:
+#             params = self.mkParams()
+#         if srcdir is None:
+#             srcdir = self.srcdir
 
+#         ret = self.uses.mkTask(name, depends, rundir, srcdir, params)
 
-@dc.dataclass
-class TaskCtorParam(TaskCtor):
-    params : Dict[str,Any] = dc.field(default_factory=dict)
-
-    _log : ClassVar = logging.getLogger("TaskCtorParam")
-
-    def mkTask(self, name : str, depends, rundir, srcdir=None, params=None):
-        self._log.debug("--> %s::mkTask" % self.name)
-        if params is None:
-            params = self.mkParams()
-        if srcdir is None:
-            srcdir = self.srcdir
-
-        ret = self.uses.mkTask(name, depends, rundir, srcdir, params)
-
-        self.applyParams(ret.params)
-        self._log.debug("<-- %s::mkTask" % self.name)
-
-        return ret
-
-    def applyParams(self, params):
-        self._log.debug("--> %s::applyParams: %s %s" % (self.name, str(type(self.params)), str(type(params))))
-        if self.params is not None:
-            for k,v in self.params.items():
-                self._log.debug("  change %s %s=>%s" % (
-                    k, 
-                    str(getattr(params, k)),
-                    str(v)))
-                setattr(params, k, v)
-        else:
-            self._log.debug("  no params")
-        self._log.debug("<-- %s::applyParams: %s" % (self.name, str(self.params)))
+#         self.applyParams(ret.params)
+#         self._log.debug("<-- %s::mkTask" % self.name)
 
-@dc.dataclass
-class TaskCtorParamCls(TaskCtor):
-    params_ctor : Callable = None
+#         return ret
 
-    _log : ClassVar = logging.getLogger("TaskCtorParamType")
+#     def applyParams(self, params):
+#         self._log.debug("--> %s::applyParams: %s %s" % (self.name, str(type(self.params)), str(type(params))))
+#         if self.params is not None:
+#             for k,v in self.params.items():
+#                 self._log.debug("  change %s %s=>%s" % (
+#                     k, 
+#                     str(getattr(params, k)),
+#                     str(v)))
+#                 setattr(params, k, v)
+#         else:
+#             self._log.debug("  no params")
+#         self._log.debug("<-- %s::applyParams: %s" % (self.name, str(self.params)))
 
-    def mkParams(self):
-        self._log.debug("--> %s::mkParams" % str(self.name))
-        params = self.params_ctor()
-        self._log.debug("<-- %s::mkParams: %s" % (str(self.name), str(type(params))))
-        return params
+# @dc.dataclass
+# class TaskCtorParamCls(TaskCtor):
+#     params_ctor : Callable = None
 
-@dc.dataclass
-class TaskCtorCls(TaskCtor):
-    task_ctor : Callable = None
+#     _log : ClassVar = logging.getLogger("TaskCtorParamType")
 
-    _log : ClassVar = logging.getLogger("TaskCtorCls")
+#     def mkParams(self):
+#         self._log.debug("--> %s::mkParams" % str(self.name))
+#         params = self.params_ctor()
+#         self._log.debug("<-- %s::mkParams: %s" % (str(self.name), str(type(params))))
+#         return params
 
-    def mkTask(self, name : str, depends, rundir, srcdir=None, params=None):
-        self._log.debug("--> %s::mkTask (%s) srcdir=%s" % (self.name, str(self.task_ctor), srcdir))
+# @dc.dataclass
+# class TaskCtorCls(TaskCtor):
+#     task_ctor : Callable = None
 
-        if srcdir is None:
-            srcdir = self.srcdir
+#     _log : ClassVar = logging.getLogger("TaskCtorCls")
 
-        if params is None:
-            params = self.mkParams()
+#     def mkTask(self, name : str, depends, rundir, srcdir=None, params=None):
+#         self._log.debug("--> %s::mkTask (%s) srcdir=%s" % (self.name, str(self.task_ctor), srcdir))
 
-        ret = self.task_ctor(
-            name=name, 
-            depends=depends, 
-            rundir=rundir, 
-            srcdir=srcdir, 
-            params=params)
+#         if srcdir is None:
+#             srcdir = self.srcdir
 
-        # Update parameters on the way back
-        self.applyParams(ret.params)
+#         if params is None:
+#             params = self.mkParams()
 
-        self._log.debug("<-- %s::mkTask" % self.name)
-        return ret
+#         ret = self.task_ctor(
+#             name=name, 
+#             depends=depends, 
+#             rundir=rundir, 
+#             srcdir=srcdir, 
+#             params=params)
 
-@dc.dataclass
-class TaskCtorProxy(TaskCtor):
-    task_ctor : TaskCtor = None
-    param_ctor : Callable = None
+#         # Update parameters on the way back
+#         self.applyParams(ret.params)
+
+#         self._log.debug("<-- %s::mkTask" % self.name)
+#         return ret
 
-    _log : ClassVar = logging.getLogger("TaskCtorProxy")
+# @dc.dataclass
+# class TaskCtorProxy(TaskCtor):
+#     task_ctor : TaskCtor = None
+#     param_ctor : Callable = None
 
-    def mkTask(self, *args, **kwargs):
-        self._log.debug("--> %s::mkTask" % self.name)
-        ret = self.task_ctor.mkTask(*args, **kwargs)
-        self._log.debug("<-- %s::mkTask" % self.name)
-        return ret
+#     _log : ClassVar = logging.getLogger("TaskCtorProxy")
 
-    def mkParams(self, params=None):
-        self._log.debug("--> %s::mkParams: %s" % (self.name, str(self.params)))
+#     def mkTask(self, *args, **kwargs):
+#         self._log.debug("--> %s::mkTask" % self.name)
+#         ret = self.task_ctor.mkTask(*args, **kwargs)
+#         self._log.debug("<-- %s::mkTask" % self.name)
+#         return ret
 
-        if params is None and self.param_ctor is not None:
-            params = self.param_ctor()
+#     def mkParams(self, params=None):
+#         self._log.debug("--> %s::mkParams: %s" % (self.name, str(self.params)))
 
-        params = self.task_ctor.mkParams(params)
+#         if params is None and self.param_ctor is not None:
+#             params = self.param_ctor()
 
-        if self.params is not None:
-            for k,v in self.params.items():
-                self._log.debug("  change %s %s=>%s" % (
-                    k, 
-                    str(getattr(params, k)),
-                    str(v)))
-                setattr(params, k, v)
-        self._log.debug("<-- %s::mkParams: %s" % (self.name, str(self.params)))
-        return params
+#         params = self.task_ctor.mkParams(params)
+
+#         if self.params is not None:
+#             for k,v in self.params.items():
+#                 self._log.debug("  change %s %s=>%s" % (
+#                     k, 
+#                     str(getattr(params, k)),
+#                     str(v)))
+#                 setattr(params, k, v)
+#         self._log.debug("<-- %s::mkParams: %s" % (self.name, str(self.params)))
+#         return params
 
 
 @dc.dataclass
 class Task(object):
     """Executable view of a task"""
+    # Ctor fields -- must specify on construction
     name : str
-    params : TaskParams
-    srcdir : str = None
-    session : 'TaskGraphRunner' = None
-    basedir : str = None
-    memento : TaskMemento = None
-    depends : List['Task'] = dc.field(default_factory=list)
-    output : Any = None
-
-    # Implementation data below
-    basedir : str = dc.field(default=None)
+    srcdir : str
+    params : TaskParamsCtor 
+
+    # Runtime fields -- these get populated during execution
+    changed : bool = False
+    needs : List['Task'] = dc.field(default_factory=list)
+    dependents : List['Task'] = dc.field(default_factory=list)
     rundir : str = dc.field(default=None)
-    impl : str = None
-    body: Dict[str,Any] = dc.field(default_factory=dict)
-    impl_t : Any = None
+    output : TaskDataOutput = dc.field(default=None)
 
     _log : ClassVar = logging.getLogger("Task")
 
-    def init(self, runner, basedir):
-        self.session = runner
-        self.basedir = basedir
-
-    def getMemento(self, T) -> TaskMemento:
-        if os.path.isfile(os.path.join(self.rundir, "memento.json")):
-            with open(os.path.join(self.rundir, "memento.json"), "r") as fp:
-                try:
-                    data = json.load(fp)
-                    self.memento = T(**data)
-                except Exception as e:
-                    self._log.critical("Failed to load memento %s: %s" % (
-                        os.path.join(self.rundir, "memento.json"), str(e)))
-                    os.unlink(os.path.join(self.rundir, "memento.json"))
-        return self.memento
-
-    def setMemento(self, memento : TaskMemento):
-        self.memento = memento
-
-    async def isUpToDate(self, memento) -> bool:
-        return False
-
-    async def do_run(self, session) -> TaskData:
-        self._log.info("--> %s (%s) do_run - %d depends" % (
-            self.name, 
-            str(type(self)),
-            len(self.depends)))
-
-        self.session = session
-
-        if len(self.depends) > 0:
-            deps_o = []
-            for d in self.depends:
-                dep_o = d.getOutput()
-                if dep_o is None:
-                    raise Exception("Null output for %s" % d.name)
-                deps_o.append(dep_o)
-
-            input = TaskData.merge(deps_o)
-            input.src = self.name
-            input.deps[self.name] = list(inp.name for inp in self.depends)
-        else:
-            input = TaskData()
-
-
-        # Mark the source of this data as being this task
-        input.src = self.name
-
-        self.init_rundir()
-
-        self.output = await self.run(input)
-
-        if self.output is None:
-            raise Exception("No output produced by %s" % self.name)
-            result = TaskData()
-
-        # Write-back the memento, if specified
-        self.save_memento()
-
-        # Combine data from the deps to produce a result
-        self._log.info("<-- %s (%s) do_run - %d depends" % (
-            self.name, 
-            str(type(self)),
-            len(self.depends)))
-        return self.output
-
-    async def run(self, input : TaskData) -> TaskData:
+    async def run(self,
+                  runner : 'TaskRunner',
+                  input : TaskDataInput) -> TaskDataResult:
         raise NotImplementedError("TaskImpl.run() not implemented")
     
-    def init_rundir(self):
-        if not os.path.isdir(self.rundir):
-            os.makedirs(self.rundir)
-
-    def save_memento(self):
-        if self.memento is not None:
-            with open(os.path.join(self.rundir, "memento.json"), "w") as fp:
-                fp.write(self.memento.model_dump_json(indent=2))
-    
-    def getOutput(self) -> TaskData:
-        return self.output
-
-    def getField(self, name : str) -> Any:
-        if name in self.__dict__.keys():
-            return self.__dict__[name]
-        elif name in self.__pydantic_extra__.keys():
-            return self.__pydantic_extra__[name]
-        else:
-            raise Exception("No such field %s" % name)
-        
-
-
+    @staticmethod
+    def ctor(paramT):
+        def wrapper(T):
+            def mk_task(name, srcdir, params):
+                return T(name, srcdir, params)
+            def mk_params(**params):
+                ctor = TaskParamsCtor(paramT)
+                ctor.values.append(paramT(**params))
+                return ctor
+            T.mkTask = mk_task
+            T.mkParams = mk_params
+            return T
+        return wrapper
 

dv_flow/mgr/task_ctor.py

@@ -0,0 +1,43 @@
+import os
+import json
+import dataclasses as dc
+import logging
+from pydantic import BaseModel
+from typing import Any, Callable, ClassVar, Dict, List, Tuple
+from .task_data import TaskDataOutput, TaskDataResult
+
+# TaskParamsCtor accepts an evaluation context and returns a task parameter object
+TaskParamsCtor = Callable[[object], Any]
+
+@dc.dataclass
+class TaskCtor(object):
+    name : str
+    uses : 'TaskCtor' = None
+    srcdir : str = None
+
+    _log : ClassVar = logging.getLogger("TaskCtor")
+
+    def mkTask(self, name : str, srcdir=None) -> 'Task':
+        """Creates a task object"""
+        if srcdir is None:
+            srcdir = self.srcdir
+
+        if self.uses is not None:
+            return self.uses.mkTask(name, srcdir)
+        else:
+            raise NotImplementedError("TaskCtor.mkTask() not implemented for %s" % str(type(self)))
+    
+    def mkParams(self, params=None) -> TaskParamsCtor:
+        """Creates a list of task-parameter objects used produce the params object"""
+        self._log.debug("--> %s::mkParams" % self.name)
+        if self.uses is not None:
+            params = self.uses.mkParams()
+        else:
+            params = TaskParams()
+        self._log.debug("<-- %s::mkParams: %s" % (self.name, str(params)))
+
+        return params
+
+    def applyParams(self, params):
+        if self.uses is not None:
+            self.uses.applyParams(params)

dv_flow/mgr/task_data.py

@@ -27,6 +27,34 @@ from typing import Any, ClassVar, Dict, Set, List, Tuple
 from .fileset import FileSet
 from toposort import toposort
 
+class TaskMarkerLoc(BaseModel):
+    path : str
+    line : int = dc.Field(default=-1)
+    pos : int = dc.Field(default=-1)
+
+class TaskMarker(BaseModel):
+    msg : str
+    severity : str
+    loc : TaskMarkerLoc = dc.Field(default=None)
+
+class TaskDataInput(BaseModel):
+    changed : bool
+    srcdir : str
+    rundir : str
+    params : Any
+    memento : Any
+
+class TaskDataResult(BaseModel):
+    changed : bool = dc.Field(default=True)
+    output : List[Any] = dc.Field(default_factory=list)
+    memento : Any = dc.Field(default=None)
+    markers : List[TaskMarker] = dc.Field(default_factory=list)
+    status : int = dc.Field(default=0)
+
+class TaskDataOutput(BaseModel):
+    changed : bool
+    output : List[Any]
+
 class TaskDataParamOpE(enum.Enum):
     Set = enum.auto()
     Append = enum.auto()

dv_flow/mgr/task_def.py

@@ -40,7 +40,7 @@ class TaskDef(BaseModel):
     doc : str = dc.Field(default="")
     depends : List[Union[str,TaskSpec]] = dc.Field(default_factory=list, alias="needs")
     params: List[ParamDef] = dc.Field(default_factory=list, alias="with")
-    out: List[TaskOutput] = dc.Field(default_factory=list)
+#    out: List[TaskOutput] = dc.Field(default_factory=list)
 
     def copy(self) -> 'TaskDef':
         ret = TaskDef(

dv_flow/mgr/task_exec_data.py

@@ -0,0 +1,22 @@
+import pydantic.dataclasses as dc
+from pydantic import BaseModel
+from typing import Any, Dict, List
+
+
+class TaskExecData(BaseModel):
+    """Data from a single exection of a task"""
+    name : str
+    start : str
+    finish : str
+    status : int
+    memento : Any
+    markers : List[Any]
+
+class FlowExecData(BaseModel):
+    """
+    Data from multiple tasks executions. 'info' holds information
+    across multiple flow invocations. 'tasks' holds the names of
+    tasks executed in the most-recent invocation.
+    """
+    info : Dict[str, TaskExecData] = dc.Field(default_factory=dict)
+    tasks : List[str] = dc.Field(default_factory=list)

dv_flow/mgr/task_graph_builder.py

@@ -25,7 +25,7 @@ import logging
 from .package import Package
 from .package_def import PackageDef, PackageSpec
 from .pkg_rgy import PkgRgy
-from .task import Task, TaskCtor, TaskSpec
+from .task import Task
 from typing import Dict, List
 
 @dc.dataclass
@@ -37,7 +37,7 @@ class TaskGraphBuilder(object):
     _pkg_s : List[Package] = dc.field(default_factory=list)
     _pkg_m : Dict[PackageSpec,Package] = dc.field(default_factory=dict)
     _pkg_spec_s : List[PackageDef] = dc.field(default_factory=list)
-    _task_m : Dict[TaskSpec,Task] = dc.field(default_factory=dict)
+    _task_m : Dict['TaskSpec',Task] = dc.field(default_factory=dict)
     _logger : logging.Logger = None
 
     def __post_init__(self):
@@ -173,7 +173,7 @@ class TaskGraphBuilder(object):
 
         return pkg
         
-    def getTaskCtor(self, spec : TaskSpec, pkg : PackageDef = None) -> 'TaskCtor':
+    def getTaskCtor(self, spec : 'TaskSpec', pkg : PackageDef = None) -> 'TaskCtor':
         self._logger.debug("--> getTaskCtor %s" % spec.name)
         spec_e = spec.name.split(".")
         task_name = spec_e[-1]

dv_flow/mgr/task_graph_runner.py

@@ -21,20 +21,19 @@
 #****************************************************************************
 import asyncio
 import os
-import yaml
 import dataclasses as dc
-from typing import Any, Callable, ClassVar, Dict, List
+from typing import Any, Callable, ClassVar, Dict, List, Tuple
 from .task import Task
 from .task_data import TaskData
+from .task_runner import TaskRunner
 
 @dc.dataclass
-class TaskGraphRunner(object):
+class TaskGraphRunner(TaskRunner):
     """Session manages execution of a task graph"""
 
-    rundir : str
-
     _inst : ClassVar['TaskGraphRunner'] = None
 
+
     # Search path for .dfs files
     create_subprocess : Callable = asyncio.create_subprocess_exec
     _root_dir : str = None

dv_flow/mgr/task_graph_runner_local.py

@@ -30,7 +30,7 @@ from .fragment_def import FragmentDef
 from .package import Package
 from .pkg_rgy import PkgRgy
 from .package_def import PackageDef, PackageSpec
-from .task import Task, TaskSpec, TaskCtor
+from .task import Task
 from .task_data import TaskData
 from .task_graph_runner import TaskGraphRunner
 

dv_flow/mgr/task_params_ctor.py

@@ -0,0 +1,31 @@
+import dataclasses as dc
+from typing import Any, List
+
+
+@dc.dataclass
+class TaskParamsCtor(object):
+
+    paramT : Any = None
+    # List of parameter-setting sets, starting with inner-most
+    values : List[Any] = dc.field(default_factory=list)
+
+    def mk(self, input) -> Any:
+        params = self.paramT()
+
+        # Now, process each field 
+        for field in dc.fields(self.paramT):
+            # Find the outer-most setting of the value
+            last_value_i = -1
+            for i in range(len(self.values)-1, -1, -1):
+                if hasattr(self.values[i], field.name) and getattr(self.values[i], field.name) is not None:
+                    val = getattr(self.values[i], field.name)
+                    # TODO: check if 'val' is a set or mutator
+                    last_value_i = i
+                    break
+            for i in range(last_value_i, -1, -1):
+                if hasattr(self.values[i], field.name):
+                    val = getattr(self.values[i], field.name)
+                    setattr(params, field.name, val)
+
+        print("params: %s" % str(params))
+        return params

dv_flow/mgr/task_runner.py

@@ -0,0 +1,46 @@
+import dataclasses as dc
+from .task_data import TaskDataInput, TaskDataResult
+from typing import Any, Callable, List, Tuple
+
+@dc.dataclass
+class TaskRunner(object):
+    rundir : str
+
+    # List of [Listener:Callable[Task],Recurisve:bool]
+    listeners : List[Tuple[Callable['Task','Reason'], bool]] = dc.field(default_factory=list)
+
+    async def do_run(self, 
+                  task : 'Task',
+                  memento : Any = None) -> 'TaskDataResult':
+        return await self.run(task, memento)
+
+    async def run(self, 
+                  task : 'Task',
+                  memento : Any = None) -> 'TaskDataResult':
+        pass
+
+@dc.dataclass
+class SingleTaskRunner(TaskRunner):
+
+    async def run(self, 
+                  task : 'Task',
+                  memento : Any = None) -> 'TaskDataResult':
+        changed = False
+        for dep in task.needs:
+            changed |= dep.changed
+
+        # TODO: create an evaluator for substituting param values
+        eval = None
+
+        params = task.params.mk(eval)
+
+        input = TaskDataInput(
+            changed=changed,
+            srcdir=task.srcdir,
+            rundir=self.rundir,
+            params=params,
+            memento=memento)
+        
+        ret = await task.run(self, input)
+
+        return ret

{dv_flow_mgr-0.0.1.13335470340a1.dist-info → dv_flow_mgr-0.0.1.13478128278a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.13335470340a1
+Version: 0.0.1.13478128278a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.13335470340a1.dist-info → dv_flow_mgr-0.0.1.13478128278a1.dist-info}/RECORD

@@ -1,23 +1,29 @@
 dv_flow/mgr/__init__.py,sha256=IZA7I1u7RH34DrJXSkETxWFpK5Jn_A2zXfnCAzJ8XxA,289
 dv_flow/mgr/__main__.py,sha256=uik6gDAvtZNG0yyDKYc4FTl6R1QHAR543jNG2lCAa6E,1771
-dv_flow/mgr/expr_parser.py,sha256=4gVpWvBWkI6KfTN2PAXhZtAgKIRLxfFs5Gc5IAa27Dg,605
+dv_flow/mgr/expr_eval.py,sha256=pfnTDurXPTURgpdzlno5XtXlkoPfnJmAAq45G9T2D3k,1517
+dv_flow/mgr/expr_parser.py,sha256=aR8n6hgk5xmFknIaSVC02K-So-eU4pkkFtwb7zWsSPs,5474
 dv_flow/mgr/fileset.py,sha256=FNvC5sU2ArxJ0OO3v8dXTv8zX-bZ5t0a0ljne0fQQ1o,1150
 dv_flow/mgr/fragment_def.py,sha256=cyzp1XeWtNOaagScmeS-BPsoXj9j2LTBbKq5ZUioz8I,1641
 dv_flow/mgr/out,sha256=d8GGBi3J43fhdLBlnsUbzBfRe0TD0QTP3nOTz54l2bI,200
-dv_flow/mgr/package.py,sha256=MDnt9q4pD_Aex_hSK_EUQ38IiaxOihgxbMmk5dH1deQ,1674
-dv_flow/mgr/package_def.py,sha256=Sv3KS5pDysUrtPBWElCvmWt2pnkTipZcZGYwKr5N7Mc,14967
+dv_flow/mgr/package.py,sha256=878twhPD-E1pFlDNUtuyeFEgJ_Y89b560og4St-Iwrs,1679
+dv_flow/mgr/package_def.py,sha256=dwvs-7kGEp7wFrBK2Fybysb-vSKwtWOtxTrg1VIaL14,14975
 dv_flow/mgr/package_import_spec.py,sha256=ah3r15v5Jdub2poc3sgi6Uar1L3oGoYsCPPNiOHV-a4,1760
 dv_flow/mgr/param_def.py,sha256=e2WvRCMArbgcsKC4TKQqZTscZmCdo3WvVR6w3LN0VK8,727
+dv_flow/mgr/parsetab.py,sha256=enSOnMQ-woIsMEzHyeYiefvhAl8enxfX9Ct_o8-jkqs,3780
 dv_flow/mgr/pkg_rgy.py,sha256=2R_EaeBDJn5qUq9DzSnLc37wUP36MWSv-p0LgUjJRAg,4471
-dv_flow/mgr/task.py,sha256=giy5elwIS_fab6wGnA8JCeZIEeX0nG5zpcq5ifGYSOs,9087
-dv_flow/mgr/task_data.py,sha256=ZKoYUhXGyLXXm4GvpS-UDwJ_lZH72lgpzOdHm1XDObY,10386
-dv_flow/mgr/task_def.py,sha256=6qDDgge5RXssL5TAZR01dGb_Vjvbj72tJirg66O-Yt4,1818
-dv_flow/mgr/task_graph_builder.py,sha256=4dfy_T26MYI7ls2xiofr4V0ItZHDehU2iw6UBPt-SZQ,7074
-dv_flow/mgr/task_graph_runner.py,sha256=NwNYcOJ952lPMLwIIlYE9CoDdedqvcw2fWHYUsKFXuU,2164
-dv_flow/mgr/task_graph_runner_local.py,sha256=66JYJcu-W60LjfuT9UkbcKslNqDcD5q4UR7ZAWrSaGM,4707
+dv_flow/mgr/task.py,sha256=kLQSvnVwj9ROIDtxq8lLu-4mJizTxOqvUeogmgN6QAA,5976
+dv_flow/mgr/task_ctor.py,sha256=hlfl-UVvyjzLFN6D0Oel9eBs0xUQPqCX7gQ0uEHoL7o,1382
+dv_flow/mgr/task_data.py,sha256=fkaVzQ81nJBXy4uPWPdnVCNoD9L7rvH5Abt8LXzvnJs,11098
+dv_flow/mgr/task_def.py,sha256=WAW1fPXUfUiQcfmgCx0iCMddMBKATDa5RsRztJWRbUk,1819
+dv_flow/mgr/task_exec_data.py,sha256=aT__kmVmNxqnS_GbTRS1vrYgKiv8cH-HUSmRb6YYKsI,640
+dv_flow/mgr/task_graph_builder.py,sha256=-lRSjWU2UJf3euVIFtoVIU7Qdox7MI1sKERWg1k7U_g,7058
+dv_flow/mgr/task_graph_runner.py,sha256=jUGI49QvxUCfQoKQDDk2psbeapIcCg72qNOW1JipHzM,2182
+dv_flow/mgr/task_graph_runner_local.py,sha256=OrydPwtQ8E7hYWvSXx0h7lI3nfUNFyklULhsyMwz9dA,4687
 dv_flow/mgr/task_impl_data.py,sha256=bFPijoKrh9x7fZN2DsvRJp0UHo-gGM0VjtDQISyfhFk,321
 dv_flow/mgr/task_memento.py,sha256=C7VTQpBhDEoYuDmE6YTM-6TLMLnqHp6Y0Vat1aTgtCs,1096
 dv_flow/mgr/task_output.py,sha256=l-W-FvVo6YDah1RQS-I9N0KUtB3vp-kl7lxIdmNz0l4,178
+dv_flow/mgr/task_params_ctor.py,sha256=aXgB8o9xFPjaEjGW_xYkEC0N0apzGzGUPDj7g2ZLvus,1112
+dv_flow/mgr/task_runner.py,sha256=XrVYZ13qIbGDkLgDT-NAq_qd2plCqVjHKEe25MD-7xE,1272
 dv_flow/mgr/type.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dv_flow/mgr/type_def.py,sha256=KdhuNlfw-NKU-4VZFCnMPyj775yEB7cpr5tz73a9yuQ,259
 dv_flow/mgr/util.py,sha256=06eVyURF4ga-s8C9Sd3ZSDebwO4QS0XXaB8xADVbWRc,1437
@@ -27,9 +33,9 @@ dv_flow/mgr/std/fileset.py,sha256=uP7bGntRq-Tn5_GEFnt0_J_OAmfvep3GlCwCuE8by4o,27
 dv_flow/mgr/std/flow.dv,sha256=j9wLrF3Ghh1ZLbJxmk7WiNiRYUYEer-8CCUA5hsgtfk,1409
 dv_flow/mgr/std/message.py,sha256=BPTHnEMD4tBufQ9LvsS9Sa_0xjaJATbBpwqosWslvVA,193
 dv_flow/mgr/std/task_null.py,sha256=KObmjG_4D08GJ1k6neqKIQrFY72Sj0jLnwXxEkq5HA0,321
-dv_flow_mgr-0.0.1.13335470340a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dv_flow_mgr-0.0.1.13335470340a1.dist-info/METADATA,sha256=NP0fraxF2IX1pTGM8-F-suzwkkbWU09BFex8ktZSVc8,13276
-dv_flow_mgr-0.0.1.13335470340a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-dv_flow_mgr-0.0.1.13335470340a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
-dv_flow_mgr-0.0.1.13335470340a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
-dv_flow_mgr-0.0.1.13335470340a1.dist-info/RECORD,,
+dv_flow_mgr-0.0.1.13478128278a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-0.0.1.13478128278a1.dist-info/METADATA,sha256=HCQCME5rxfNs5HNZ0WmoQtIQMTWNtZzkGrfwhXoO09I,13276
+dv_flow_mgr-0.0.1.13478128278a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+dv_flow_mgr-0.0.1.13478128278a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
+dv_flow_mgr-0.0.1.13478128278a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
+dv_flow_mgr-0.0.1.13478128278a1.dist-info/RECORD,,