dv-flow-mgr 0.0.1.13577785562a1__py3-none-any.whl → 0.0.1.13644197178a1__py3-none-any.whl

dv_flow/mgr/__init__.py

@@ -7,4 +7,5 @@ from .task_data import *
 from .task_graph_runner import TaskGraphRunner
 from .task_graph_runner_local import TaskGraphRunnerLocal
 from .task_graph_builder import TaskGraphBuilder
+from .task_node import task
 

dv_flow/mgr/package_def.py

@@ -33,7 +33,7 @@ from typing import Any, Dict, List, Callable, Tuple, ClassVar
 from .fragment_def import FragmentDef
 from .package import Package
 from .package_import_spec import PackageImportSpec, PackageSpec
-#from .task import TaskCtorCls, TaskCtorParam, TaskCtorParamCls
+from .task_node import TaskNodeCtor, TaskNodeCtorProxy, TaskNodeCtorTask
 from .task_ctor import TaskCtor
 from .task_def import TaskDef, TaskSpec
 from .std.task_null import TaskNull
@@ -151,20 +151,7 @@ class PackageDef(BaseModel):
             if decl_params:
                 self._log.debug("Type declares new parameters")
                 # We need to combine base parameters with new parameters
-                ptype_m = {
-                    "str" : str,
-                    "int" : int,
-                    "float" : float,
-                    "bool" : bool,
-                    "list" : List
-                }
-                pdflt_m = {
-                    "str" : "",
-                    "int" : 0,
-                    "float" : 0.0,
-                    "bool" : False,
-                    "list" : []
-                }
+
                 for p in task.params.keys():
                     param = task.params[p]
                     self._log.debug("param: %s" % str(param))
@@ -221,16 +208,24 @@ class PackageDef(BaseModel):
 
     def mkTaskCtor(self, session, task, srcdir, tasks_m) -> TaskCtor:
         self._log.debug("--> %s::mkTaskCtor %s (srcdir: %s)" % (self.name, task.name, srcdir))
+        base_ctor_t : TaskCtor = None
         ctor_t : TaskCtor = None
+        base_params : BaseModel = None
+        callable = None
+        needs = [] if task.needs is None else task.needs.copy()
+
+        if task.uses is not None:
+            base_ctor_t = self.getTaskCtor(session, task.uses, tasks_m)
+            base_params = base_ctor_t.mkTaskParams()
 
         # Determine the implementation constructor first
-        if task.pyclass is not None:
+        if task.pytask is not None:
             # Built-in impl
             # Now, lookup the class
-            self._log.debug("Use PyClass implementation")
-            last_dot = task.pyclass.rfind('.')
-            clsname = task.pyclass[last_dot+1:]
-            modname = task.pyclass[:last_dot]
+            self._log.debug("Use PyTask implementation")
+            last_dot = task.pytask.rfind('.')
+            clsname = task.pytask[last_dot+1:]
+            modname = task.pytask[:last_dot]
 
             try:
                 if modname not in sys.modules:
@@ -245,26 +240,26 @@ class PackageDef(BaseModel):
                 
             if not hasattr(mod, clsname):
                 raise Exception("Class %s not found in module %s" % (clsname, modname))
-            task_ctor = getattr(mod, clsname)
-
-            # Determine if we need to use a new 
-
-            if task.uses is not None:
-                uses = self.getTaskCtor(session, task.uses, tasks_m)
-            else:
-                uses = None
+            callable = getattr(mod, clsname)
 
-            ctor_t = TaskCtorCls(
+        # Determine if we need to use a new 
+        paramT = self._getParamT(task, base_params)
+        
+        if callable is not None:
+            ctor_t = TaskNodeCtorTask(
                 name=task.name,
-                uses=uses,
-                task_ctor=task_ctor,
-                srcdir=srcdir)
-        elif task.uses is not None:
+                srcdir=srcdir,
+                paramT=paramT, # TODO: need to determine the parameter type
+                needs=needs, # TODO: need to determine the needs
+                task=callable)
+        elif base_ctor_t is not None:
             # Use the existing (base) to create the implementation
-            ctor_t = TaskCtor(
+            ctor_t = TaskNodeCtorProxy(
                 name=task.name,
-                uses=self.getTaskCtor(session, task.uses, tasks_m),
-                srcdir=srcdir)
+                srcdir=srcdir,
+                paramT=paramT, # TODO: need to determine the parameter type
+                needs=needs,
+                uses=base_ctor_t)
         else:
             self._log.debug("Use 'Null' as the class implementation")
             ctor_t = TaskCtorCls(
@@ -272,11 +267,77 @@ class PackageDef(BaseModel):
                 task_ctor=TaskNull,
                 srcdir=srcdir)
 
-        ctor_t = self.handleParams(task, ctor_t)
-        ctor_t.depends.extend(task.depends)
+# TODO:
+#        ctor_t = self.handleParams(task, ctor_t)
+#        ctor_t.depends.extend(task.depends)
 
         self._log.debug("<-- %s::mkTaskCtor %s" % (self.name, task.name))
         return ctor_t
+    
+    def _getParamT(self, task, base_t : BaseModel):
+        # Get the base parameter type (if available)
+        # We will build a new type with updated fields
+
+        ptype_m = {
+            "str" : str,
+            "int" : int,
+            "float" : float,
+            "bool" : bool,
+            "list" : List
+        }
+        pdflt_m = {
+            "str" : "",
+            "int" : 0,
+            "float" : 0.0,
+            "bool" : False,
+            "list" : []
+        }
+
+        fields = []
+        field_m : Dict[str,int] = {}
+
+        # First, pull out existing fields (if there's a base type)
+        if base_t is not None:
+            self._log.debug("Base type: %s" % str(base_t))
+            for name,f in base_t.model_fields.items():
+                ff : dc.Field = f
+                fields.append(f)
+                field_m[name] = (f.annotation, getattr(base_t, name))
+        else:
+            self._log.debug("No base type")
+
+        for p in task.params.keys():
+            param = task.params[p]
+            self._log.debug("param: %s %s (%s)" % (p, str(param), str(type(param))))
+            if hasattr(param, "type") and param.type is not None:
+                ptype_s = param.type
+                if ptype_s not in ptype_m.keys():
+                    raise Exception("Unknown type %s" % ptype_s)
+                ptype = ptype_m[ptype_s]
+
+                if p in field_m.keys():
+                    raise Exception("Duplicate field %s" % p)
+                if param.value is not None:
+                    field_m[p] = (ptype, param.value)
+                else:
+                    field_m[p] = (ptype, pdflt_m[ptype_s])
+                self._log.debug("Set param=%s to %s" % (p, str(field_m[p][1])))
+            else:
+                if p not in field_m.keys():
+                    raise Exception("Field %s not found" % p)
+                if type(param) != dict:
+                    value = param
+                elif "value" in param.keys():
+                    value = param["value"]
+                else:
+                    raise Exception("No value specified for param %s: %s" % (
+                        p, str(param)))
+                field_m[p] = (field_m[p][0], value)
+                self._log.debug("Set param=%s to %s" % (p, str(field_m[p][1])))
+
+        params_t = pydantic.create_model("Task%sParams" % task.name, **field_m)
+
+        return params_t
 
     @staticmethod
     def load(path, exp_pkg_name=None):

dv_flow/mgr/param_def.py

@@ -14,7 +14,6 @@ class ComplexType(BaseModel):
     map : Union[MapType, None] = None
 
 class ParamDef(BaseModel):
-    name : str
     doc : str = None
     type : Union[str, 'ComplexType'] = None
     value : Union[Any, None] = None

dv_flow/mgr/param_ref_eval.py

@@ -7,8 +7,8 @@ from .eval_jq import eval_jq
 @dc.dataclass
 class ParamRefEval(object):
 
-    parser : ExprParser = ExprParser()
-    expr_eval : ExprEval = ExprEval()
+    parser : ExprParser = dc.field(default_factory=ExprParser)
+    expr_eval : ExprEval = dc.field(default_factory=ExprEval)
 
     def __post_init__(self):
         self.expr_eval.methods["jq"] = eval_jq

dv_flow/mgr/task_def.py

@@ -35,11 +35,11 @@ class TaskDef(BaseModel):
     name : str
 #    type : Union[str,TaskSpec] = dc.Field(default_factory=list)
     uses : str = dc.Field(default=None)
-    pyclass : str = dc.Field(default=None)
+    pytask : str = dc.Field(default=None)
     desc : str = dc.Field(default="")
     doc : str = dc.Field(default="")
-    depends : List[Union[str,TaskSpec]] = dc.Field(default_factory=list, alias="needs")
-    params: List[ParamDef] = dc.Field(default_factory=list, alias="with")
+    needs : List[Union[str,TaskSpec]] = dc.Field(default_factory=list, alias="needs")
+    params: Dict[str,Union[str,ParamDef]] = dc.Field(default_factory=dict, alias="with")
 #    out: List[TaskOutput] = dc.Field(default_factory=list)
 
     def copy(self) -> 'TaskDef':

dv_flow/mgr/task_graph_builder.py

@@ -96,20 +96,26 @@ class TaskGraphBuilder(object):
 
         self._logger.debug("ctor_t: %s" % ctor_t.name)
 
-        depends = []
+        needs = []
 
-        for dep in ctor_t.depends:
-            if not dep in self._task_m.keys():
-                task = self._mkTaskGraph(dep, rundir)
-                self._task_m[dep] = task
-                pass
-            depends.append(self._task_m[dep])
+        # for dep in ctor_t.depends:
+        #     if not dep in self._task_m.keys():
+        #         task = self._mkTaskGraph(dep, rundir)
+        #         self._task_m[dep] = task
+        #         pass
+        #     depends.append(self._task_m[dep])
 
         # The returned task should have all param references resolved
-        task = ctor_t.mkTask(
+        params = ctor_t.mkTaskParams()
+
+        if params is None:
+            raise Exception("ctor %s returned None for params" % str(ctor_t))
+
+        task = ctor_t.mkTaskNode(
+            params=params,
             name=task_name,
-            depends=depends,
-            rundir=rundir)
+            needs=needs)
+        task.rundir = rundir
         
         self._task_m[task.name] = task
 

dv_flow/mgr/task_node.py

@@ -1,4 +1,5 @@
-
+import os
+import sys
 import dataclasses as dc
 import pydantic.dataclasses as pdc
 import logging
@@ -27,6 +28,10 @@ class TaskNode(object):
 
     _log : ClassVar = logging.getLogger("TaskNode")
 
+    def __post_init__(self):
+        if self.needs is None:
+            self.needs = []
+
     async def do_run(self, 
                   runner,
                   rundir,
@@ -96,38 +101,110 @@ class TaskNodeCtor(object):
     - Produces a TaskNode
     """
     name : str
+    srcdir : str
+    paramT : Any
 
-    def mkTaskNode(self, srcdir, params, name=None) -> TaskNode:
+    def getNeeds(self) -> List[str]:
+        return []
+
+    def mkTaskNode(self,
+                   params,
+                   srcdir=None,
+                   name=None,
+                   needs=None) -> TaskNode:
         raise NotImplementedError("mkTaskNode in type %s" % str(type(self)))
 
-    def mkTaskParams(self, params : Dict) -> Any:
-        raise NotImplementedError("mkTaskParams in type %s" % str(type(self)))
+    def mkTaskParams(self, params : Dict = None) -> Any:
+        obj = self.paramT()
+
+        # Apply user-specified params
+        if params is not None:
+            for key,value in params.items():
+                if not hasattr(obj, key):
+                    raise Exception("Parameters class %s does not contain field %s" % (
+                        str(type(obj)),
+                        key))
+                else:
+                    if isinstance(value, Param):
+                        if value.append is not None:
+                            ex_value = getattr(obj, key, [])
+                            ex_value.extend(value.append)
+                            setattr(obj, key, ex_value)
+                        elif value.prepend is not None:
+                            ex_value = getattr(obj, key, [])
+                            value = value.copy()
+                            value.extend(ex_value)
+                            setattr(obj, key, value)
+                            pass
+                        else:
+                            raise Exception("Unhandled value spec: %s" % str(value))
+                    else:
+                        setattr(obj, key, value)
+        return obj
+
+@dc.dataclass
+class TaskNodeCtorDefBase(TaskNodeCtor):
+    """Task defines its own needs, that will need to be filled in"""
+    needs : List['str']
+
+    def __post_init__(self):
+        if self.needs is None:
+            self.needs = []
+
+    def getNeeds(self) -> List[str]:
+        return self.needs
+
+@dc.dataclass
+class TaskNodeCtorProxy(TaskNodeCtorDefBase):
+    """Task has a 'uses' clause, so we delegate creation of the node"""
+    uses : TaskNodeCtor
+
+    def mkTaskNode(self, params, srcdir=None, name=None, needs=None) -> TaskNode:
+        if srcdir is None:
+            srcdir = self.srcdir
+        node = self.uses.mkTaskNode(params=params, srcdir=srcdir, name=name, needs=needs)
+        return node
+    
+@dc.dataclass
+class TaskNodeCtorTask(TaskNodeCtorDefBase):
+    task : Callable[['TaskRunner','TaskDataInput'],'TaskDataResult']
+
+    def mkTaskNode(self, params, srcdir=None, name=None, needs=None) -> TaskNode:
+        if srcdir is None:
+            srcdir = self.srcdir
+
+        node = TaskNode(name, srcdir, params, self.task, needs=needs)
+        node.task = self.task
+
+        return node
 
 @dc.dataclass
 class TaskNodeCtorWrapper(TaskNodeCtor):
     T : Any
-    paramT : Any
 
     def __call__(self, 
-                 srcdir, 
-                 name=None, 
-                 params=None, 
+                 name=None,
+                 srcdir=None,
+                 params=None,
                  needs=None,
                  **kwargs):
         """Convenience method for direct creation of tasks"""
         if params is None:
             params = self.mkTaskParams(kwargs)
         
-        node = self.mkTaskNode(srcdir, params, name)
-        if needs is not None:
-            node.needs.extend(needs)
+        node = self.mkTaskNode(
+            srcdir=srcdir, 
+            params=params, 
+            name=name, 
+            needs=needs)
+
         return node
 
-    def mkTaskNode(self, srcdir, params, name=None) -> TaskNode:
-        node = TaskNode(name, srcdir, params, self.T)
+    def mkTaskNode(self, params, srcdir=None, name=None, needs=None) -> TaskNode:
+        node = TaskNode(name, srcdir, params, self.T, needs=needs)
         return node
 
-    def mkTaskParams(self, params : Dict) -> Any:
+    def mkTaskParams(self, params : Dict = None) -> Any:
         obj = self.paramT()
 
         # Apply user-specified params
@@ -155,7 +232,16 @@ class TaskNodeCtorWrapper(TaskNodeCtor):
         return obj
 
 def task(paramT):
+    """Decorator to wrap a task method as a TaskNodeCtor"""
     def wrapper(T):
-        ctor = TaskNodeCtorWrapper(T.__name__, T, paramT)
+        task_mname = T.__module__
+        task_module = sys.modules[task_mname]
+        ctor = TaskNodeCtorWrapper(
+            T.__name__, 
+            os.path.dirname(os.path.abspath(task_module.__file__)), 
+            paramT,
+            T)
         return ctor
     return wrapper
+
+

{dv_flow_mgr-0.0.1.13577785562a1.dist-info → dv_flow_mgr-0.0.1.13644197178a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.13577785562a1
+Version: 0.0.1.13644197178a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.13577785562a1.dist-info → dv_flow_mgr-0.0.1.13644197178a1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-dv_flow/mgr/__init__.py,sha256=IZA7I1u7RH34DrJXSkETxWFpK5Jn_A2zXfnCAzJ8XxA,289
+dv_flow/mgr/__init__.py,sha256=_IH_cJws79HRCdn0-Q11WHFuMQYA6oPCZIufeXT-BOc,317
 dv_flow/mgr/__main__.py,sha256=uik6gDAvtZNG0yyDKYc4FTl6R1QHAR543jNG2lCAa6E,1771
 dv_flow/mgr/eval_jq.py,sha256=Ue-qkyNW6uOu2Wy8u6nrTbPPY7ujaHd0-4iki0eV3Ec,294
 dv_flow/mgr/expr_eval.py,sha256=Mp0TvrV6gZWcj1uYwLfPNI-ARwwdAfwntE5byLBfBuY,2276
@@ -7,24 +7,24 @@ dv_flow/mgr/fileset.py,sha256=FNvC5sU2ArxJ0OO3v8dXTv8zX-bZ5t0a0ljne0fQQ1o,1150
 dv_flow/mgr/fragment_def.py,sha256=cyzp1XeWtNOaagScmeS-BPsoXj9j2LTBbKq5ZUioz8I,1641
 dv_flow/mgr/out,sha256=d8GGBi3J43fhdLBlnsUbzBfRe0TD0QTP3nOTz54l2bI,200
 dv_flow/mgr/package.py,sha256=878twhPD-E1pFlDNUtuyeFEgJ_Y89b560og4St-Iwrs,1679
-dv_flow/mgr/package_def.py,sha256=dwvs-7kGEp7wFrBK2Fybysb-vSKwtWOtxTrg1VIaL14,14975
+dv_flow/mgr/package_def.py,sha256=m3vnoEcV4jEhOg_ijwYPTplrF4YN5dVJYJhVBTL5AgM,17408
 dv_flow/mgr/package_import_spec.py,sha256=ah3r15v5Jdub2poc3sgi6Uar1L3oGoYsCPPNiOHV-a4,1760
 dv_flow/mgr/param.py,sha256=3BY-ucig6JRw73FhjyJQL-vpd57qhAzixgZ8I5FoUpw,553
-dv_flow/mgr/param_def.py,sha256=e2WvRCMArbgcsKC4TKQqZTscZmCdo3WvVR6w3LN0VK8,727
-dv_flow/mgr/param_ref_eval.py,sha256=dpV6IPdTtZHLy8maTsN61Ce1Cc5M5dLQW5sauNujEMU,1090
+dv_flow/mgr/param_def.py,sha256=gLua-EQiY8V2CFX-2svLRIlrs8PEeGh4-EPtn4a2Mng,712
+dv_flow/mgr/param_ref_eval.py,sha256=U8QhDf1n_9bLnExdc1oQamq5-pOUXrFalOX4oyR9UoM,1138
 dv_flow/mgr/parsetab.py,sha256=enSOnMQ-woIsMEzHyeYiefvhAl8enxfX9Ct_o8-jkqs,3780
 dv_flow/mgr/pkg_rgy.py,sha256=2R_EaeBDJn5qUq9DzSnLc37wUP36MWSv-p0LgUjJRAg,4471
 dv_flow/mgr/task.py,sha256=kLQSvnVwj9ROIDtxq8lLu-4mJizTxOqvUeogmgN6QAA,5976
 dv_flow/mgr/task_ctor.py,sha256=hlfl-UVvyjzLFN6D0Oel9eBs0xUQPqCX7gQ0uEHoL7o,1382
 dv_flow/mgr/task_data.py,sha256=9c5NrVZWECHFXrCPOpwZBQXRhr1O3ni5aFeP2I1P5Rw,11398
-dv_flow/mgr/task_def.py,sha256=WAW1fPXUfUiQcfmgCx0iCMddMBKATDa5RsRztJWRbUk,1819
+dv_flow/mgr/task_def.py,sha256=gOqh-sUE7MKC1fsobFxcO2jlF-jtjBJRFzWB2iDYlZY,1831
 dv_flow/mgr/task_exec_data.py,sha256=aT__kmVmNxqnS_GbTRS1vrYgKiv8cH-HUSmRb6YYKsI,640
-dv_flow/mgr/task_graph_builder.py,sha256=-lRSjWU2UJf3euVIFtoVIU7Qdox7MI1sKERWg1k7U_g,7058
+dv_flow/mgr/task_graph_builder.py,sha256=iprAMw0LVROOxWa563OHg8KTEm4IOS9hZVymjDeCnEo,7243
 dv_flow/mgr/task_graph_runner.py,sha256=jUGI49QvxUCfQoKQDDk2psbeapIcCg72qNOW1JipHzM,2182
 dv_flow/mgr/task_graph_runner_local.py,sha256=OrydPwtQ8E7hYWvSXx0h7lI3nfUNFyklULhsyMwz9dA,4687
 dv_flow/mgr/task_impl_data.py,sha256=bFPijoKrh9x7fZN2DsvRJp0UHo-gGM0VjtDQISyfhFk,321
 dv_flow/mgr/task_memento.py,sha256=C7VTQpBhDEoYuDmE6YTM-6TLMLnqHp6Y0Vat1aTgtCs,1096
-dv_flow/mgr/task_node.py,sha256=uCsyNT9IQu0cok27qTEi4iTB6FqobXbx7hbVAmjELKc,5130
+dv_flow/mgr/task_node.py,sha256=J_i53KntyljY7gwSqBq-lK_Gbu0jryNp4Xq_jUsa0PA,7903
 dv_flow/mgr/task_output.py,sha256=l-W-FvVo6YDah1RQS-I9N0KUtB3vp-kl7lxIdmNz0l4,178
 dv_flow/mgr/task_params_ctor.py,sha256=aXgB8o9xFPjaEjGW_xYkEC0N0apzGzGUPDj7g2ZLvus,1112
 dv_flow/mgr/task_runner.py,sha256=JmerE6CyXiJJK7-n6P65OzwW9UTYFQqV8Cck7eHe0B8,3882
@@ -37,9 +37,9 @@ dv_flow/mgr/std/fileset.py,sha256=uP7bGntRq-Tn5_GEFnt0_J_OAmfvep3GlCwCuE8by4o,27
 dv_flow/mgr/std/flow.dv,sha256=j9wLrF3Ghh1ZLbJxmk7WiNiRYUYEer-8CCUA5hsgtfk,1409
 dv_flow/mgr/std/message.py,sha256=BPTHnEMD4tBufQ9LvsS9Sa_0xjaJATbBpwqosWslvVA,193
 dv_flow/mgr/std/task_null.py,sha256=KObmjG_4D08GJ1k6neqKIQrFY72Sj0jLnwXxEkq5HA0,321
-dv_flow_mgr-0.0.1.13577785562a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dv_flow_mgr-0.0.1.13577785562a1.dist-info/METADATA,sha256=UeJMpl2RnCOQX98UgtpVXI3qjSPhe4Zs_Gcg4-Cv4hE,13276
-dv_flow_mgr-0.0.1.13577785562a1.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-dv_flow_mgr-0.0.1.13577785562a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
-dv_flow_mgr-0.0.1.13577785562a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
-dv_flow_mgr-0.0.1.13577785562a1.dist-info/RECORD,,
+dv_flow_mgr-0.0.1.13644197178a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-0.0.1.13644197178a1.dist-info/METADATA,sha256=yOdqkLx8LGFA6PcCcaQ3eMIwC3_lM0iup5Tx0iEra4U,13276
+dv_flow_mgr-0.0.1.13644197178a1.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+dv_flow_mgr-0.0.1.13644197178a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
+dv_flow_mgr-0.0.1.13644197178a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
+dv_flow_mgr-0.0.1.13644197178a1.dist-info/RECORD,,