dv-flow-mgr 1.9.0__py3-none-any.whl → 1.9.14960981876rc0__py3-none-any.whl

dv_flow/mgr/__init__.py

@@ -32,6 +32,6 @@ from .task_runner import TaskSetRunner
 from .task_listener_log import TaskListenerLog
 
 VERSION="1.9.0"
-SUFFIX=""
+SUFFIX="14960981876rc0"
 __version__="%s%s" % (VERSION, SUFFIX)
 

dv_flow/mgr/exec_gen_callable.py

@@ -0,0 +1,85 @@
+import dataclasses as dc
+import importlib
+import importlib.util
+import logging
+import os
+import sys
+from typing import ClassVar, List
+from .task_data import TaskDataResult
+
+@dc.dataclass
+class ExecGenCallable(object):
+    body : str
+    _log : ClassVar = logging.getLogger("ExecGenCallable")
+
+    def __call__(self, ctxt, input):
+        self._log.debug("--> ExecCallable")
+        self._log.debug("Body:\n%s" % self.body)
+
+        # If it is a single line, then we have a spec to load
+        # Otherwise, we have an inline task
+
+        if self.body.find("\n") == -1:
+            # Two forms:
+            # <path>::method
+            # <method-path>
+
+            ci = self.body.find("::")
+            if ci != -1:
+                # have a file to load
+                file = self.body[:ci]
+                method = self.body[ci+2:]
+                spec = importlib.util.spec_from_file_location(input.name, file)
+                module = importlib.util.module_from_spec(spec)
+                sys.modules[input.name] = module
+                spec.loader.exec_module(module)
+
+                callable = getattr(module, method)
+                pass
+            else:
+            #     self._log.debug("Use PyTask implementation")
+                last_dot = self.body.rfind('.')
+                clsname = self.body[last_dot+1:]
+                modname = self.body[:last_dot]
+
+                try:
+                    if modname not in sys.modules:
+                        if input.srcdir not in sys.path:
+                           sys.path.append(input.srcdir)
+                        mod = importlib.import_module(modname)
+                    else:
+                        mod = sys.modules[modname]
+                except ModuleNotFoundError as e:
+                    raise Exception("Failed to import module %s: %s" % (modname, str(e)))
+                
+                if not hasattr(mod, clsname):
+                    raise Exception("Method %s not found in module %s" % (clsname, modname))
+                callable = getattr(mod, clsname)
+        else:
+            text_lines = self.body.splitlines()
+
+            least_whitespace = 2^32
+            have_content = False
+            for line in text_lines:
+                line_no_leading_ws = line.lstrip()
+                if line_no_leading_ws != "":
+                    have_content = True
+                    leading_ws = len(line) - len(line_no_leading_ws)
+                    if leading_ws < least_whitespace:
+                        least_whitespace = leading_ws
+            # Remove leading whitespace
+            if have_content:
+                for i,line in enumerate(text_lines):
+                    if len(line) >= least_whitespace:
+                        text_lines[i] = line[least_whitespace:]
+
+            method = "def pytask(ctxt, input):\n" + "\n".join(["    %s" % l for l in text_lines])
+
+            exec(method)
+
+            callable = locals()['pytask']
+
+        result = callable(ctxt, input)
+
+        self._log.debug("<-- ExecCallable")
+        return result

dv_flow/mgr/package_loader.py

@@ -13,7 +13,7 @@ from .package import Package
 from .param_def import ComplexType
 from .ext_rgy import ExtRgy
 from .srcinfo import SrcInfo
-from .task import Task
+from .task import Task, Strategy, StrategyGenerate
 from .task_def import TaskDef, PassthroughE, ConsumesE, RundirE
 from .task_data import TaskMarker, TaskMarkerLoc, SeverityE
 from .type import Type
@@ -626,6 +626,20 @@ class PackageLoader(object):
                     raise Exception("Failed to find task %s" % need)
                 task.needs.append(nt)
 
+            if taskdef.strategy is not None:
+                self._log.debug("Task %s strategy: %s" % (task.name, str(taskdef.strategy)))
+                if taskdef.strategy.generate is not None:
+                    shell = taskdef.strategy.generate.shell
+                    if shell is None:
+                        shell = "pytask"
+                    task.strategy = Strategy(
+                        generate=StrategyGenerate(
+                            shell=shell,
+                            run=taskdef.strategy.generate.run))
+
+#                task.strategy = taskdef.strategy
+
+
             # Determine how to implement this task
             if taskdef.body is not None and len(taskdef.body) > 0:
                 self._mkTaskBody(task, taskdef)

dv_flow/mgr/task.py

@@ -1,5 +1,5 @@
 import dataclasses as dc
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any, Callable, Dict, List, Tuple, Union
 from .srcinfo import SrcInfo
 from .task_def import TaskDef, RundirE, PassthroughE, ConsumesE
 from .task_node_ctor import TaskNodeCtor
@@ -9,6 +9,16 @@ class Need(object):
     task : 'Task'
     cond : str = None
 
+@dc.dataclass
+class StrategyGenerate(object):
+    shell : str = "pytask"
+    run : str = None
+
+@dc.dataclass
+class Strategy(object):
+    generate : StrategyGenerate = None
+    matrix : Dict[str, List[Any]] = dc.field(default_factory=dict)
+
 @dc.dataclass
 class Task(object):
     """
@@ -31,6 +41,7 @@ class Task(object):
     rundir : RundirE = None
     # TODO: strategy / matrix
     subtasks : List['Task'] = dc.field(default_factory=list)
+    strategy : Strategy = dc.field(default=None)
     run : str = None
     shell : str = "bash"
     srcinfo : SrcInfo = None

dv_flow/mgr/task_def.py

@@ -50,8 +50,13 @@ class PassthroughE(enum.Enum):
     All = "all"
     Unused = "unused"
 
+class GenerateSpec(BaseModel):
+    shell: Union[str, None] = dc.Field(default=None)
+    run: str
+
 class StrategyDef(BaseModel):
     chain: Union[bool, None] = dc.Field(default=None)
+    generate: Union[GenerateSpec, None] = dc.Field(default=None)
     matrix : Union[Dict[str,List[Any]],None] = dc.Field(
         default=None,
         description="Matrix of parameter values to explore")

dv_flow/mgr/task_gen_ctxt.py

@@ -0,0 +1,35 @@
+
+import dataclasses as dc
+from typing import Any, List
+from .task_node import TaskNode
+
+@dc.dataclass
+class TaskGenInputData(object):
+    params : Any
+
+@dc.dataclass
+class TaskGenCtxt(object):
+    rundir : str
+    input : TaskNode
+    builder : 'TaskGraphBuilder'
+    body : List['Task'] = dc.field(default_factory=list)
+    tasks : List['TaskNode'] = dc.field(default_factory=list)
+
+    def mkTaskNode(self, type_t, name=None, srcdir=None, needs=None, **kwargs):
+        return self.builder.mkTaskNode(type_t, name, srcdir, needs, **kwargs)
+
+    def addTask(self, task : 'TaskNode'):
+        if task is None:
+            raise Exception("Task is None")
+        self.tasks.append(task)
+
+    def mkName(self, leaf):
+        # TODO: add on context
+        return leaf
+
+    def marker(self, m):
+        pass
+
+    def error(self, msg):
+        pass
+

dv_flow/mgr/task_graph_builder.py

@@ -29,10 +29,12 @@ from .package_def import PackageDef, PackageSpec
 from .package_loader import PackageLoader
 from .param_ref_eval import ParamRefEval
 from .name_resolution import NameResolutionContext, TaskNameResolutionScope
+from .exec_gen_callable import ExecGenCallable
 from .ext_rgy import ExtRgy
 from .task import Task
 from .task_def import RundirE
 from .task_data import TaskMarker, TaskMarkerLoc, SeverityE
+from .task_gen_ctxt import TaskGenCtxt, TaskGenInputData
 from .task_node import TaskNode
 from .task_node_ctor import TaskNodeCtor
 from .task_node_compound import TaskNodeCompound
@@ -480,22 +482,25 @@ class TaskGraphBuilder(object):
 
         # Determine how to build this node
         if iff:
-            if self._isCompound(task):
-                ret = self._mkTaskCompoundNode(
-                    task, 
-                    name=name,
-                    srcdir=srcdir,
-                    params=params,
-                    hierarchical=hierarchical,
-                    eval=eval)
+            if task.strategy is not None:
+                ret = self._applyStrategy(task, name, srcdir, params, hierarchical, eval)
             else:
-                ret = self._mkTaskLeafNode(
-                    task, 
-                    name=name,
-                    srcdir=srcdir,
-                    params=params,
-                    hierarchical=hierarchical,
-                    eval=eval)
+                if self._isCompound(task):
+                    ret = self._mkTaskCompoundNode(
+                        task, 
+                        name=name,
+                        srcdir=srcdir,
+                        params=params,
+                        hierarchical=hierarchical,
+                        eval=eval)
+                else:
+                    ret = self._mkTaskLeafNode(
+                        task, 
+                        name=name,
+                        srcdir=srcdir,
+                        params=params,
+                        hierarchical=hierarchical,
+                        eval=eval)
         else:
             if name is None:
                 name = task.name
@@ -524,6 +529,105 @@ class TaskGraphBuilder(object):
 
         return ret        
     
+    def _applyStrategy(self, task, name, srcdir, params, hierarchical, eval):
+        self._log.debug("--> _applyStrategy %s" % task.name)
+
+        if name is None:
+            name = task.name
+
+        if srcdir is None:
+            srcdir = os.path.dirname(task.srcinfo.file)
+
+        if params is None:
+            params = task.paramT()
+
+        ret = TaskNodeCompound(
+            name=name,
+            srcdir=srcdir,
+            params=params,
+            ctxt=self._ctxt)
+
+        if ret.input is None:
+            raise Exception("Task %s does not have an input" % task.name)
+
+        self._gatherNeeds(task, ret)
+
+        ctxt = TaskGenCtxt(
+            rundir=self.get_rundir(),
+            input=ret.input,
+            builder=self
+        )
+
+        # In both cases, the result 'lives' inside a compound task
+
+        res = None
+        if task.strategy.generate is not None:
+            callable = ExecGenCallable(body=task.strategy.generate.run)
+            input = TaskGenInputData(params=params)
+
+            res = callable(ctxt, input)
+        elif len(task.strategy.matrix):
+            matrix = {}
+            matrix_items = []
+            for k in task.strategy.matrix.keys():
+                matrix[k] = None
+                matrix_items.append((k, task.strategy.matrix[k]))
+
+            res = self._applyStrategyMatrix(task.tasks, matrix_items, 0)
+            
+            pass
+
+
+#        tasks = [ret.input]
+        tasks = []
+        tasks.extend(ret.tasks[1:])
+
+        tasks.extend(ctxt.tasks.copy())
+        if res is not None:
+            if isinstance(res, list):
+                tasks.extend(res)
+            else:
+                tasks.append(res)
+
+        # Finish hooking this up...
+        for tn in tasks:
+            if tn is None:
+                raise Exception("Generator yielded a null class")
+            referenced = None
+            for tt in tasks:
+                for tnn,_ in tt.needs:
+                    if tn == tnn:
+                        referenced = tnn
+                        break
+
+            refs_internal = None
+            for nn,_ in tn.first.needs:
+                for tnn in tasks:
+                    if nn == tnn:
+                        refs_internal = tnn
+                        break
+                if refs_internal is not None:
+                    break
+            
+            if not refs_internal:
+                if ret.input is None:
+                    raise Exception("Adding None input")
+                if tn == ret.input:
+                    raise Exception("Adding input to itself")
+                tn.needs.append((ret.input, False))
+            
+            if referenced is None:
+                if tn is None:
+                    raise Exception("Adding None input")
+                ret.needs.append((tn, False))
+
+        self._log.debug("<-- _applyStrategy %s" % task.name)
+        return ret
+    
+    def _applyStrategyMatrix(self, tasks, matrix_items, idx):
+        raise Exception("_applyStrategyMatrix not implemented")
+
+    
     def _isCompound(self, task):
         if isinstance(task, Task):
             if task.subtasks is not None and len(task.subtasks):

dv_flow/mgr/task_node_leaf.py

@@ -49,6 +49,10 @@ class TaskNodeLeaf(TaskNode):
         for i,(need,block) in enumerate(self.needs):
             self._log.debug("dep %s dep_m: %s" % (need.name, str(dep_m)))
             if not block:
+                if need.output is None:
+                    raise Exception("Need %s has no output (in %s)" % (
+                        need.name,
+                        self.name))
                 for subdep in need.output.dep_m.keys():
                     if subdep not in dep_m.keys():
                         dep_m[subdep] = []

{dv_flow_mgr-1.9.0.dist-info → dv_flow_mgr-1.9.14960981876rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dv-flow-mgr
-Version: 1.9.0
+Version: 1.9.14960981876rc0
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-1.9.0.dist-info → dv_flow_mgr-1.9.14960981876rc0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-dv_flow/mgr/__init__.py,sha256=XEEeRw_ZtMVNVh1mpM2XZQWxTclTlRrlh9BLZp09RhE,1301
+dv_flow/mgr/__init__.py,sha256=A_xkZr8PLXmBBnX9KfRO0LLgliSssvdgayAw39syRIU,1315
 dv_flow/mgr/__main__.py,sha256=BogNdBkXhgg05E8_IumNkVoag6WwvfbpiI8346oOtPo,3844
 dv_flow/mgr/cond_def.py,sha256=2ZkzPusqVkN1fFMTvkDl9O_OJLPdD_cK3xzX9J75RMw,343
 dv_flow/mgr/config.py,sha256=b2MVlVVNB0psk8x4bQRAYshkpNJrtyMtV1Ymhmx9AfM,137
@@ -6,6 +6,7 @@ dv_flow/mgr/config_def.py,sha256=N5nBc_2VPrx0pCDyIe-VPhQ7ItnbyJqrak3ZiPEAbtI,104
 dv_flow/mgr/data_callable.py,sha256=-GeMgBUdUEcdAkWsqvUYcpKqUd5BQsFgi6ARflnp2B8,687
 dv_flow/mgr/eval_jq.py,sha256=bRsHtaN51qIOiZK1VJV52W0-vj5VH0nQ7XIFebZi5kI,1129
 dv_flow/mgr/exec_callable.py,sha256=NbhTVqvrTd-vsTi3cJ3iGrpVe8tfAqxFsSopaNXApRE,3117
+dv_flow/mgr/exec_gen_callable.py,sha256=KLAKgu0c6nrEzfy85qeogH64VOFDBa6anslSBajykOE,3039
 dv_flow/mgr/expr_eval.py,sha256=ojHo3U_QGqm0CWVSEGtFJRo3jrjiTKhy_B58yRNxN8g,4814
 dv_flow/mgr/expr_parser.py,sha256=LMLe4RTbgcQVIpuZIi1vowHsKRgRmzKzomGvlJL8kJI,7110
 dv_flow/mgr/ext_rgy.py,sha256=BeHbTjB6KFcI5xhbl_O00YQW7MQ8efCFTD_Y8xf7I44,5651
@@ -20,7 +21,7 @@ dv_flow/mgr/out,sha256=d8GGBi3J43fhdLBlnsUbzBfRe0TD0QTP3nOTz54l2bI,200
 dv_flow/mgr/package.py,sha256=L1ba5xguOZWuyg9s4MfafKgSoIka3SgAQzYAmcV7XgQ,4430
 dv_flow/mgr/package_def.py,sha256=1O1SBJATnw-zeYKRNENgw-RWxZK0hxNQhnncfE_NR-U,5674
 dv_flow/mgr/package_import_spec.py,sha256=aZMpnS9a5NFY76_pYXEuO3-Mkc_xFzy73fdrUe_54Dc,1760
-dv_flow/mgr/package_loader.py,sha256=DUNSolfB-9E0KgmCCEjz6ezWm4xlwgPRpWgxcugFEmM,34771
+dv_flow/mgr/package_loader.py,sha256=jfZj1TuYWmuSEVT8Yon8bC0GWJtdynWwugbgZEJsQKY,35389
 dv_flow/mgr/package_node.py,sha256=CccD2ECiIXy9JBkRR3c7qDxeBiLiPbRT9CqR-Al_niI,214
 dv_flow/mgr/param.py,sha256=kkxMRGf6mPjSZJsjgLKH2vJL62Sn0ZESvjBLkEYOp20,1386
 dv_flow/mgr/param_def.py,sha256=9uO-7UI4OotYixQBo8Nwb2o8FQRFxxJylYCZm9rjt48,1917
@@ -31,10 +32,11 @@ dv_flow/mgr/pytask_callable.py,sha256=JiFN2vq6qanI6FXJBeidwiBSn6t0oC0izPJ01c5QJT
 dv_flow/mgr/root_package.py,sha256=nYLOpZVQEDbY0Ui5X5UNeUHXOvqRENew7B2zpC2V2gQ,136
 dv_flow/mgr/shell_callable.py,sha256=IWb0C4L-v_hfnvnhS3g2rhXqutURex_U-dBJgdNxP7I,302
 dv_flow/mgr/srcinfo.py,sha256=xrfp-relVr7hYNbxOjY5jqX4H0nNMm9HK5AAM6_Piyk,366
-dv_flow/mgr/task.py,sha256=n3v9A9HvgMWNeQhXVB_L2qRjgjounXMs9-bbxtyEau8,2088
+dv_flow/mgr/task.py,sha256=XQ-mVyDGvpU4gB6GmnNeTQhxKqFXiblShh7qd_-SIbM,2387
 dv_flow/mgr/task_data.py,sha256=lN7Iq8YTitEMGG4rZqYQi6Ri2HuPgBQ5oGQbW-63T8c,12436
-dv_flow/mgr/task_def.py,sha256=8NPwtTROfWDkMqcO9mKXV4dw0sC4mCMmnsNuv8uTdTY,5094
-dv_flow/mgr/task_graph_builder.py,sha256=2u5iOa4vMZaAxbImVknMLQLV6TEVbxXzEfsE7d3YXKI,29380
+dv_flow/mgr/task_def.py,sha256=lhiOL7E_TfW7Ytg5qipE6d2cqTkfVqAOs1MZgtCqeLg,5257
+dv_flow/mgr/task_gen_ctxt.py,sha256=1QQqqgaCjJSlzTV0SASpTb69vINFrDq3aqjYCKUkx3s,846
+dv_flow/mgr/task_graph_builder.py,sha256=pqzazJjA36g-ETuvxfEl6JV6035koJJPqSF_L-Egtgg,32716
 dv_flow/mgr/task_graph_dot_writer.py,sha256=qK4Imy9o2_F1aKoU1tJ-qoBHslq2BhSMbdjAUPfpN7I,6009
 dv_flow/mgr/task_listener_log.py,sha256=Ai-6X5BOoGsaNTgnlXEW0-czrjJm7__ShNK501CUmko,4337
 dv_flow/mgr/task_listener_trace.py,sha256=7B-V2YK8uXd_QSGBq_7USmbe5CleVvwumU0CpsmxZvg,6224
@@ -48,7 +50,7 @@ dv_flow/mgr/task_node_ctor_proxy.py,sha256=ViOFJ64JM4-CGFZNl89BghFuKSQ66kZVqSj4v
 dv_flow/mgr/task_node_ctor_task.py,sha256=d49g90TyPCMFR8BuWWqp4ym-MW5vGSdDR0V47Ru28JY,2232
 dv_flow/mgr/task_node_ctor_wrapper.py,sha256=KMSYd4ik2Si7UXhNMeXAPHBwwrFkuhpeTMeYRpWv0hw,3718
 dv_flow/mgr/task_node_ctxt.py,sha256=dQp68ppFENK0zGo7TKaFXHInPCCsFai1yZwoBKDwJ20,166
-dv_flow/mgr/task_node_leaf.py,sha256=od6INBSqMkKZTbCAGJFXzZ3_eD-g7_rt_v3HNa61cQ8,8926
+dv_flow/mgr/task_node_leaf.py,sha256=nULlu-veyPC-PXdoQhahkLgqpdKNzaoD0XUlCw0EuZ4,9109
 dv_flow/mgr/task_output.py,sha256=ZwyvwnYj_gHOEFAEOH3m24Xfc4Cn77hb1j7LkX8_3C4,1086
 dv_flow/mgr/task_params_ctor.py,sha256=qlrzibGAFHmbqOu88jEoh1wOSFHu68Gwdgc259-50e8,1915
 dv_flow/mgr/task_run_ctxt.py,sha256=q88OsuKAHC_GipZ7jBSjbybARsJkgcJhvbKjjj2TweA,3801
@@ -74,9 +76,9 @@ dv_flow/mgr/util/util.py,sha256=BO7iqP_c9ttmXkojq7nKDN-g8wl1_Pco9k-KnrXxjwE,1889
 dv_flow/mgr/util/cmds/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dv_flow/mgr/util/cmds/cmd_schema.py,sha256=IJzZdxCSEgIQ79LpYiM7UqJ9RJ-7yraqmBN2XVgAgXA,1752
 dv_flow/mgr/util/cmds/cmd_workspace.py,sha256=egmaIXpe5L-TePwmcfisfrG6tdiTUWSjqa9Za5WChVs,890
-dv_flow_mgr-1.9.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dv_flow_mgr-1.9.0.dist-info/METADATA,sha256=1BuEuGg1VHbODzmAwS0AxdWZMJva1UWSsaqHiRSrjFk,13322
-dv_flow_mgr-1.9.0.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
-dv_flow_mgr-1.9.0.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
-dv_flow_mgr-1.9.0.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
-dv_flow_mgr-1.9.0.dist-info/RECORD,,
+dv_flow_mgr-1.9.14960981876rc0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-1.9.14960981876rc0.dist-info/METADATA,sha256=0O2YxHZdwX3mB_2MU1YJl_QweoVoTFWlMcem5I1hMec,13335
+dv_flow_mgr-1.9.14960981876rc0.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
+dv_flow_mgr-1.9.14960981876rc0.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
+dv_flow_mgr-1.9.14960981876rc0.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
+dv_flow_mgr-1.9.14960981876rc0.dist-info/RECORD,,