dv-flow-mgr 0.0.1.13667970156a1__py3-none-any.whl → 0.0.1.13689116055a1__py3-none-any.whl

dv_flow/mgr/__main__.py

@@ -25,12 +25,15 @@ from .cmds.cmd_run import CmdRun
 
 def get_parser():
     parser = argparse.ArgumentParser(description='dv_flow_mgr')
-    parser.add_argument("-d", "--debug", 
-                        help="Enable debug",
-                        action="store_true")
-    parser.add_argument("-v", "--verbose", 
-                        help="Enable verbose output",
-                        action="store_true")
+    # parser.add_argument("-d", "--debug", 
+    #                     help="Enable debug",
+    #                     action="store_true")
+    parser.add_argument("--log-level", 
+                        help="Configures debug level [INFO, DEBUG]",
+                        choices=("NONE", "INFO", "DEBUG"))
+    # parser.add_argument("-v", "--verbose", 
+    #                     help="Enable verbose output",
+    #                     action="store_true")
     subparsers = parser.add_subparsers(required=True)
 
     run_parser = subparsers.add_parser('run', help='run a flow')
@@ -43,10 +46,12 @@ def main():
     parser = get_parser()
     args = parser.parse_args()
 
-    if args.debug:
-        logging.basicConfig(level=logging.DEBUG)
-    elif args.verbose:
-        logging.basicConfig(level=logging.INFO)
+    if args.log_level is not None and args.log_level != "NONE":
+        opt_m = {
+            "INFO": logging.INFO,
+            "DEBUG": logging.DEBUG
+        }
+        logging.basicConfig(level=opt_m[args.log_level])
 
     args.func(args)
 

dv_flow/mgr/cmds/cmd_run.py

@@ -6,6 +6,8 @@ from ..task_graph_runner import TaskGraphRunner
 from ..util import loadProjPkgDef
 from ..task_graph_builder import TaskGraphBuilder
 from ..task_graph_runner_local import TaskGraphRunnerLocal
+from ..task_runner import TaskSetRunner
+from ..task_listener_log import TaskListenerLog
 from ..pkg_rgy import PkgRgy
 
 
@@ -57,7 +59,9 @@ class CmdRun(object):
         rundir = os.path.join(pkg.basedir, "rundir")
 
         builder = TaskGraphBuilder(root_pkg=pkg, rundir=rundir)
-        runner = TaskGraphRunnerLocal(rundir)
+        runner = TaskSetRunner(rundir)
+
+        runner.add_listener(TaskListenerLog().event)
 
         tasks = []
 
@@ -69,27 +73,4 @@ class CmdRun(object):
 
         asyncio.run(runner.run(tasks))
 
-#        rgy = PkgRgy.inst()
-#        rgy.registerPackage(pkg)
-
-
-        # srcdir = os.getcwd()
-
-        # session = Session(srcdir, rundir)
-
-        # package = session.load(srcdir)
-
-        # graphs = []
-        # for task in args.tasks:
-        #     if task.find(".") == -1:
-        #         task = package.name + "." + task
-        #     subgraph = session.mkTaskGraph(task)
-        #     graphs.append(subgraph)
-
-        # awaitables = [subgraph.do_run() for subgraph in graphs]
-        # print("%d awaitables" % len(awaitables))
-
-        # out = asyncio.get_event_loop().run_until_complete(asyncio.gather(*awaitables))
-
-        # print("out: %s" % str(out))
 

dv_flow/mgr/package_def.py

@@ -134,7 +134,7 @@ class PackageDef(BaseModel):
         ctor_t : TaskCtor = None
         base_params : BaseModel = None
         callable = None
-        passthrough = False
+        passthrough = task.passthrough
         needs = [] if task.needs is None else task.needs.copy()
 
         if task.uses is not None:
@@ -142,6 +142,11 @@ class PackageDef(BaseModel):
             base_ctor_t = self.getTaskCtor(session, task.uses, tasks_m)
             base_params = base_ctor_t.mkTaskParams()
 
+            if base_ctor_t is None:
+                self._log.error("Failed to load task ctor %s" % task.uses)
+        else:
+            self._log.debug("No 'uses' specified")
+
         # Determine the implementation constructor first
         if task.pytask is not None:
             # Built-in impl
@@ -376,7 +381,7 @@ class PackageDef(BaseModel):
 
         with open(file, "r") as fp:
             doc = yaml.load(fp, Loader=yaml.FullLoader)
-            PackageDef._log.debug("doc: %s" % str(doc), flush=True)
+            PackageDef._log.debug("doc: %s" % str(doc))
             if "fragment" in doc.keys():
                 # Merge the package definition
                 frag = FragmentDef(**(doc["fragment"]))

dv_flow/mgr/std/message.py

@@ -2,5 +2,5 @@
 from dv_flow.mgr import Task, TaskDataResult
 
 async def Message(runner, input) -> TaskDataResult:
-        print("%s: %s" % (input.name, input.params.msg))
+        print("%s: %s" % (input.name, input.params.msg), flush=True)
         return TaskDataResult()

dv_flow/mgr/task_graph_builder.py

@@ -179,14 +179,18 @@ class TaskGraphBuilder(object):
 
         return pkg
     
-    def mkTaskNode(self, typename, name=None, srcdir=None, needs=None, **kwargs):
-        ctor = self.getTaskCtor(typename)
-        params = ctor.mkTaskParams(**kwargs)
-        return ctor.mkTaskNode(
+    def mkTaskNode(self, task_t, name=None, srcdir=None, needs=None, **kwargs):
+        self._logger.debug("--> mkTaskNode: %s" % task_t)
+        ctor = self.getTaskCtor(task_t)
+        self._logger.debug("ctor: %s" % ctor.name)
+        params = ctor.mkTaskParams(kwargs)
+        ret = ctor.mkTaskNode(
             params=params,
             name=name, 
             srcdir=srcdir, 
             needs=needs)
+        self._logger.debug("<-- mkTaskNode: %s" % task_t)
+        return ret
         
     def getTaskCtor(self, spec : Union[str,'TaskSpec'], pkg : PackageDef = None) -> 'TaskCtor':
         from .task_def import TaskSpec

dv_flow/mgr/task_listener_log.py

@@ -5,15 +5,21 @@ from rich.console import Console
 class TaskListenerLog(object):
     console : Console = dc.field(default_factory=Console)
     level : int = 0
+    quiet : bool = True
 
     def event(self, task : 'Task', reason : 'Reason'):
         if reason == 'enter':
             self.level += 1
-            self.console.print("[green]>[%d][/green] Task %s" % (self.level, task.name))
+            if not self.quiet:
+                self.console.print("[green]>[%d][/green] Task %s" % (self.level, task.name))
         elif reason == 'leave':
             for m in task.result.markers:
                 print("  %s" % m)
-            self.console.print("[green]<[%d][/green] Task %s" % (self.level, task.name))
+            if self.quiet:
+                if task.result.changed:
+                    self.console.print("[green]Done:[/green] %s" % (task.name,))
+            else:
+                self.console.print("[green]<[%d][/green] Task %s" % (self.level, task.name))
             self.level -= 1
         else:
             self.console.print("[red]-[/red] Task %s" % task.name)

dv_flow/mgr/task_node.py

@@ -3,6 +3,7 @@ import sys
 import dataclasses as dc
 import pydantic.dataclasses as pdc
 import logging
+import toposort
 from typing import Any, Callable, ClassVar, Dict, List
 from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
 from .task_params_ctor import TaskParamsCtor
@@ -38,19 +39,46 @@ class TaskNode(object):
                   runner,
                   rundir,
                   memento : Any = None) -> 'TaskDataResult':
+        self._log.debug("--> do_run: %s" % self.name)
         changed = False
         for dep in self.needs:
             changed |= dep.changed
 
         # TODO: Form dep-map from inputs
-        # TODO: Order param sets according to dep-map
-        in_params = []
+
+        dep_m = {}
+        for need in self.needs:
+            self._log.debug("dep %s dep_m: %s" % (need.name, str(dep_m)))
+            for subdep in need.output.dep_m.keys():
+                if subdep not in dep_m.keys():
+                    dep_m[subdep] = []
+                dep_m[subdep].extend(need.output.dep_m[subdep])
+
+        self._log.debug("input dep_m: %s" % str(dep_m))
+        sorted = toposort.toposort(dep_m)
+
+        in_params_m = {}
         for need in self.needs:
-            in_params.extend(need.output.output)
+            for p in need.output.output:
+                if p.src not in in_params_m.keys():
+                    in_params_m[p.src] = []
+                in_params_m[p.src].append(p)
+
+        # in_params holds parameter sets ordered by dependency
+        in_params = []
+        for sorted_s in sorted:
+            self._log.debug("sorted_s: %s" % str(sorted_s))
+            for dep in sorted_s:
+                if dep in in_params_m.keys():
+                    self._log.debug("(%s) Extend with: %s" % (dep, str(in_params_m[dep])))
+                    in_params.extend(in_params_m[dep])
+
+        self._log.debug("in_params[1]: %s" % ",".join(p.src for p in in_params))
 
         # Create an evaluator for substituting param values
         eval = ParamRefEval()
 
+        self._log.debug("in_params[2]: %s" % ",".join(p.src for p in in_params))
         eval.setVar("in", in_params)
 
         for name,field in self.params.model_fields.items():
@@ -58,6 +86,7 @@ class TaskNode(object):
             if type(value) == str:
                 if value.find("${{") != -1:
                     new_val = eval.eval(value)
+                    self._log.debug("Param %s: Evaluate expression \"%s\" => \"%s\"" % (name, value, new_val))
                     setattr(self.params, name, new_val)
             elif isinstance(value, list):
                 for i,elem in enumerate(value):
@@ -75,20 +104,40 @@ class TaskNode(object):
             params=self.params,
             memento=memento)
 
-        # TODO: notify of task start
+        self._log.debug("--> Call task method %s" % str(self.task))
         self.result : TaskDataResult = await self.task(self, input)
-        # TODO: notify of task complete
+        self._log.debug("<-- Call task method %s" % str(self.task))
 
-        # TODO: form a dep map from the outgoing param sets
-        dep_m = {}
+        output=self.result.output.copy()
+        for out in output:
+            out.src = self.name
+
+        self._log.debug("output[1]: %s" % str(output))
+
+        if self.passthrough:
+            self._log.debug("passthrough: %s" % self.name)
+            # Add an entry for ourselves
+            dep_m[self.name] = list(need.name for need in self.needs)
+
+            for need in self.needs:
+                output.extend(need.output.output)
+        else:
+            # empty dependency map
+            dep_m = {
+                self.name : []
+            }
+
+        self._log.debug("output dep_m: %s" % str(dep_m))
+        self._log.debug("output[2]: %s" % str(output))
 
         # Store the result
         self.output = TaskDataOutput(
             changed=self.result.changed,
             dep_m=dep_m,
-            output=self.result.output.copy())
+            output=output)
 
         # TODO: 
+        self._log.debug("<-- do_run: %s" % self.name)
 
         return self.result
 

dv_flow/mgr/task_output.py

@@ -1,6 +1,7 @@
 from pydantic import BaseModel, Field
-from typing import Any, List
+from typing import Any, Dict, List
 
 class TaskOutput(BaseModel):
     type : str
-    params : List[Any] = Field(default_factory=list, alias="with")
+    params : List[Any] = Field(default_factory=list, alias="with")
+    deps : Dict[str,List[str]] = Field(default_factory=dict)

{dv_flow_mgr-0.0.1.13667970156a1.dist-info → dv_flow_mgr-0.0.1.13689116055a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.13667970156a1
+Version: 0.0.1.13689116055a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.13667970156a1.dist-info → dv_flow_mgr-0.0.1.13689116055a1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 dv_flow/mgr/__init__.py,sha256=2iTpSe1M8g_u2UlAQFa3XFfqcw8gBOWdBvrstce1c5E,403
-dv_flow/mgr/__main__.py,sha256=uik6gDAvtZNG0yyDKYc4FTl6R1QHAR543jNG2lCAa6E,1771
+dv_flow/mgr/__main__.py,sha256=_qiAyACu_Ugo64EIQDWmC0v92nHeaZCxequXTKoCRyo,2030
 dv_flow/mgr/eval_jq.py,sha256=Ue-qkyNW6uOu2Wy8u6nrTbPPY7ujaHd0-4iki0eV3Ec,294
 dv_flow/mgr/expr_eval.py,sha256=Mp0TvrV6gZWcj1uYwLfPNI-ARwwdAfwntE5byLBfBuY,2276
 dv_flow/mgr/expr_parser.py,sha256=FhBTkKKkJXubGHUzmcMWubPfPklYLGzVz1jK4-dYb44,5702
@@ -7,7 +7,7 @@ dv_flow/mgr/fileset.py,sha256=ewyjKLzuOecZg0vtabN_UkbzNEnk1sXgrH8uDyeKB60,1184
 dv_flow/mgr/fragment_def.py,sha256=cyzp1XeWtNOaagScmeS-BPsoXj9j2LTBbKq5ZUioz8I,1641
 dv_flow/mgr/out,sha256=d8GGBi3J43fhdLBlnsUbzBfRe0TD0QTP3nOTz54l2bI,200
 dv_flow/mgr/package.py,sha256=878twhPD-E1pFlDNUtuyeFEgJ_Y89b560og4St-Iwrs,1679
-dv_flow/mgr/package_def.py,sha256=ek60ERvWFOVlL71_Fgk1-RWQMrkxNt4USrfZXwLS2d0,14603
+dv_flow/mgr/package_def.py,sha256=EaDPLljg6iewOuZwUgAF4lTkLH9fYAgigD2klXaKBRE,14779
 dv_flow/mgr/package_import_spec.py,sha256=ah3r15v5Jdub2poc3sgi6Uar1L3oGoYsCPPNiOHV-a4,1760
 dv_flow/mgr/param.py,sha256=3BY-ucig6JRw73FhjyJQL-vpd57qhAzixgZ8I5FoUpw,553
 dv_flow/mgr/param_def.py,sha256=gLua-EQiY8V2CFX-2svLRIlrs8PEeGh4-EPtn4a2Mng,712
@@ -19,29 +19,29 @@ dv_flow/mgr/task_ctor.py,sha256=hlfl-UVvyjzLFN6D0Oel9eBs0xUQPqCX7gQ0uEHoL7o,1382
 dv_flow/mgr/task_data.py,sha256=F0X-e0IHTh06Z0F67TWIUPKuJYf6JPzZyMPrDzOJvK0,11409
 dv_flow/mgr/task_def.py,sha256=T06dvQegPJunH8ekuN5aPxIVcaUWiv5qkfqbZnCacLk,1923
 dv_flow/mgr/task_exec_data.py,sha256=aT__kmVmNxqnS_GbTRS1vrYgKiv8cH-HUSmRb6YYKsI,640
-dv_flow/mgr/task_graph_builder.py,sha256=okqJDDu6qyubXQoyI-r0jUTd53Tq7ISSYCymZh_Cppw,7708
+dv_flow/mgr/task_graph_builder.py,sha256=fEqJalmDDDzayJbuyDWAcmumDGK3XgnZmcScoBY8HGg,7887
 dv_flow/mgr/task_graph_runner.py,sha256=jUGI49QvxUCfQoKQDDk2psbeapIcCg72qNOW1JipHzM,2182
 dv_flow/mgr/task_graph_runner_local.py,sha256=OrydPwtQ8E7hYWvSXx0h7lI3nfUNFyklULhsyMwz9dA,4687
 dv_flow/mgr/task_impl_data.py,sha256=bFPijoKrh9x7fZN2DsvRJp0UHo-gGM0VjtDQISyfhFk,321
-dv_flow/mgr/task_listener_log.py,sha256=5N-WG6ii-R6XPbgPR6zcfKjABOoGYab6I_04Lfwux5Q,705
+dv_flow/mgr/task_listener_log.py,sha256=u7hm8Q8ZVCIY25mNQEI3DIgSMJ_zJ5nbvqukGMXyido,934
 dv_flow/mgr/task_memento.py,sha256=C7VTQpBhDEoYuDmE6YTM-6TLMLnqHp6Y0Vat1aTgtCs,1096
-dv_flow/mgr/task_node.py,sha256=BxkjUJU7WdIhyBbP7FMBFY6EgHenhnnYGGdGW3Xukck,8450
-dv_flow/mgr/task_output.py,sha256=l-W-FvVo6YDah1RQS-I9N0KUtB3vp-kl7lxIdmNz0l4,178
+dv_flow/mgr/task_node.py,sha256=5R51VLuHDFnY3c5E8kLccxFRoshJRB1bDKJXIJr9EKc,10394
+dv_flow/mgr/task_output.py,sha256=dkJhhNRFGFQSkVsw_bGirK0_0ghTxBYq1LjRMZCVWnA,245
 dv_flow/mgr/task_params_ctor.py,sha256=aXgB8o9xFPjaEjGW_xYkEC0N0apzGzGUPDj7g2ZLvus,1112
 dv_flow/mgr/task_runner.py,sha256=Ky10BWKy_NhvJp2nsbB0pyS8n1K0pKatBEmTdD1ZKOc,7071
 dv_flow/mgr/type.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dv_flow/mgr/type_def.py,sha256=KdhuNlfw-NKU-4VZFCnMPyj775yEB7cpr5tz73a9yuQ,259
 dv_flow/mgr/util.py,sha256=06eVyURF4ga-s8C9Sd3ZSDebwO4QS0XXaB8xADVbWRc,1437
-dv_flow/mgr/cmds/cmd_run.py,sha256=eths8kT7mBmpZqwOuMtpKAaux4rg-f7hPBxxTHbpKT4,2903
+dv_flow/mgr/cmds/cmd_run.py,sha256=r5j-T121oXYDyOYfrCZSjzeblp1awiAn17HkPtLaGg4,2372
 dv_flow/mgr/share/flow.json,sha256=lNmZex9NXkYbyb2aZseQfUOkV9CMyfH0iLODEI7EPBw,5096
 dv_flow/mgr/std/exec.py,sha256=ETx9xSxhdCD_iw6pcmhrafDCJ-41AneyEAPwQf3q-3w,452
 dv_flow/mgr/std/fileset.py,sha256=abHgTxz25FD1ep5QXlCZKt4gCyfplJsjx6LyfQ11f_Q,2683
 dv_flow/mgr/std/flow.dv,sha256=jlFOh3xVECOzHws7x6YvJ9eCIGHM5gsPeEnheiGOukY,1553
-dv_flow/mgr/std/message.py,sha256=CWrBKImbXKe2d7hJ223U3Ifuxo54zLpFPJviE8BUJvk,188
+dv_flow/mgr/std/message.py,sha256=T8Ye44784rD6CT2qIAP5SD7sKPdAlGMFimc4IaPCxxE,200
 dv_flow/mgr/std/task_null.py,sha256=UKwUnqwFPBY8BO44ZAPcgehQB59kHZFa1qyZc1TwUqE,196
-dv_flow_mgr-0.0.1.13667970156a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dv_flow_mgr-0.0.1.13667970156a1.dist-info/METADATA,sha256=KkpZBeRE5Av-3wHTQewdBoyjAtM3voZTDxeXsjrsjhs,13276
-dv_flow_mgr-0.0.1.13667970156a1.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-dv_flow_mgr-0.0.1.13667970156a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
-dv_flow_mgr-0.0.1.13667970156a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
-dv_flow_mgr-0.0.1.13667970156a1.dist-info/RECORD,,
+dv_flow_mgr-0.0.1.13689116055a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-0.0.1.13689116055a1.dist-info/METADATA,sha256=fuUi6tX_RwxMXl0XsApdKkyUlKyKOgYaouavIevi0w4,13276
+dv_flow_mgr-0.0.1.13689116055a1.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+dv_flow_mgr-0.0.1.13689116055a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
+dv_flow_mgr-0.0.1.13689116055a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
+dv_flow_mgr-0.0.1.13689116055a1.dist-info/RECORD,,