dv-flow-mgr 0.0.1.12849118090a1__py3-none-any.whl → 0.0.1.12911707440a1__py3-none-any.whl

dv_flow/mgr/task_graph_builder.py

@@ -0,0 +1,190 @@
+import os
+import dataclasses as dc
+from .package import Package
+from .package_def import PackageDef, PackageSpec
+from .pkg_rgy import PkgRgy
+from .task import Task, TaskCtor, TaskSpec
+from typing import Dict, List
+
+@dc.dataclass
+class TaskGraphBuilder(object):
+    """The Task-Graph Builder knows how to discover packages and construct task graphs"""
+    root_pkg : PackageDef
+    rundir : str
+    pkg_rgy : PkgRgy = None
+    _pkg_s : List[Package] = dc.field(default_factory=list)
+    _pkg_m : Dict[PackageSpec,Package] = dc.field(default_factory=dict)
+    _pkg_def_m : Dict[str,PackageDef] = dc.field(default_factory=dict)
+    _pkg_spec_s : List[PackageDef] = dc.field(default_factory=list)
+    _task_m : Dict[TaskSpec,Task] = dc.field(default_factory=dict)
+
+    def __post_init__(self):
+        if self.pkg_rgy is None:
+            self.pkg_rgy = PkgRgy.inst()
+
+        if self.root_pkg is not None:
+            self._pkg_spec_s.append(self.root_pkg)
+            pkg = self.root_pkg.mkPackage(self)
+            self._pkg_spec_s.pop()
+            self._pkg_m[PackageSpec(self.root_pkg.name)] = pkg
+
+    def push_package(self, pkg : Package):
+        self._pkg_s.append(pkg)
+
+    def pop_package(self, pkg : Package):
+        self._pkg_s.pop()
+
+    def package(self):
+        return self._pkg_s[-1]
+
+    def mkTaskGraph(self, task : str) -> Task:
+        self._pkg_s.clear()
+        self._task_m.clear()
+
+        return self._mkTaskGraph(task, self.rundir)
+        
+    def _mkTaskGraph(self, task : str, parent_rundir : str) -> Task:
+
+        elems = task.split(".")
+
+        pkg_name = ".".join(elems[0:-1])
+        task_name = elems[-1]
+
+        if pkg_name == "":
+            if len(self._pkg_spec_s) == 0:
+                raise Exception("No package context for %s" % task)
+            pkg_spec = self._pkg_spec_s[-1]
+            pkg_name = pkg_spec.name
+        else:
+            pkg_spec = PackageSpec(pkg_name)
+
+        rundir = os.path.join(parent_rundir, pkg_name, task_name)
+
+        print("pkg_spec: %s" % str(pkg_spec))
+        self._pkg_spec_s.append(pkg_spec)
+        pkg = self.getPackage(pkg_spec)
+        
+        self._pkg_s.append(pkg)
+
+        ctor_t : TaskCtor = pkg.getTaskCtor(task_name)
+
+        depends = []
+
+        # The returned task should have all param references resolved
+        print("task_ctor=%s" % str(ctor_t.task_ctor), flush=True)
+        task = ctor_t.task_ctor(
+            name=task_name,
+            session=self,
+            params=ctor_t.mkParams(),
+            depends=depends,
+            rundir=rundir,
+            srcdir=ctor_t.srcdir)
+        
+        for i,d in enumerate(task.depend_refs):
+            if d in self._task_m.keys():
+                task.depends.append(self._task_m[d])
+            else:
+                print("mkTaskGraph: %s" % d)
+                task.depends.append(self._mkTaskGraph(d, parent_rundir))
+
+        self._task_m[task.name] = task
+
+        self._pkg_s.pop()
+        self._pkg_spec_s.pop()
+
+        return task
+
+    def getPackage(self, spec : PackageSpec) -> Package:
+        # Obtain the active package definition
+        print("getPackage: %s len: %d" % (spec.name, len(self._pkg_spec_s)))
+        if len(self._pkg_spec_s) > 0:
+            pkg_spec = self._pkg_spec_s[-1]
+            print("pkg_spec: %s ; root_pkg: %s" % (pkg_spec.name, self.root_pkg.name))
+            if self.root_pkg.name == pkg_spec.name:
+                pkg_def = self.root_pkg
+            else:
+                pkg_def = self.pkg_rgy.getPackage(pkg_spec.name)
+        else:
+            pkg_def = None
+
+        print("spec: %s ; _pkg_def_m: %s" % (str(spec), str(self._pkg_def_m.keys())))
+
+        # Need a stack to track which package we are currently in
+        # Need a map to get a concrete package from a name with parameterization
+
+        print("pkg_s: %d %s" % (len(self._pkg_s), (self._pkg_s[-1].name if len(self._pkg_s) else "<unknown>")))
+
+        # Note: _pkg_m needs to be context specific, such that imports from
+        # one package don't end up visible in another
+        if len(self._pkg_s) and spec.name == self._pkg_s[-1].name:
+            pkg = self._pkg_s[-1]
+        if spec in self._pkg_m.keys():
+            pkg = self._pkg_m[spec]
+        elif spec in self._pkg_def_m.keys():
+            pkg = self._pkg_def_m[spec].mkPackage(self)
+            self._pkg_m[spec] = pkg
+        else:
+            pkg = None
+
+            if pkg_def is not None:
+                # Look for an import alias
+                print("imports: %s" % str(pkg_def.imports))
+                for imp in pkg_def.imports:
+                    print("imp: %s" % str(imp))
+                    if imp.alias is not None and imp.alias == spec.name:
+                        # Found the alias name. Just need to get an instance of this package
+                        tgt_pkg_spec = PackageSpec(imp.name)
+                        if tgt_pkg_spec in self._pkg_m.keys():
+                            pkg = self._pkg_m[tgt_pkg_spec]
+                        elif tgt_pkg_spec in self._pkg_def_m.keys():
+                            base = self._pkg_def_m[tgt_pkg_spec]
+                            pkg = base.mkPackage(self, spec.params)
+                            self._pkg_m[spec] = pkg
+                        elif imp.path is not None:
+                            # See if we can load the package
+                            print("TODO: load referenced package")
+                        else:
+                            raise Exception("Import alias %s not found" % imp.name)
+                        break
+                    else:
+                        # Need to compare the spec with the full import spec
+                        imp_spec = PackageSpec(imp.name)
+                        # TODO: set parameters
+                        if imp_spec == spec:
+                            base = self._pkg_def_m[PackageSpec(spec.name)]
+                            pkg = base.mkPackage(self, spec.params)
+                            self._pkg_m[spec] = pkg
+                            break
+            
+            if pkg is None:
+                print("Checking registry")
+                p_def =  self.pkg_rgy.getPackage(spec.name)
+
+                if p_def is not None:
+                    pkg = p_def.mkPackage(self)
+
+            if pkg is None:
+                raise Exception("Failed to find package %s from package %s" % (
+                    spec.name, (pkg_def.name if pkg_def is not None else "<null>")))
+
+        return pkg
+        
+    def getTaskCtor(self, spec : TaskSpec, pkg : PackageDef = None) -> 'TaskCtor':
+        spec_e = spec.name.split(".")
+        task_name = spec_e[-1]
+
+        if len(spec_e) == 1:
+            # Just have a task name. Use the current package
+            if len(self._pkg_s) == 0:
+                raise Exception("No package context for task %s" % spec.name)
+            pkg = self._pkg_s[-1]
+        else:
+            pkg_name = ".".join(spec_e[0:-1])
+
+            try:
+                pkg = self.getPackage(PackageSpec(pkg_name))
+            except Exception as e:
+                print("Failed to find package %s while looking for task %s" % (pkg_name, spec.name))
+                raise e
+
+        return pkg.getTaskCtor(task_name)

dv_flow/mgr/task_graph_runner.py

@@ -0,0 +1,71 @@
+#****************************************************************************
+#* session.py
+#*
+#* Copyright 2023 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import asyncio
+import os
+import yaml
+import dataclasses as dc
+from typing import Any, Callable, ClassVar, Dict, List
+from .task import Task
+from .task_data import TaskData
+
+@dc.dataclass
+class TaskGraphRunner(object):
+    """Session manages execution of a task graph"""
+
+    rundir : str
+
+    _inst : ClassVar['TaskGraphRunner'] = None
+
+    # Search path for .dfs files
+    create_subprocess : Callable = asyncio.create_subprocess_exec
+    _root_dir : str = None
+
+    async def exec(self, *args, **kwargs):
+        return self.create_subprocess(*args, **kwargs)
+
+    # def load(self):
+    #     if not os.path.isdir(self.srcdir):
+    #         raise Exception("Root directory %s does not exist" % self.srcdir)
+
+    #     if not os.path.isfile(os.path.join(self.srcdir, "flow.dv")):
+    #         raise Exception("No root flow file")
+
+    #     self._root_dir = os.path.dirname(self.srcdir)
+    #     self.package = PackageDef.load(os.path.join(self.srcdir, "flow.dv"), [])
+
+    #     return self.package
+
+
+    async def run(self, task : str) -> 'TaskData':
+        impl = self.mkTaskGraph(task)
+        return await impl.do_run()
+    
+    async def runTask(self, task : Task) -> 'TaskData':
+        return await task.do_run()
+    
+    def queueTask(self, task : Task):
+        """Queue a task for execution"""
+        pass
+
+
+
+

dv_flow/mgr/task_graph_runner_local.py

@@ -0,0 +1,79 @@
+#****************************************************************************
+#* session.py
+#*
+#* Copyright 2023 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import asyncio
+import os
+import yaml
+import dataclasses as dc
+from typing import Any, Callable, ClassVar, Dict, List, Union
+from .fragment_def import FragmentDef
+from .package import Package
+from .pkg_rgy import PkgRgy
+from .package_def import PackageDef, PackageSpec
+from .task import Task, TaskSpec, TaskCtor
+from .task_data import TaskData
+from .task_graph_runner import TaskGraphRunner
+
+@dc.dataclass
+class TaskGraphRunnerLocal(TaskGraphRunner):
+    """Session manages execution of a task graph"""
+
+    rundir : str
+    nproc : int = -1
+    _workers : List = dc.field(default_factory=list)
+
+    _inst : ClassVar['TaskGraphRunner'] = None
+
+    # Search path for .dfs files
+    create_subprocess : Callable = asyncio.create_subprocess_exec
+    _root_dir : str = None
+
+    def __post_init__(self):
+        if self.nproc == -1:
+            self.nproc = os.cpu_count()
+        for _ in range(self.nproc):
+            self._workers.append(LocalRunnerWorker(self))
+
+
+    async def exec(self, *args, **kwargs):
+        return await self.create_subprocess(*args, **kwargs)
+
+    async def run(self, task : Union[Task,List[Task]]) -> List['TaskData']:
+        if isinstance(task, Task):
+            task = [task]
+        
+        run_o = list(t.do_run() for t in task)
+
+        return await asyncio.gather(*run_o)
+    
+    async def runTask(self, task : Task) -> 'TaskData':
+        return await task.do_run()
+    
+    def queueTask(self, task : Task):
+        """Queue a task for execution"""
+        pass
+
+@dc.dataclass
+class LocalRunnerWorker(object):
+    runner : TaskGraphRunnerLocal
+    pass
+
+

dv_flow/mgr/util.py

@@ -0,0 +1,19 @@
+import os
+import yaml
+from .package_def import PackageDef
+
+def loadProjPkgDef(path):
+    """Locates the project's flow spec and returns the PackageDef"""
+
+    dir = path
+    ret = None
+    while dir != "/" and dir != "" and os.path.isdir(dir):
+        if os.path.exists(os.path.join(dir, "flow.dv")):
+            with open(os.path.join(dir, "flow.dv")) as f:
+                data = yaml.load(f, Loader=yaml.FullLoader)
+                if "package" in data.keys():
+                    ret = PackageDef.load(os.path.join(dir, "flow.dv"))
+                    break
+        dir = os.path.dirname(dir)
+    return ret
+

{dv_flow_mgr-0.0.1.12849118090a1.dist-info → dv_flow_mgr-0.0.1.12911707440a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12849118090a1
+Version: 0.0.1.12911707440a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

dv_flow_mgr-0.0.1.12911707440a1.dist-info/RECORD

@@ -0,0 +1,31 @@
+dv_flow/mgr/__init__.py,sha256=Ms7IJnAbW08GwaanCI64ad_QWZd_XPGfua0WhYr57Jg,107
+dv_flow/mgr/__main__.py,sha256=pHXC_c-XSPUYQsBDxk5y7LtQ5kA5pKGhhzu4ko5jh7k,513
+dv_flow/mgr/fileset.py,sha256=FNvC5sU2ArxJ0OO3v8dXTv8zX-bZ5t0a0ljne0fQQ1o,1150
+dv_flow/mgr/flow.py,sha256=UdgJOIqBS2wTRpO-sNWCCqO9oQFxDfGPGVD0r42aTrA,1562
+dv_flow/mgr/fragment_def.py,sha256=p5i6ONtBWlDHTBFsduu3Z36_76Bn8PCIylp_xoZ7jfQ,1552
+dv_flow/mgr/package.py,sha256=k6gaDun9mJeGwGsFP5YOYOaFLmKb3KyPZy5wGRgJd_E,1965
+dv_flow/mgr/package_def.py,sha256=hT8wr4ghH61rjK6VEovqTKVUuO_EwrCB5JQuFR-whiU,11455
+dv_flow/mgr/package_import_spec.py,sha256=bStPa727wAKMcixydVY1Ht6ylzXsSMy2K31HWPXhc9k,921
+dv_flow/mgr/parameters.py,sha256=kUjRss5VtMMz5eL3-Z_M6BS-wFs7MhQu3ANXO54UPo0,896
+dv_flow/mgr/pkg_rgy.py,sha256=y7HOBZAgKHL4ItnFvjT0lWC_ne81qJi4lJGNLt69au0,2581
+dv_flow/mgr/task.py,sha256=gaMlOiV4pKmEBzewTkVle3RN2zfKPkSdYW3oJX0j_Jg,6421
+dv_flow/mgr/task_data.py,sha256=-6Dqa3oUI7RJc1Js2SRSnhxNTcASkamXFYMN6UiknZQ,10376
+dv_flow/mgr/task_def.py,sha256=96hSwqJo0MazJ1VcLhovYRmNCplsNLt47AumtyjSddU,1690
+dv_flow/mgr/task_graph_builder.py,sha256=qx-59f14-oJJi2qx_zYTaP3PvvxxQCNyYqQ6jDjx_OM,7218
+dv_flow/mgr/task_graph_runner.py,sha256=x2e-wpJFhZ4Zkov30PtC8CuKyOK6htA1SKzmrcVBI9M,2154
+dv_flow/mgr/task_graph_runner_local.py,sha256=-69HoOGkEPHL897RJGB7gItjqXDL_8eam267af4G77E,2420
+dv_flow/mgr/task_memento.py,sha256=C7VTQpBhDEoYuDmE6YTM-6TLMLnqHp6Y0Vat1aTgtCs,1096
+dv_flow/mgr/util.py,sha256=WKwMF4vwYdte5wzieSTDpZTmZU0sjQro3Ofi9moCayE,613
+dv_flow/mgr/cmds/cmd_run.py,sha256=PqAbPMwqovaaq14tnNrCvP7-De8lMI09X0R7d6RIbwY,2691
+dv_flow/mgr/share/flow.json,sha256=lNmZex9NXkYbyb2aZseQfUOkV9CMyfH0iLODEI7EPBw,5096
+dv_flow/mgr/std/fileset.py,sha256=qY4RMqTHZaFZk68Y3oXtDv2_Ezu1r4wYvaRvr0GTyIY,2352
+dv_flow/mgr/std/flow.dv,sha256=pSpzrPPEu_L8DHccGfArxsKYgUfyQidShZc0ShgGtsY,500
+dv_flow/mgr/std/message.py,sha256=BPTHnEMD4tBufQ9LvsS9Sa_0xjaJATbBpwqosWslvVA,193
+dv_flow/mgr/std/task_fileset.py,sha256=UzTYONvK0X9rgy3rP9LiX4giBU8SyCCJav0LSNUJ1Qg,3140
+dv_flow/mgr/std/task_null.py,sha256=UEJ3fIoIMYWVsagiQC7GHD23UES7WoH4wtq94b4tcs4,265
+dv_flow_mgr-0.0.1.12911707440a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-0.0.1.12911707440a1.dist-info/METADATA,sha256=tjOZgtwJrZwNzN011TtAKggCWYSik3b60ZXkBNB3MbI,13252
+dv_flow_mgr-0.0.1.12911707440a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+dv_flow_mgr-0.0.1.12911707440a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
+dv_flow_mgr-0.0.1.12911707440a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
+dv_flow_mgr-0.0.1.12911707440a1.dist-info/RECORD,,

dv_flow_mgr-0.0.1.12911707440a1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+dfm = dv_flow.mgr.__main__:main

dv_flow_mgr-0.0.1.12911707440a1.dist-info/top_level.txt

@@ -0,0 +1 @@
+dv_flow

dv_flow_mgr/cmds/cmd_run.py

@@ -1,28 +0,0 @@
-import asyncio
-import os
-from ..session import Session
-
-class CmdRun(object):
-
-    def __call__(self, args):
-        srcdir = os.getcwd()
-        rundir = os.path.join(srcdir, "rundir")
-
-        session = Session(srcdir, rundir)
-
-        package = session.load(srcdir)
-
-        graphs = []
-        for task in args.tasks:
-            if task.find(".") == -1:
-                task = package.name + "." + task
-            subgraph = session.mkTaskGraph(task)
-            graphs.append(subgraph)
-
-        awaitables = [subgraph.do_run() for subgraph in graphs]
-        print("%d awaitables" % len(awaitables))
-
-        out = asyncio.get_event_loop().run_until_complete(asyncio.gather(*awaitables))
-
-        print("out: %s" % str(out))
-