dv-flow-mgr 0.0.1.12849118090a1__py3-none-any.whl → 0.0.1.12919555073a1__py3-none-any.whl

dv_flow/mgr/__init__.py

@@ -0,0 +1,9 @@
+
+from .package_def import *
+from .task_graph_runner import *
+from .task import *
+from .task_data import *
+from .task_graph_runner import TaskGraphRunner
+from .task_graph_runner_local import TaskGraphRunnerLocal
+from .task_graph_builder import TaskGraphBuilder
+

{dv_flow_mgr → dv_flow/mgr}/__main__.py

@@ -7,7 +7,7 @@ def get_parser():
     subparsers = parser.add_subparsers(required=True)
 
     run_parser = subparsers.add_parser('run', help='run a flow')
-    run_parser.add_argument("tasks", nargs='+', help="tasks to run")
+    run_parser.add_argument("tasks", nargs='*', help="tasks to run")
     run_parser.set_defaults(func=CmdRun())
 
     return parser

dv_flow/mgr/cmds/cmd_run.py

@@ -0,0 +1,90 @@
+import asyncio
+import os
+from ..task_graph_runner import TaskGraphRunner
+from ..util import loadProjPkgDef
+from ..task_graph_builder import TaskGraphBuilder
+from ..task_graph_runner_local import TaskGraphRunnerLocal
+from ..pkg_rgy import PkgRgy
+
+
+class CmdRun(object):
+
+    def __call__(self, args):
+
+        # First, find the project we're working with
+        pkg = loadProjPkgDef(os.getcwd())
+
+        if pkg is None:
+            raise Exception("Failed to find a 'flow.dv' file that defines a package in %s or its parent directories" % os.getcwd())
+
+        print("pkg: %s" % pkg.name)
+
+        if len(args.tasks) > 0:
+            pass
+        else:
+            # Print out available tasks
+            tasks = []
+            for task in pkg.tasks:
+                tasks.append(task)
+            for frag in pkg.fragment_l:
+                for task in frag.tasks:
+                    tasks.append(task)
+            tasks.sort(key=lambda x: x.name)
+
+            max_name_len = 0
+            for t in tasks:
+                if len(t.name) > max_name_len:
+                    max_name_len = len(t.name)
+
+            print("No task specified. Available Tasks:")
+            for t in tasks:
+                desc = t.desc
+                if desc is None or t.desc == "":
+                    "<no descripion>"
+                print("%s - %s" % (t.name.ljust(max_name_len), desc))
+
+            pass
+
+        # Create a session around <pkg>
+        # Need to select a backend
+        # Need somewhere to store project config data
+        # Maybe separate into a task-graph builder and a task-graph runner
+
+        # TODO: allow user to specify run root -- maybe relative to some fixed directory?
+        rundir = os.path.join(pkg.basedir, "rundir")
+
+        builder = TaskGraphBuilder(root_pkg=pkg, rundir=rundir)
+        runner = TaskGraphRunnerLocal(rundir)
+
+        tasks = []
+
+        for spec in args.tasks:
+            task = builder.mkTaskGraph(spec)
+            tasks.append(task)
+
+        asyncio.run(runner.run(tasks))
+
+#        rgy = PkgRgy.inst()
+#        rgy.registerPackage(pkg)
+
+
+        # srcdir = os.getcwd()
+
+        # session = Session(srcdir, rundir)
+
+        # package = session.load(srcdir)
+
+        # graphs = []
+        # for task in args.tasks:
+        #     if task.find(".") == -1:
+        #         task = package.name + "." + task
+        #     subgraph = session.mkTaskGraph(task)
+        #     graphs.append(subgraph)
+
+        # awaitables = [subgraph.do_run() for subgraph in graphs]
+        # print("%d awaitables" % len(awaitables))
+
+        # out = asyncio.get_event_loop().run_until_complete(asyncio.gather(*awaitables))
+
+        # print("out: %s" % str(out))
+

{dv_flow_mgr → dv_flow/mgr}/package_def.py

@@ -19,7 +19,9 @@
 #*     Author: 
 #*
 #****************************************************************************
+import os
 import json
+import yaml
 import importlib
 import sys
 import pydantic
@@ -32,7 +34,7 @@ from .package import Package
 from .package_import_spec import PackageImportSpec, PackageSpec
 from .task import TaskCtor, TaskParams
 from .task_def import TaskDef, TaskSpec
-from .tasklib.builtin_pkg import TaskNull
+from .std.task_null import TaskNull
 
 
 class PackageDef(BaseModel):
@@ -144,7 +146,8 @@ class PackageDef(BaseModel):
                 else:
                     mod = sys.modules[modname]
             except ModuleNotFoundError as e:
-                raise Exception("Failed to import module %s" % modname)
+                raise Exception("Failed to import module %s (basedir=%s): %s" % (
+                    modname, self.basedir, str(e)))
                 
             if not hasattr(mod, clsname):
                 raise Exception("Class %s not found in module %s" % (clsname, modname))
@@ -170,7 +173,15 @@ class PackageDef(BaseModel):
                 "str" : str,
                 "int" : int,
                 "float" : float,
-                "bool" : bool
+                "bool" : bool,
+                "list" : List
+            }
+            pdflt_m = {
+                "str" : "",
+                "int" : 0,
+                "float" : 0.0,
+                "bool" : False,
+                "list" : []
             }
             for p in task.params.keys():
                 param = task.params[p]
@@ -185,7 +196,7 @@ class PackageDef(BaseModel):
                     if "value" in param.keys():
                         field_m[p] = (ptype, param["value"])
                     else:
-                        field_m[p] = (ptype, )
+                        field_m[p] = (ptype, pdflt_m[ptype_s])
                 else:
                     if p not in field_m.keys():
                         raise Exception("Field %s not found" % p)
@@ -208,3 +219,84 @@ class PackageDef(BaseModel):
 
         return ctor_t
 
+    @staticmethod
+    def load(path, exp_pkg_name=None):
+        return PackageDef._loadPkgDef(path, exp_pkg_name, [])
+        pass
+
+    @staticmethod
+    def _loadPkgDef(root, exp_pkg_name, file_s):
+        if root in file_s:
+            raise Exception("Recursive file processing @ %s: %s" % (root, ",".join(file_s)))
+        file_s.append(root)
+        ret = None
+        with open(root, "r") as fp:
+            print("open %s" % root)
+            doc = yaml.load(fp, Loader=yaml.FullLoader)
+            if "package" not in doc.keys():
+                raise Exception("Missing 'package' key in %s" % root)
+            pkg = PackageDef(**(doc["package"]))
+            pkg.basedir = os.path.dirname(root)
+
+#            for t in pkg.tasks:
+#                t.basedir = os.path.dirname(root)
+
+        if exp_pkg_name is not None:
+            if exp_pkg_name != pkg.name:
+                raise Exception("Package name mismatch: %s != %s" % (exp_pkg_name, pkg.name))
+            # else:
+            #     self._pkg_m[exp_pkg_name] = [PackageSpec(pkg.name)
+            #     self._pkg_spec_s.append(PackageSpec(pkg.name))
+
+        # if not len(self._pkg_spec_s):
+        #     self._pkg_spec_s.append(PackageSpec(pkg.name))
+        # else:
+        # self._pkg_def_m[PackageSpec(pkg.name)] = pkg
+
+        print("pkg: %s" % str(pkg))
+
+        print("fragments: %s" % str(pkg.fragments))
+        for spec in pkg.fragments:
+            PackageDef._loadFragmentSpec(pkg, spec, file_s)
+
+        file_s.pop()
+
+        return pkg
+    
+    @staticmethod
+    def _loadFragmentSpec(pkg, spec, file_s):
+        # We're either going to have:
+        # - File path
+        # - Directory path
+
+        if os.path.isfile(os.path.join(pkg.basedir, spec)):
+            PackageDef._loadFragmentFile(pkg, spec, file_s)
+        elif os.path.isdir(os.path.join(pkg.basedir, spec)):
+            PackageDef._loadFragmentDir(pkg, os.path.join(pkg.basedir, spec), file_s)
+        else:
+            raise Exception("Fragment spec %s not found" % spec)
+
+    @staticmethod
+    def _loadFragmentDir(pkg, dir, file_s):
+        for file in os.listdir(dir):
+            if os.path.isdir(os.path.join(dir, file)):
+                PackageDef._loadFragmentDir(pkg, os.path.join(dir, file), file_s)
+            elif os.path.isfile(os.path.join(dir, file)) and file == "flow.dv":
+                PackageDef._loadFragmentFile(pkg, os.path.join(dir, file), file_s)
+
+    @staticmethod
+    def _loadFragmentFile(pkg, file, file_s):
+        if file in file_s:
+            raise Exception("Recursive file processing @ %s: %s" % (file, ", ".join(file_s)))
+        file_s.append(file)
+
+        with open(file, "r") as fp:
+            doc = yaml.load(fp, Loader=yaml.FullLoader)
+            print("doc: %s" % str(doc), flush=True)
+            if "fragment" in doc.keys():
+                # Merge the package definition
+                frag = FragmentDef(**(doc["fragment"]))
+                frag.basedir = os.path.dirname(file)
+                pkg.fragment_l.append(frag)
+            else:
+                print("Warning: file %s is not a fragment" % file)

dv_flow/mgr/pkg_rgy.py

@@ -0,0 +1,78 @@
+import os
+import sys
+from typing import Dict, Tuple
+from .package_def import PackageDef
+
+class PkgRgy(object):
+    _inst = None
+
+    def __init__(self):
+        self._pkgpath = []
+        self._pkg_m : Dict[str, Tuple[str,PackageDef]] = {}
+
+    def hasPackage(self, name, search_path=False):
+        if name in self._pkg_m.keys():
+            return True
+        elif search_path:
+            for p in self._pkgpath:
+                if os.path.exists(os.path.join(p, name)):
+                    return True
+        else:
+            return False
+    
+    def getPackage(self, name):
+        if name in self._pkg_m.keys():
+            if self._pkg_m[name][1] is None:
+                pkg_def = PackageDef.load(self._pkg_m[name][0])
+                # Load the package
+                self._pkg_m[name] = (
+                    self._pkg_m[name][0],
+                    pkg_def
+                )
+                pass
+            return self._pkg_m[name][1]
+        else:
+            # Go search the package path
+            return None
+
+    def registerPackage(self, pkg_def):
+        if pkg_def.name in self._pkg_m.keys():
+            raise Exception("Duplicate package %s" % pkg_def.name)
+        self._pkg_m[pkg_def.name] = pkg_def
+
+    def _discover_plugins(self):
+        # Register built-in package
+        self._pkg_m["std"] = (os.path.join(os.path.dirname(__file__), "std/flow.dv"), None)
+
+        if sys.version_info < (3,10):
+            from importlib_metadata import entry_points
+        else:
+            from importlib.metadata import entry_points
+
+        discovered_plugins = entry_points(group='dv_flow.mgr')
+        for p in discovered_plugins:
+            try:
+                mod = p.load()
+
+                if hasattr(mod, "dvfm_packages"):
+                    pkg_m = mod.dvfm_packages()
+                    
+                    for name,path in pkg_m.items():
+                        if name in self._pkg_m.keys():
+                            raise Exception("Package %s already registered using path %s. Conflicting path: %s" % (
+                                name, self._pkg_m[name][0], path))
+                        self._pkg_m[name] = (path, None)
+            except Exception as e:
+                print("Error loading plugin %s: %s" % (p.name, str(e)))
+                raise e
+
+        # self._pkgs = {}
+        # for pkg in self._load_pkg_list():
+        #     self._pkgs[pkg.name] = pkg
+
+    @classmethod
+    def inst(cls):
+        if cls._inst is None:
+            cls._inst = cls()
+            cls._inst._discover_plugins()
+        return cls._inst

{dv_flow_mgr/tasklib → dv_flow/mgr}/std/fileset.py

@@ -4,8 +4,8 @@ import fnmatch
 import glob
 import pydantic.dataclasses as dc
 from typing import List, Tuple
-from dv_flow_mgr import Task, TaskData, TaskMemento
-from dv_flow_mgr import FileSet as _FileSet
+from dv_flow.mgr import Task, TaskData, TaskMemento
+from dv_flow.mgr import FileSet as _FileSet
 
 class TaskFileSetMemento(TaskMemento):
     files : List[Tuple[str,float]] = dc.Field(default_factory=list)
@@ -45,7 +45,7 @@ class FileSet(Task):
             for file in included_files:
                 if not any(glob.fnmatch.fnmatch(file, os.path.join(glob_root, pattern)) for pattern in self.params.exclude):
                     memento.files.append((file, os.path.getmtime(os.path.join(glob_root, file))))
-                    fs.files.append(file[len(glob_root):])
+                    fs.files.append(file[len(glob_root)+1:])
 
         # Check to see if the filelist or fileset have changed
         # Only bother doing this if the upstream task data has not changed

{dv_flow_mgr/tasklib → dv_flow/mgr}/std/flow.dv

@@ -4,13 +4,13 @@ package:
 
   tasks:
   - name: Message
-    pyclass: message.Message
+    pyclass: dv_flow.mgr.std.message.Message
     with:
       msg:
         type: str
         value: ""
   - name: FileSet
-    pyclass: fileset.FileSet
+    pyclass: dv_flow.mgr.std.fileset.FileSet
     with:
       base:
         type: str
@@ -24,3 +24,7 @@ package:
       exclude:
         type: str
         value: ""
+#  - name: Exec
+#    pyclass: dv_flow.mgr.std.exec.Exec
+#    with: {}
+      

{dv_flow_mgr/tasklib → dv_flow/mgr}/std/message.py

@@ -1,5 +1,5 @@
 
-from dv_flow_mgr import Task, TaskData
+from dv_flow.mgr import Task, TaskData
 
 class Message(Task):
     async def run(self, input : TaskData) -> TaskData:

{dv_flow_mgr/tasklib → dv_flow/mgr}/std/task_fileset.py

@@ -2,11 +2,11 @@ import os
 import glob
 import fnmatch
 import pydantic.dataclasses as dc
-from ...fileset import FileSet
-from ...package import TaskCtor
-from ...task import Task, TaskParams
-from ...task_data import TaskData
-from ...task_memento import TaskMemento
+from ..fileset import FileSet
+from ..package import TaskCtor
+from ..task import Task, TaskParams
+from ..task_data import TaskData
+from ..task_memento import TaskMemento
 from typing import List, Tuple
 
 class TaskFileSet(Task):

dv_flow/mgr/std/task_null.py

@@ -0,0 +1,10 @@
+from ..task import Task
+from ..task_data import TaskData
+
+class TaskNull(Task):
+    """The Null task simply propagates its input to its output"""
+
+    async def run(self, input : TaskData) -> TaskData:
+        # No memento ; data pass-through
+        return input
+

{dv_flow_mgr → dv_flow/mgr}/task.py

@@ -63,10 +63,10 @@ class TaskCtor(object):
 class Task(object):
     """Executable view of a task"""
     name : str
-    session : 'Session'
     params : TaskParams
-    basedir : str
     srcdir : str = None
+    session : 'TaskGraphRunner' = None
+    basedir : str = None
     memento : TaskMemento = None
     depend_refs : List['TaskSpec'] = dc.field(default_factory=list)
     depends : List[int] = dc.field(default_factory=list)
@@ -82,6 +82,10 @@ class Task(object):
     body: Dict[str,Any] = dc.field(default_factory=dict)
     impl_t : Any = None
 
+    def init(self, runner, basedir):
+        self.session = runner
+        self.basedir = basedir
+
     def getMemento(self, T) -> TaskMemento:
         if os.path.isfile(os.path.join(self.rundir, "memento.json")):
             with open(os.path.join(self.rundir, "memento.json"), "r") as fp:
@@ -110,7 +114,7 @@ class Task(object):
             print("deps_o: %s" % str(deps_o))
 
 
-            print("deps_m: %s" % str(deps_m))
+#            print("deps_m: %s" % str(deps_m))
 
             # Merge the output of the dependencies into a single input data
 #            if len(self.depends) > 1:
@@ -119,7 +123,10 @@ class Task(object):
             # Now that we have a clean input object, we need
             # to build the dep map
 
-            input = self.depends[0].output.copy()
+#            input = self.depends[0].output.copy()
+            input = TaskData.merge(deps_o)
+            input.src = self.name
+            input.deps[self.name] = list(inp.name for inp in self.depends)
         else:
             input = TaskData()
         

{dv_flow_mgr → dv_flow/mgr}/task_data.py

@@ -69,20 +69,34 @@ class TaskData(BaseModel):
     def getFileSets(self, type=None, order=True) -> List[FileSet]:
         ret = []
 
+        print("getFileSets: filesets=%s" % str(self.filesets))
+
         if order:
             # The deps map specifies task dependencies
 
             candidate_fs = []
             for fs in self.filesets:
+                print("fs: %s" % str(fs))
                 if type is None or fs.type in type:
                     candidate_fs.append(fs)
-
+            print("self.deps: %s" % str(self.deps))
             order = toposort(self.deps)
 
+            print("order: %s" % str(order))
+
             for order_s in order:
-                for fs in candidate_fs:
+                print("order_s: %s" % str(order_s))
+                i = 0
+                while i < len(candidate_fs):
+                    fs = candidate_fs[i]
+                    print("fs.src: %s" % fs.src)
                     if fs.src in order_s:
+                            print("Add fileset")
                             ret.append(fs)
+                            candidate_fs.pop(i)
+                    else:
+                        i += 1
+            ret.extend(candidate_fs)
         else:
             for fs in self.filesets:
                 if type is None or fs.type in type:
@@ -93,7 +107,9 @@ class TaskData(BaseModel):
     def copy(self) -> 'TaskData':
         ret = TaskData()
         ret.src = self.src
+        ret.basedir = self.basedir
         ret.params = self.params.copy()
+        ret.filesets = self.filesets.copy()
         for d in self.deps:
             ret.deps.append(d.clone())
         ret.changed = self.changed

{dv_flow_mgr → dv_flow/mgr}/task_def.py

@@ -34,7 +34,8 @@ class TaskDef(BaseModel):
 #    type : Union[str,TaskSpec] = dc.Field(default_factory=list)
     uses : str = dc.Field(default=None)
     pyclass : str = dc.Field(default=None)
-    doc : str = dc.Field(default=None)
+    desc : str = dc.Field(default="")
+    doc : str = dc.Field(default="")
     depends : List[Union[str,TaskSpec]] = dc.Field(default_factory=list, alias="needs")
     params: Dict[str,Any] = dc.Field(default_factory=dict, alias="with")