dv-flow-mgr 0.0.1.13979842530a1__py3-none-any.whl → 0.0.1.14097297609a1__py3-none-any.whl

dv_flow/mgr/__init__.py

@@ -21,10 +21,10 @@
 #****************************************************************************
 from .package_def import *
 from .pkg_rgy import PkgRgy
-from .task import *
 from .task_data import *
 from .task_graph_builder import TaskGraphBuilder
-from .task_node import task
+from .task_runner import TaskRunner
+from .task_node_ctor_wrapper import task
 from .task_runner import TaskSetRunner
 from .task_listener_log import TaskListenerLog
 

dv_flow/mgr/__main__.py

@@ -21,10 +21,14 @@
 #****************************************************************************
 import argparse
 import logging
+from .cmds.cmd_graph import CmdGraph
 from .cmds.cmd_run import CmdRun
+from .cmds.cmd_show import CmdShow
 
 def get_parser():
-    parser = argparse.ArgumentParser(description='dv_flow_mgr')
+    parser = argparse.ArgumentParser(
+        description='dv_flow_mgr',
+        prog='dfm')
     # parser.add_argument("-d", "--debug", 
     #                     help="Enable debug",
     #                     action="store_true")
@@ -36,6 +40,16 @@ def get_parser():
     #                     action="store_true")
     subparsers = parser.add_subparsers(required=True)
 
+    graph_parser = subparsers.add_parser('graph', 
+                                         help='Generates the graph of a task')
+    graph_parser.add_argument("task", nargs="?", help="task to graph")
+    graph_parser.add_argument("-f", "--format", help="Specifies the output format",
+                              default="dot")
+    graph_parser.add_argument("-o", "--output", 
+                              help="Specifies the output file",
+                              default="-")
+    graph_parser.set_defaults(func=CmdGraph())
+
     run_parser = subparsers.add_parser('run', help='run a flow')
     run_parser.add_argument("tasks", nargs='*', help="tasks to run")
     run_parser.add_argument("-j",
@@ -43,6 +57,17 @@ def get_parser():
                         type=int, default=-1)
     run_parser.set_defaults(func=CmdRun())
 
+    show_parser = subparsers.add_parser('show', 
+                                        help='Display information about a task or tasks')
+    show_parser.add_argument("task", nargs='?', help="task to show")
+    show_parser.add_argument("-a", "--all",
+                        action="store_true",
+                        help="Shows all tasks required for the subject to run")
+    show_parser.add_argument("-v", "--verbose",
+                        action="store_true",
+                        help="Shows additional information about tasks")
+    show_parser.set_defaults(func=CmdShow())
+
     return parser
 
 def main():

dv_flow/mgr/cmds/cmd_graph.py

@@ -0,0 +1,82 @@
+#****************************************************************************
+#* cmd_graph.py
+#*
+#* Copyright 2023-2025 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*  
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*  
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import asyncio
+import os
+import logging
+from typing import ClassVar
+from ..util import loadProjPkgDef
+from ..task_graph_builder import TaskGraphBuilder
+from ..task_runner import TaskSetRunner
+from ..task_listener_log import TaskListenerLog
+from ..pkg_rgy import PkgRgy
+from ..task_graph_dot_writer import TaskGraphDotWriter
+
+
+class CmdGraph(object):
+    _log : ClassVar = logging.getLogger("CmdGraph")
+
+    def __call__(self, args):
+
+        # First, find the project we're working with
+        pkg = loadProjPkgDef(os.getcwd())
+
+        if pkg is None:
+            raise Exception("Failed to find a 'flow.dv' file that defines a package in %s or its parent directories" % os.getcwd())
+
+        self._log.debug("Root flow file defines package: %s" % pkg.name)
+
+        if args.task is None:
+            # Print out available tasks
+            tasks = []
+            for task in pkg.tasks:
+                tasks.append(task)
+            for frag in pkg._fragment_l:
+                for task in frag.tasks:
+                    tasks.append(task)
+            tasks.sort(key=lambda x: x.name)
+
+            max_name_len = 0
+            for t in tasks:
+                if len(t.name) > max_name_len:
+                    max_name_len = len(t.name)
+
+            print("No task specified. Available Tasks:")
+            for t in tasks:
+                desc = t.desc
+                if desc is None or t.desc == "":
+                    "<no descripion>"
+                print("%s - %s" % (t.name.ljust(max_name_len), desc))
+        else:
+            rundir = os.path.join(pkg._basedir, "rundir")
+
+            builder = TaskGraphBuilder(root_pkg=pkg, rundir=rundir)
+
+            t = builder.mkTaskGraph(pkg.name + "." + args.task)
+
+            TaskGraphDotWriter().write(
+                t,
+                "-"
+            )
+
+        return 0
+
+

dv_flow/mgr/cmds/cmd_run.py

@@ -50,7 +50,7 @@ class CmdRun(object):
             tasks = []
             for task in pkg.tasks:
                 tasks.append(task)
-            for frag in pkg.fragment_l:
+            for frag in pkg._fragment_l:
                 for task in frag.tasks:
                     tasks.append(task)
             tasks.sort(key=lambda x: x.name)
@@ -75,7 +75,7 @@ class CmdRun(object):
         # Maybe separate into a task-graph builder and a task-graph runner
 
         # TODO: allow user to specify run root -- maybe relative to some fixed directory?
-        rundir = os.path.join(pkg.basedir, "rundir")
+        rundir = os.path.join(pkg._basedir, "rundir")
 
         builder = TaskGraphBuilder(root_pkg=pkg, rundir=rundir)
         runner = TaskSetRunner(rundir)

dv_flow/mgr/cmds/cmd_show.py

@@ -0,0 +1,107 @@
+#****************************************************************************
+#* cmd_show.py
+#*
+#* Copyright 2023-2025 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*  
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*  
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import asyncio
+import os
+import logging
+import toposort
+from typing import ClassVar
+from ..util import loadProjPkgDef
+from ..task_graph_builder import TaskGraphBuilder
+from ..task_runner import TaskSetRunner
+from ..task_listener_log import TaskListenerLog
+from ..pkg_rgy import PkgRgy
+from ..task_graph_dot_writer import TaskGraphDotWriter
+
+
+class CmdShow(object):
+    _log : ClassVar = logging.getLogger("CmdGraph")
+
+    def __call__(self, args):
+
+        # First, find the project we're working with
+        pkg = loadProjPkgDef(os.getcwd())
+
+        if pkg is None:
+            raise Exception("Failed to find a 'flow.dv' file that defines a package in %s or its parent directories" % os.getcwd())
+
+        self._log.debug("Root flow file defines package: %s" % pkg.name)
+
+        if args.task is None:
+            # Print out available tasks
+            tasks = []
+            for task in pkg.tasks:
+                tasks.append(task)
+            for frag in pkg._fragment_l:
+                for task in frag.tasks:
+                    tasks.append(task)
+            tasks.sort(key=lambda x: x.name)
+
+            max_name_len = 0
+            for t in tasks:
+                if len(t.name) > max_name_len:
+                    max_name_len = len(t.name)
+
+            print("No task specified. Available Tasks:")
+            for t in tasks:
+                desc = t.desc
+                if desc is None or t.desc == "":
+                    "<no descripion>"
+                print("%s - %s" % (t.name.ljust(max_name_len), desc))
+        else:
+            rundir = os.path.join(pkg._basedir, "rundir")
+
+            builder = TaskGraphBuilder(root_pkg=pkg, rundir=rundir)
+
+
+            if args.all:
+                t = builder.mkTaskGraph(pkg.name + "." + args.task)
+                dep_m = TaskSetRunner(rundir=None).buildDepMap(t)
+
+                order = list(toposort.toposort(dep_m))
+
+                print("Task: %s" % t.name)
+
+                last_s = set()
+                for i,s in enumerate(order):
+                    print("-- TaskSet %d --" % (i+1))
+                    for t in s:
+                        print("  - %s" % t.name)
+                        if args.verbose and len(t.params.model_fields.items()):
+                            print("    params:")
+                            for name,field in t.params.model_fields.items():
+                                print("    - %s: %s" % (name, getattr(t.params, name)))
+                        if len(t.needs):
+                            print("    needs:")
+                            for n in t.needs:
+                                print("    - %s%s" % (
+                                    n[0].name,
+                                    ("*" if n[0] in last_s else "")))
+                    last_s = s
+                        
+                pass
+            else:
+                # Show show info about the current task
+                pass
+
+        return 0
+
+

dv_flow/mgr/fileset.py

@@ -29,6 +29,7 @@ class FileSet(BaseModel):
     basedir : str
     name : str = ""
     src : str = None
+    seq : int = -1
     files : List[str] = dc.Field(default_factory=list)
     incdirs: List[str] = dc.Field(default_factory=list)
     params : Dict[str,str] = dc.Field(default_factory=dict)

dv_flow/mgr/fragment_def.py

@@ -34,7 +34,7 @@ class FragmentDef(BaseModel):
     fragments: List[str] = dc.Field(default_factory=list)
     types : List[TypeDef] = dc.Field(default_factory=list)
 
-    basedir : str = None
+    _basedir : str = None
 
     def getTask(self, name : str) -> 'TaskDef':
         for t in self.tasks:

dv_flow/mgr/package.py

@@ -22,7 +22,7 @@
 import dataclasses as dc
 import logging
 from typing import Any, ClassVar, Dict
-from .task_ctor import TaskCtor
+from .task_node_ctor import TaskNodeCtor
 
 @dc.dataclass
 class Package(object):
@@ -30,11 +30,11 @@ class Package(object):
     params : Dict[str,Any] = dc.field(default_factory=dict)
     # Package holds constructors for tasks
     # - Dict holds the default parameters for the task
-    tasks : Dict[str,TaskCtor] = dc.field(default_factory=dict)
+    tasks : Dict[str,TaskNodeCtor] = dc.field(default_factory=dict)
     types : Dict[str,Any] = dc.field(default_factory=dict)
     _log : ClassVar = logging.getLogger("Package")
 
-    def getTaskCtor(self, name : str) -> TaskCtor:
+    def getTaskCtor(self, name : str) -> TaskNodeCtor:
         self._log.debug("-- %s::getTaskCtor: %s" % (self.name, name))
         if name not in self.tasks.keys():
             raise Exception("Task %s not present in package %s" % (name, self.name))

dv_flow/mgr/package_def.py

@@ -21,7 +21,6 @@
 #****************************************************************************
 import io
 import os
-import json
 import yaml
 import importlib
 import logging
@@ -34,10 +33,13 @@ from .fragment_def import FragmentDef
 from .package import Package
 from .package_import_spec import PackageImportSpec, PackageSpec
 from .param_def import ParamDef
-from .task_node import TaskNodeCtor, TaskNodeCtorProxy, TaskNodeCtorTask
-from .task_ctor import TaskCtor
-from .task_def import TaskDef, TaskSpec
-from .std.task_null import TaskNull, TaskNullParams
+from .task_def import TaskDef
+from .task_node_ctor import TaskNodeCtor
+from .task_node_ctor_proxy import TaskNodeCtorProxy
+from .task_node_ctor_task import TaskNodeCtorTask
+from .task_node_ctor_compound import TaskNodeCtorCompound
+from .task_node_ctor_compound_proxy import TaskNodeCtorCompoundProxy
+from .std.task_null import TaskNull
 from .type_def import TypeDef
 
 
@@ -63,15 +65,33 @@ class PackageDef(BaseModel):
         default_factory=list, alias="with",
         description="List of package parameters to set")
 
-    fragment_l : List['FragmentDef'] = dc.Field(default_factory=list, exclude=True)
-    subpkg_m : Dict[str,'PackageDef'] = dc.Field(default_factory=dict, exclude=True)
+    _fragment_l : List['FragmentDef'] = []
+    _subpkg_m : Dict[str,'PackageDef'] = {}
 
 #    import_m : Dict['PackageSpec','Package'] = dc.Field(default_factory=dict)
 
-    basedir : str = None
+    _basedir : str = None
     _log : ClassVar = logging.getLogger("PackageDef")
 
+    @property
+    def fragment_l(self):
+        return self._fragment_l
+    
+    @property
+    def subpkg_m(self):
+        return self._subpkg_m
+
+    @property
+    def basedir(self):
+        return self._basedir
+    
+    @basedir.setter
+    def basedir(self, v):
+        self._basedir = v
+
     def __post_init__(self):
+        self._fragment_l = []
+        self._subpkg_m = {}
         for t in self.tasks:
             t.fullname = self.name + "." + t.name
 
@@ -86,18 +106,18 @@ class PackageDef(BaseModel):
 
         session.push_package(ret, add=True)
 
-        tasks_m : Dict[str,str,TaskCtor]= {}
+        tasks_m : Dict[str,str,TaskNodeCtor]= {}
 
         for task in self.tasks:
             if task.name in tasks_m.keys():
                 raise Exception("Duplicate task %s" % task.name)
-            tasks_m[task.name] = (task, self.basedir, ) # We'll add a TaskCtor later
+            tasks_m[task.name] = (task, self._basedir, ) # We'll add a TaskNodeCtor later
 
-        for frag in self.fragment_l:
+        for frag in self._fragment_l:
             for task in frag.tasks:
                 if task.name in tasks_m.keys():
                     raise Exception("Duplicate task %s" % task.name)
-                tasks_m[task.name] = (task, frag.basedir, ) # We'll add a TaskCtor later
+                tasks_m[task.name] = (task, frag._basedir, ) # We'll add a TaskNodeCtor later
 
         # Now we have a unified map of the tasks declared in this package
         for name in list(tasks_m.keys()):
@@ -143,10 +163,23 @@ class PackageDef(BaseModel):
             ctor_t = tasks_m[task_name][2]
         return ctor_t
 
-    def mkTaskCtor(self, session, task, srcdir, tasks_m) -> TaskCtor:
+    def mkTaskCtor(self, session, task, srcdir, tasks_m) -> TaskNodeCtor:
         self._log.debug("--> %s::mkTaskCtor %s (srcdir: %s)" % (self.name, task.name, srcdir))
-        base_ctor_t : TaskCtor = None
-        ctor_t : TaskCtor = None
+
+        if len(task.tasks) > 0:
+            # Compound task
+            ctor = self._mkCompoundTaskCtor(session, task, srcdir, tasks_m)
+        else:
+            # Leaf task
+            ctor = self._mkLeafTaskCtor(session, task, srcdir, tasks_m)
+        
+        return ctor
+
+    
+    def _mkLeafTaskCtor(self, session, task, srcdir, tasks_m) -> TaskNodeCtor:
+        self._log.debug("--> _mkLeafTaskCtor")
+        base_ctor_t : TaskNodeCtor = None
+        ctor_t : TaskNodeCtor = None
         base_params : BaseModel = None
         callable = None
         passthrough = task.passthrough
@@ -179,14 +212,14 @@ class PackageDef(BaseModel):
 
             try:
                 if modname not in sys.modules:
-                    if self.basedir not in sys.path:
-                        sys.path.append(self.basedir)
+                    if self._basedir not in sys.path:
+                        sys.path.append(self._basedir)
                     mod = importlib.import_module(modname)
                 else:
                     mod = sys.modules[modname]
             except ModuleNotFoundError as e:
-                raise Exception("Failed to import module %s (basedir=%s): %s" % (
-                    modname, self.basedir, str(e)))
+                raise Exception("Failed to import module %s (_basedir=%s): %s" % (
+                    modname, self._basedir, str(e)))
                 
             if not hasattr(mod, clsname):
                 raise Exception("Method %s not found in module %s" % (clsname, modname))
@@ -227,7 +260,62 @@ class PackageDef(BaseModel):
 
         self._log.debug("<-- %s::mkTaskCtor %s" % (self.name, task.name))
         return ctor_t
-    
+
+    def _mkCompoundTaskCtor(self, session, task, srcdir, tasks_m) -> TaskNodeCtor:
+        self._log.debug("--> _mkCompoundTaskCtor")
+        base_ctor_t : TaskNodeCtor = None
+        ctor_t : TaskNodeCtor = None
+        base_params : BaseModel = None
+        callable = None
+        passthrough = task.passthrough
+        consumes = [] if task.consumes is None else task.consumes.copy()
+        needs = [] if task.needs is None else task.needs.copy()
+        fullname = self.name + "." + task.name
+
+
+        if task.uses is not None:
+            self._log.debug("Uses: %s" % task.uses)
+            base_ctor_t = self.getTaskCtor(session, task.uses, tasks_m)
+            base_params = base_ctor_t.mkTaskParams()
+
+            # Once we have passthrough, we can't turn it off
+            passthrough |= base_ctor_t.passthrough
+            consumes.extend(base_ctor_t.consumes)
+
+            if base_ctor_t is None:
+                self._log.error("Failed to load task ctor %s" % task.uses)
+
+        # Determine if we need to use a new 
+        paramT = self._getParamT(session, task, base_params)
+
+        if base_ctor_t is not None:
+            ctor_t = TaskNodeCtorCompoundProxy(
+                name=fullname,
+                srcdir=srcdir,
+                paramT=paramT,
+                passthrough=passthrough,
+                consumes=consumes,
+                needs=needs,
+                task_def=task,
+                uses=base_ctor_t)
+        else:
+            self._log.debug("No 'uses' specified")
+            ctor_t = TaskNodeCtorCompound(
+                name=fullname,
+                srcdir=srcdir,
+                paramT=paramT,
+                passthrough=passthrough,
+                consumes=consumes,
+                needs=needs,
+                task_def=task)
+
+        for t in task.tasks:
+            ctor_t.tasks.append(self.mkTaskCtor(session, t, srcdir, tasks_m))
+
+        
+        self._log.debug("<-- %s::mkTaskCtor %s (%d)" % (self.name, task.name, len(ctor_t.tasks)))
+        return ctor_t
+
     def _getParamT(self, session, task, base_t : BaseModel):
         self._log.debug("--> _getParamT %s" % task.fullname)
         # Get the base parameter type (if available)
@@ -326,10 +414,10 @@ class PackageDef(BaseModel):
             except Exception as e:
                 PackageDef._log.error("Failed to load package from %s" % root)
                 raise e
-            pkg.basedir = os.path.dirname(root)
+            pkg._basedir = os.path.dirname(root)
 
 #            for t in pkg.tasks:
-#                t.basedir = os.path.dirname(root)
+#                t._basedir = os.path.dirname(root)
 
         if exp_pkg_name is not None:
             if exp_pkg_name != pkg.name:
@@ -347,7 +435,7 @@ class PackageDef(BaseModel):
             PackageDef._loadFragmentSpec(pkg, spec, file_s)
 
         if len(pkg.imports) > 0:
-            cls._log.info("Loading imported packages (basedir=%s)" % pkg.basedir)
+            cls._log.info("Loading imported packages (_basedir=%s)" % pkg._basedir)
         for imp in pkg.imports:
             if type(imp) == str:
                 imp_path = imp
@@ -359,8 +447,8 @@ class PackageDef(BaseModel):
             cls._log.info("Loading imported package %s" % imp_path)
 
             if not os.path.isabs(imp_path):
-                cls._log.debug("basedir: %s ; imp_path: %s" % (pkg.basedir, imp_path))
-                imp_path = os.path.join(pkg.basedir, imp_path)
+                cls._log.debug("_basedir: %s ; imp_path: %s" % (pkg._basedir, imp_path))
+                imp_path = os.path.join(pkg._basedir, imp_path)
             
             # Search down the tree looking for a flow.dv file
             if os.path.isdir(imp_path):
@@ -389,7 +477,7 @@ class PackageDef(BaseModel):
                 
             sub_pkg = PackageDef.load(imp_path)
             cls._log.info("Loaded imported package %s" % sub_pkg.name)
-            pkg.subpkg_m[sub_pkg.name] = sub_pkg
+            pkg._subpkg_m[sub_pkg.name] = sub_pkg
 
         file_s.pop()
 
@@ -407,10 +495,10 @@ class PackageDef(BaseModel):
         if "package" not in doc.keys():
             raise Exception("Missing 'package' key in %s" % root)
         pkg = PackageDef(**(doc["package"]))
-        pkg.basedir = None
+        pkg._basedir = None
 
 #            for t in pkg.tasks:
-#                t.basedir = os.path.dirname(root)
+#                t._basedir = os.path.dirname(root)
 
         if exp_pkg_name is not None:
             if exp_pkg_name != pkg.name:
@@ -427,10 +515,10 @@ class PackageDef(BaseModel):
         # - File path
         # - Directory path
 
-        if os.path.isfile(os.path.join(pkg.basedir, spec)):
+        if os.path.isfile(os.path.join(pkg._basedir, spec)):
             PackageDef._loadFragmentFile(pkg, spec, file_s)
-        elif os.path.isdir(os.path.join(pkg.basedir, spec)):
-            PackageDef._loadFragmentDir(pkg, os.path.join(pkg.basedir, spec), file_s)
+        elif os.path.isdir(os.path.join(pkg._basedir, spec)):
+            PackageDef._loadFragmentDir(pkg, os.path.join(pkg._basedir, spec), file_s)
         else:
             raise Exception("Fragment spec %s not found" % spec)
 
@@ -454,7 +542,7 @@ class PackageDef(BaseModel):
             if "fragment" in doc.keys():
                 # Merge the package definition
                 frag = FragmentDef(**(doc["fragment"]))
-                frag.basedir = os.path.dirname(file)
-                pkg.fragment_l.append(frag)
+                frag._basedir = os.path.dirname(file)
+                pkg._fragment_l.append(frag)
             else:
                 print("Warning: file %s is not a fragment" % file)

dv_flow/mgr/param_def.py

@@ -23,15 +23,20 @@ from typing import Any, List, Union
 from pydantic import BaseModel, Field
 
 class ListType(BaseModel):
-    item : Union[str, 'ComplexType']
+#    item : Union[str, 'ComplexType']
+    item : Union[str, Any]
 
 class MapType(BaseModel):
-    key : Union[str, 'ComplexType']
-    item : Union[str, 'ComplexType']
+#    key : Union[str, 'ComplexType']
+#    item : Union[str, 'ComplexType']
+    key : Union[str, Any]
+    item : Union[str, Any]
 
 class ComplexType(BaseModel):
     list : Union[ListType, None] = None
     map : Union[MapType, None] = None
+#    list : Union[Any, None] = None
+#    map : Union[Any, None] = None
 
 class ParamDef(BaseModel):
     doc : str = None

dv_flow/mgr/std/message.py

@@ -19,7 +19,7 @@
 #*     Author: 
 #*
 #****************************************************************************
-from dv_flow.mgr import Task, TaskDataResult
+from dv_flow.mgr import TaskDataResult
 
 async def Message(runner, input) -> TaskDataResult:
         print("%s: %s" % (input.name, input.params.msg), flush=True)