dv-flow-mgr 0.0.1.14013608039a1__py3-none-any.whl → 0.0.1.14110086671a1__py3-none-any.whl

dv_flow/mgr/__init__.py

@@ -21,10 +21,10 @@
 #****************************************************************************
 from .package_def import *
 from .pkg_rgy import PkgRgy
-from .task import *
 from .task_data import *
 from .task_graph_builder import TaskGraphBuilder
-from .task_node import task
+from .task_runner import TaskRunner
+from .task_node_ctor_wrapper import task
 from .task_runner import TaskSetRunner
 from .task_listener_log import TaskListenerLog
 

dv_flow/mgr/__main__.py

@@ -21,10 +21,14 @@
 #****************************************************************************
 import argparse
 import logging
+from .cmds.cmd_graph import CmdGraph
 from .cmds.cmd_run import CmdRun
+from .cmds.cmd_show import CmdShow
 
 def get_parser():
-    parser = argparse.ArgumentParser(description='dv_flow_mgr')
+    parser = argparse.ArgumentParser(
+        description='dv_flow_mgr',
+        prog='dfm')
     # parser.add_argument("-d", "--debug", 
     #                     help="Enable debug",
     #                     action="store_true")
@@ -36,6 +40,16 @@ def get_parser():
     #                     action="store_true")
     subparsers = parser.add_subparsers(required=True)
 
+    graph_parser = subparsers.add_parser('graph', 
+                                         help='Generates the graph of a task')
+    graph_parser.add_argument("task", nargs="?", help="task to graph")
+    graph_parser.add_argument("-f", "--format", help="Specifies the output format",
+                              default="dot")
+    graph_parser.add_argument("-o", "--output", 
+                              help="Specifies the output file",
+                              default="-")
+    graph_parser.set_defaults(func=CmdGraph())
+
     run_parser = subparsers.add_parser('run', help='run a flow')
     run_parser.add_argument("tasks", nargs='*', help="tasks to run")
     run_parser.add_argument("-j",
@@ -43,6 +57,17 @@ def get_parser():
                         type=int, default=-1)
     run_parser.set_defaults(func=CmdRun())
 
+    show_parser = subparsers.add_parser('show', 
+                                        help='Display information about a task or tasks')
+    show_parser.add_argument("task", nargs='?', help="task to show")
+    show_parser.add_argument("-a", "--all",
+                        action="store_true",
+                        help="Shows all tasks required for the subject to run")
+    show_parser.add_argument("-v", "--verbose",
+                        action="store_true",
+                        help="Shows additional information about tasks")
+    show_parser.set_defaults(func=CmdShow())
+
     return parser
 
 def main():

dv_flow/mgr/cmds/cmd_graph.py

@@ -0,0 +1,82 @@
+#****************************************************************************
+#* cmd_graph.py
+#*
+#* Copyright 2023-2025 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*  
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*  
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import asyncio
+import os
+import logging
+from typing import ClassVar
+from ..util import loadProjPkgDef
+from ..task_graph_builder import TaskGraphBuilder
+from ..task_runner import TaskSetRunner
+from ..task_listener_log import TaskListenerLog
+from ..pkg_rgy import PkgRgy
+from ..task_graph_dot_writer import TaskGraphDotWriter
+
+
+class CmdGraph(object):
+    _log : ClassVar = logging.getLogger("CmdGraph")
+
+    def __call__(self, args):
+
+        # First, find the project we're working with
+        pkg = loadProjPkgDef(os.getcwd())
+
+        if pkg is None:
+            raise Exception("Failed to find a 'flow.dv' file that defines a package in %s or its parent directories" % os.getcwd())
+
+        self._log.debug("Root flow file defines package: %s" % pkg.name)
+
+        if args.task is None:
+            # Print out available tasks
+            tasks = []
+            for task in pkg.tasks:
+                tasks.append(task)
+            for frag in pkg._fragment_l:
+                for task in frag.tasks:
+                    tasks.append(task)
+            tasks.sort(key=lambda x: x.name)
+
+            max_name_len = 0
+            for t in tasks:
+                if len(t.name) > max_name_len:
+                    max_name_len = len(t.name)
+
+            print("No task specified. Available Tasks:")
+            for t in tasks:
+                desc = t.desc
+                if desc is None or t.desc == "":
+                    "<no descripion>"
+                print("%s - %s" % (t.name.ljust(max_name_len), desc))
+        else:
+            rundir = os.path.join(pkg._basedir, "rundir")
+
+            builder = TaskGraphBuilder(root_pkg=pkg, rundir=rundir)
+
+            t = builder.mkTaskGraph(pkg.name + "." + args.task)
+
+            TaskGraphDotWriter().write(
+                t,
+                "-"
+            )
+
+        return 0
+
+

dv_flow/mgr/cmds/cmd_show.py

@@ -0,0 +1,107 @@
+#****************************************************************************
+#* cmd_show.py
+#*
+#* Copyright 2023-2025 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*  
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*  
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import asyncio
+import os
+import logging
+import toposort
+from typing import ClassVar
+from ..util import loadProjPkgDef
+from ..task_graph_builder import TaskGraphBuilder
+from ..task_runner import TaskSetRunner
+from ..task_listener_log import TaskListenerLog
+from ..pkg_rgy import PkgRgy
+from ..task_graph_dot_writer import TaskGraphDotWriter
+
+
+class CmdShow(object):
+    _log : ClassVar = logging.getLogger("CmdGraph")
+
+    def __call__(self, args):
+
+        # First, find the project we're working with
+        pkg = loadProjPkgDef(os.getcwd())
+
+        if pkg is None:
+            raise Exception("Failed to find a 'flow.dv' file that defines a package in %s or its parent directories" % os.getcwd())
+
+        self._log.debug("Root flow file defines package: %s" % pkg.name)
+
+        if args.task is None:
+            # Print out available tasks
+            tasks = []
+            for task in pkg.tasks:
+                tasks.append(task)
+            for frag in pkg._fragment_l:
+                for task in frag.tasks:
+                    tasks.append(task)
+            tasks.sort(key=lambda x: x.name)
+
+            max_name_len = 0
+            for t in tasks:
+                if len(t.name) > max_name_len:
+                    max_name_len = len(t.name)
+
+            print("No task specified. Available Tasks:")
+            for t in tasks:
+                desc = t.desc
+                if desc is None or t.desc == "":
+                    "<no descripion>"
+                print("%s - %s" % (t.name.ljust(max_name_len), desc))
+        else:
+            rundir = os.path.join(pkg._basedir, "rundir")
+
+            builder = TaskGraphBuilder(root_pkg=pkg, rundir=rundir)
+
+
+            if args.all:
+                t = builder.mkTaskGraph(pkg.name + "." + args.task)
+                dep_m = TaskSetRunner(rundir=None).buildDepMap(t)
+
+                order = list(toposort.toposort(dep_m))
+
+                print("Task: %s" % t.name)
+
+                last_s = set()
+                for i,s in enumerate(order):
+                    print("-- TaskSet %d --" % (i+1))
+                    for t in s:
+                        print("  - %s" % t.name)
+                        if args.verbose and len(t.params.model_fields.items()):
+                            print("    params:")
+                            for name,field in t.params.model_fields.items():
+                                print("    - %s: %s" % (name, getattr(t.params, name)))
+                        if len(t.needs):
+                            print("    needs:")
+                            for n in t.needs:
+                                print("    - %s%s" % (
+                                    n[0].name,
+                                    ("*" if n[0] in last_s else "")))
+                    last_s = s
+                        
+                pass
+            else:
+                # Show show info about the current task
+                pass
+
+        return 0
+
+

dv_flow/mgr/fileset.py

@@ -29,6 +29,7 @@ class FileSet(BaseModel):
     basedir : str
     name : str = ""
     src : str = None
+    seq : int = -1
     files : List[str] = dc.Field(default_factory=list)
     incdirs: List[str] = dc.Field(default_factory=list)
     params : Dict[str,str] = dc.Field(default_factory=dict)

dv_flow/mgr/package.py

@@ -22,7 +22,7 @@
 import dataclasses as dc
 import logging
 from typing import Any, ClassVar, Dict
-from .task_ctor import TaskCtor
+from .task_node_ctor import TaskNodeCtor
 
 @dc.dataclass
 class Package(object):
@@ -30,11 +30,11 @@ class Package(object):
     params : Dict[str,Any] = dc.field(default_factory=dict)
     # Package holds constructors for tasks
     # - Dict holds the default parameters for the task
-    tasks : Dict[str,TaskCtor] = dc.field(default_factory=dict)
+    tasks : Dict[str,TaskNodeCtor] = dc.field(default_factory=dict)
     types : Dict[str,Any] = dc.field(default_factory=dict)
     _log : ClassVar = logging.getLogger("Package")
 
-    def getTaskCtor(self, name : str) -> TaskCtor:
+    def getTaskCtor(self, name : str) -> TaskNodeCtor:
         self._log.debug("-- %s::getTaskCtor: %s" % (self.name, name))
         if name not in self.tasks.keys():
             raise Exception("Task %s not present in package %s" % (name, self.name))

dv_flow/mgr/package_def.py

@@ -21,7 +21,6 @@
 #****************************************************************************
 import io
 import os
-import json
 import yaml
 import importlib
 import logging
@@ -34,10 +33,13 @@ from .fragment_def import FragmentDef
 from .package import Package
 from .package_import_spec import PackageImportSpec, PackageSpec
 from .param_def import ParamDef
-from .task_node import TaskNodeCtor, TaskNodeCtorProxy, TaskNodeCtorTask
-from .task_ctor import TaskCtor
-from .task_def import TaskDef, TaskSpec
-from .std.task_null import TaskNull, TaskNullParams
+from .task_def import TaskDef
+from .task_node_ctor import TaskNodeCtor
+from .task_node_ctor_proxy import TaskNodeCtorProxy
+from .task_node_ctor_task import TaskNodeCtorTask
+from .task_node_ctor_compound import TaskNodeCtorCompound
+from .task_node_ctor_compound_proxy import TaskNodeCtorCompoundProxy
+from .std.task_null import TaskNull
 from .type_def import TypeDef
 
 
@@ -104,18 +106,18 @@ class PackageDef(BaseModel):
 
         session.push_package(ret, add=True)
 
-        tasks_m : Dict[str,str,TaskCtor]= {}
+        tasks_m : Dict[str,str,TaskNodeCtor]= {}
 
         for task in self.tasks:
             if task.name in tasks_m.keys():
                 raise Exception("Duplicate task %s" % task.name)
-            tasks_m[task.name] = (task, self._basedir, ) # We'll add a TaskCtor later
+            tasks_m[task.name] = (task, self._basedir, ) # We'll add a TaskNodeCtor later
 
         for frag in self._fragment_l:
             for task in frag.tasks:
                 if task.name in tasks_m.keys():
                     raise Exception("Duplicate task %s" % task.name)
-                tasks_m[task.name] = (task, frag._basedir, ) # We'll add a TaskCtor later
+                tasks_m[task.name] = (task, frag._basedir, ) # We'll add a TaskNodeCtor later
 
         # Now we have a unified map of the tasks declared in this package
         for name in list(tasks_m.keys()):
@@ -161,10 +163,23 @@ class PackageDef(BaseModel):
             ctor_t = tasks_m[task_name][2]
         return ctor_t
 
-    def mkTaskCtor(self, session, task, srcdir, tasks_m) -> TaskCtor:
+    def mkTaskCtor(self, session, task, srcdir, tasks_m) -> TaskNodeCtor:
         self._log.debug("--> %s::mkTaskCtor %s (srcdir: %s)" % (self.name, task.name, srcdir))
-        base_ctor_t : TaskCtor = None
-        ctor_t : TaskCtor = None
+
+        if len(task.tasks) > 0:
+            # Compound task
+            ctor = self._mkCompoundTaskCtor(session, task, srcdir, tasks_m)
+        else:
+            # Leaf task
+            ctor = self._mkLeafTaskCtor(session, task, srcdir, tasks_m)
+        
+        return ctor
+
+    
+    def _mkLeafTaskCtor(self, session, task, srcdir, tasks_m) -> TaskNodeCtor:
+        self._log.debug("--> _mkLeafTaskCtor")
+        base_ctor_t : TaskNodeCtor = None
+        ctor_t : TaskNodeCtor = None
         base_params : BaseModel = None
         callable = None
         passthrough = task.passthrough
@@ -245,7 +260,62 @@ class PackageDef(BaseModel):
 
         self._log.debug("<-- %s::mkTaskCtor %s" % (self.name, task.name))
         return ctor_t
-    
+
+    def _mkCompoundTaskCtor(self, session, task, srcdir, tasks_m) -> TaskNodeCtor:
+        self._log.debug("--> _mkCompoundTaskCtor")
+        base_ctor_t : TaskNodeCtor = None
+        ctor_t : TaskNodeCtor = None
+        base_params : BaseModel = None
+        callable = None
+        passthrough = task.passthrough
+        consumes = [] if task.consumes is None else task.consumes.copy()
+        needs = [] if task.needs is None else task.needs.copy()
+        fullname = self.name + "." + task.name
+
+
+        if task.uses is not None:
+            self._log.debug("Uses: %s" % task.uses)
+            base_ctor_t = self.getTaskCtor(session, task.uses, tasks_m)
+            base_params = base_ctor_t.mkTaskParams()
+
+            # Once we have passthrough, we can't turn it off
+            passthrough |= base_ctor_t.passthrough
+            consumes.extend(base_ctor_t.consumes)
+
+            if base_ctor_t is None:
+                self._log.error("Failed to load task ctor %s" % task.uses)
+
+        # Determine if we need to use a new 
+        paramT = self._getParamT(session, task, base_params)
+
+        if base_ctor_t is not None:
+            ctor_t = TaskNodeCtorCompoundProxy(
+                name=fullname,
+                srcdir=srcdir,
+                paramT=paramT,
+                passthrough=passthrough,
+                consumes=consumes,
+                needs=needs,
+                task_def=task,
+                uses=base_ctor_t)
+        else:
+            self._log.debug("No 'uses' specified")
+            ctor_t = TaskNodeCtorCompound(
+                name=fullname,
+                srcdir=srcdir,
+                paramT=paramT,
+                passthrough=passthrough,
+                consumes=consumes,
+                needs=needs,
+                task_def=task)
+
+        for t in task.tasks:
+            ctor_t.tasks.append(self.mkTaskCtor(session, t, srcdir, tasks_m))
+
+        
+        self._log.debug("<-- %s::mkTaskCtor %s (%d)" % (self.name, task.name, len(ctor_t.tasks)))
+        return ctor_t
+
     def _getParamT(self, session, task, base_t : BaseModel):
         self._log.debug("--> _getParamT %s" % task.fullname)
         # Get the base parameter type (if available)

dv_flow/mgr/std/message.py

@@ -19,7 +19,7 @@
 #*     Author: 
 #*
 #****************************************************************************
-from dv_flow.mgr import Task, TaskDataResult
+from dv_flow.mgr import TaskDataResult
 
 async def Message(runner, input) -> TaskDataResult:
         print("%s: %s" % (input.name, input.params.msg), flush=True)

dv_flow/mgr/task_data.py

@@ -48,20 +48,16 @@ class TaskMarker(BaseModel):
     severity : SeverityE
     loc : TaskMarkerLoc = dc.Field(default=None)
 
-class TaskParameterSet(BaseModel):
-    type : str = None
-    task : str = None # Name of the task that produced this param set
-    seq : int  = -1 # Order in which the param-set must appear
-
 class TaskDataInput(BaseModel):
     """
-    Input data to a task
-    - name - name of the task
-    - changed - indicates whether any of this task's dependencies have changed
-    - rundir - directory in which the task is to be run
-    - params - parameters to the task
-    - inputs - list of parameter sets 'consumed' by this task
-    - memento - memento data previously returned by this task. None if no memento is available
+    Input data to a task:
+
+    * name - name of the task
+    * changed - indicates whether any of this task's dependencies have changed
+    * rundir - directory in which the task is to be run
+    * params - parameters to the task
+    * inputs - list of `TaskDataItem` that are consumed' by this task
+    * memento - memento data previously returned by this task. None if no memento is available
     """
     name : str
     changed : bool
@@ -73,12 +69,13 @@ class TaskDataInput(BaseModel):
 
 class TaskDataResult(BaseModel):
     """
-    Result data from a task
-    - changed - indicates whether the task modified its result data
-    - output - list of output parameter sets
-    - memento - memento data to be passed to the next invocation of the task
-    - markers - list of markers produced by the task
-    - status - status code (0=success, non-zero=failure)
+    Result data from a task:
+
+    * changed - indicates whether the task modified its result data
+    * output - list of output parameter sets
+    * memento - memento data to be passed to the next invocation of the task
+    * markers - list of markers produced by the task
+    * status - status code (0=success, non-zero=failure)
     """
     changed : bool = dc.Field(default=True)
     output : List[Any] = dc.Field(default_factory=list)
@@ -113,9 +110,16 @@ class TaskDataParam(BaseModel):
     ops : List[TaskDataParamOp] = dc.Field(default_factory=list)
 
 class TaskDataItem(BaseModel):
+    """
+    Base class for task data items
+
+    * type - Name of the data item type
+    * src - Name of the task that produced this item
+    * seq - Sequence number of the item within the task
+    """
     type : str
-    src : str
-    id : str
+    src : str = None
+    seq : int = -1
 
 class TaskData(BaseModel):
     src : str = None

dv_flow/mgr/task_def.py

@@ -24,7 +24,6 @@ import enum
 from pydantic import BaseModel
 from typing import Any, Dict, List, Union, Tuple
 from .param_def import ParamDef
-from .task import Task
 from .task_output import TaskOutput
 
 @dc.dataclass
@@ -40,6 +39,12 @@ class RundirE(enum.Enum):
     Unique = "unique"
     Inherit = "inherit"
 
+class PassthroughE(enum.Enum):
+    No = "no"
+    All = "all"
+    Unused = "unused"
+
+
 class StrategyDef(BaseModel):
     matrix : Dict[str,List[Any]] = dc.Field(
         default_factory=dict,
@@ -84,8 +89,8 @@ class TaskDef(BaseModel):
     rundir : RundirE = dc.Field(
         default=RundirE.Unique,
         description="Specifies handling of this tasks's run directory")
-    passthrough: bool = dc.Field(
-        default=False,
+    passthrough: PassthroughE = dc.Field(
+        default=PassthroughE.No,
         description="Specifies whether this task should pass its inputs to its output")
     consumes : List[Any] = dc.Field(
         default_factory=list,