dv-flow-mgr 0.0.2.14182043984a1__py3-none-any.whl → 1.0.0.14370600369a1__py3-none-any.whl

dv_flow/mgr/std/flow.dv

@@ -28,7 +28,8 @@ package:
 
   tasks:
   - name: Message
-    pytask: dv_flow.mgr.std.message.Message
+    shell: pytask
+    run: dv_flow.mgr.std.message.Message
     with:
       msg:
         type: str
@@ -36,7 +37,8 @@ package:
   - name: FileSet
     doc: |
       Creates a fileset from a list of files or glob patterns
-    pytask: dv_flow.mgr.std.fileset.FileSet
+    shell: pytask
+    run: dv_flow.mgr.std.fileset.FileSet
     passthrough: all
     consumes: none
     with:
@@ -61,7 +63,8 @@ package:
         type: str
         value: ""
   - name: CreateFile
-    pytask: dv_flow.mgr.std.create_file.CreateFile
+    shell: pytask
+    run: dv_flow.mgr.std.create_file.CreateFile
     passthrough: all
     consumes: none
     doc: |
@@ -78,7 +81,8 @@ package:
       content:
         type: str
   - name: Exec
-    pytask: dv_flow.mgr.std.exec.Exec
+    shell: pytask
+    run: dv_flow.mgr.std.exec.Exec
     desc: Executes a subprocess
     with:
       shell:
@@ -105,6 +109,23 @@ package:
           Optional timestamp file to determine if running 
           the command changed the output
   types:
+  - name: DataItem
+    with:
+    - name: type
+      type: str
+  - name: FileSet
+    uses: std.DataItem
+    with:
+    - name: filetype
+      type: str
+      value: "FileSet"
+    - name: basedir
+      type: str
+    - name: files
+      type: list
+      item:
+        type: str
+
   # - name: TaskDataItem
   #   doc: |
   #     Base type for a data element produced by a task.

dv_flow/mgr/task.py

@@ -0,0 +1,68 @@
+import dataclasses as dc
+from typing import Any, Dict, List, Tuple, Union
+from .srcinfo import SrcInfo
+from .task_def import TaskDef, RundirE, PassthroughE, ConsumesE
+from .task_node_ctor import TaskNodeCtor
+
+@dc.dataclass
+class Need(object):
+    task : 'Task'
+    cond : str = None
+
+@dc.dataclass
+class Task(object):
+    """
+    Type information about a task, linking it into the package
+    to which it belongs.
+
+    Needs in the Task class point to the resolved name. Overrides
+    are applied when constructing a TaskNode DAG from tasks
+    """
+    name : str
+    desc: str = ""
+    doc : str = ""
+    paramT : Any = None
+    uses : 'Task' = None
+    needs : List[str] = dc.field(default_factory=list)
+    consumes : Union[ConsumesE, List[Dict[str, Any]]] = dc.field(default=None)
+    passthrough : Union[PassthroughE, List[Dict[str, Any]]] = dc.field(default=None)
+    rundir : RundirE = None
+    # TODO: strategy / matrix
+    subtasks : List['Task'] = dc.field(default_factory=list)
+    run : str = None
+    shell : str = "bash"
+    srcinfo : SrcInfo = None
+
+    @property
+    def leafname(self):
+        return self.name[self.name.rfind(".")+1:]
+
+    def __post_init__(self):
+        if self.name is None:
+            self.name = self.task_def.name
+
+    def dump(self):
+        task = {
+            "name": self.name,
+            "paramT": str(type(self.paramT)),
+            "rundir": str(self.rundir),
+        }
+
+        if self.uses is not None:
+            task["uses"] = self.uses.name
+        if self.needs is not None and len(self.needs):
+            task["needs"] = [n.name for n in self.needs]
+        if self.subtasks is not None and len(self.subtasks):
+            task["subtasks"] = [t.dump() for t in self.subtasks]
+        if self.run is not None:
+            task["run"] = self.run
+        if self.shell is not None:
+            task["shell"] = self.shell
+        if self.srcinfo is not None:
+            task["srcinfo"] = self.srcinfo.dump()
+
+        return task
+
+    def __hash__(self):
+        return id(self)
+

dv_flow/mgr/task_def.py

@@ -22,9 +22,10 @@
 import pydantic
 import pydantic.dataclasses as dc
 import enum
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 from typing import Any, Dict, List, Union, Tuple
 from .param_def import ParamDef
+from .srcinfo import SrcInfo
 from .task_output import TaskOutput
 
 @dc.dataclass
@@ -55,16 +56,27 @@ class StrategyDef(BaseModel):
         default=None,
         description="Matrix of parameter values to explore")
 
-class TaskExecDef(BaseModel):
+class TaskBodyDef(BaseModel):
+    model_config = ConfigDict(extra='forbid')
     pytask : Union[str, None] = dc.Field(
         default=None,
         description="Python method to execute to implement this task")
-    pydep  : Union[str, None] = dc.Field(
+    tasks: Union[List['TaskDef'],None] = dc.Field(
+        default_factory=list,
+        description="Sub-tasks")
+    shell: Union[str, None] = dc.Field(
         default=None,
-        description="Python method to check up-to-date status for this task")
+        description="Specifies the shell to run")
+    run: str = dc.Field(
+        default=None,
+        description="Shell command to execute for this task")
+#    pydep  : Union[str, None] = dc.Field(
+#        default=None,
+#        description="Python method to check up-to-date status for this task")
 
 class TasksBuilder(BaseModel):
     # TODO: control how much data this task is provided?
+    srcinfo : SrcInfo = dc.Field(default=None)
     pydef : Union[str, None] = dc.Field(
         default=None,
         description="Python method to build the subgraph")
@@ -76,23 +88,34 @@ class Tasks(BaseModel):
 
 class TaskDef(BaseModel):
     """Holds definition information (ie the YAML view) for a task"""
-    name : str = dc.Field(
+    name : Union[str, None] = dc.Field(
         title="Task Name",
-        description="The name of the task")
-    fullname : str = dc.Field(default=None)
-#    type : Union[str,TaskSpec] = dc.Field(default_factory=list)
+        description="The name of the task",
+        default=None)
+    override : Union[str, None] = dc.Field(
+        title="Overide Name",
+        description="The name of the task to override",
+        default=None)
     uses : str = dc.Field(
         default=None,
         title="Base type",
         description="Task from which this task is derived")
+    body: List['TaskDef'] = dc.Field(
+        default_factory=list,
+        description="Sub-tasks")
     pytask : str = dc.Field(
         default=None,
-        title="Python method name",
-        description="Python method to execute to implement this task")
+        description="Python-based implementation (deprecated)")
+    run : str = dc.Field(
+        default=None,
+        description="Shell-based implementation")
+    shell: str = dc.Field(
+        default="bash",
+        description="Shell to use for shell-based implementation")
     strategy : StrategyDef = dc.Field(
         default=None)
-    tasks: Union[List['TaskDef'], TasksBuilder] = dc.Field(
-        default_factory=list,
+    tasks: Union[List['TaskDef'], TasksBuilder, None] = dc.Field(
+        default=None,
         description="Sub-tasks")
     desc : str = dc.Field(
         default="",
@@ -102,7 +125,6 @@ class TaskDef(BaseModel):
         default="",
         title="Task documentation",
         description="Full documentation of the task")
-#    needs : List[Union[str,NeedSpec,TaskSpec]] = dc.Field(
     needs : List[Union[str]] = dc.Field(
         default_factory=list, 
         description="List of tasks that this task depends on")
@@ -119,18 +141,8 @@ class TaskDef(BaseModel):
     consumes : Union[ConsumesE, List[Any], None] = dc.Field(
         default=None,
         description="Specifies matching patterns for parameter sets that this task consumes")
+    srcinfo : SrcInfo = dc.Field(default=None)
     
-    def __init__(self, **data):
-#        print("--> task_def %s" % str(data))
-        super().__init__(**data)
-#        print("<-- task_def %s" % str(data))
-    
-    def copy(self) -> 'TaskDef':
-        ret = TaskDef(
-            name=self.name,
-            type=self.type,
-            depends=self.depends.copy())
-        return ret 
 
 TaskDef.model_rebuild()