dv-flow-mgr 0.0.1.13931887892a1__py3-none-any.whl → 0.0.1.13959047687a1__py3-none-any.whl

dv_flow/mgr/__init__.py

@@ -21,7 +21,6 @@
 #****************************************************************************
 from .package_def import *
 from .pkg_rgy import PkgRgy
-from .task_graph_runner import *
 from .task import *
 from .task_data import *
 from .task_graph_builder import TaskGraphBuilder

dv_flow/mgr/cmds/cmd_run.py

@@ -23,10 +23,8 @@ import asyncio
 import os
 import logging
 from typing import ClassVar
-from ..task_graph_runner import TaskGraphRunner
 from ..util import loadProjPkgDef
 from ..task_graph_builder import TaskGraphBuilder
-from ..task_graph_runner_local import TaskGraphRunnerLocal
 from ..task_runner import TaskSetRunner
 from ..task_listener_log import TaskListenerLog
 from ..pkg_rgy import PkgRgy

dv_flow/mgr/package_def.py

@@ -33,6 +33,7 @@ from typing import Any, Dict, List, Callable, Tuple, ClassVar, Union
 from .fragment_def import FragmentDef
 from .package import Package
 from .package_import_spec import PackageImportSpec, PackageSpec
+from .param_def import ParamDef
 from .task_node import TaskNodeCtor, TaskNodeCtorProxy, TaskNodeCtorTask
 from .task_ctor import TaskCtor
 from .task_def import TaskDef, TaskSpec
@@ -41,13 +42,26 @@ from .type_def import TypeDef
 
 
 class PackageDef(BaseModel):
-    name : str
+    name : str = dc.Field(
+        description="Name of the package")
     params : Dict[str,Any] = dc.Field(default_factory=dict)
     type : List[PackageSpec] = dc.Field(default_factory=list)
-    tasks : List[TaskDef] = dc.Field(default_factory=list)
-    imports : List[Union[str,PackageImportSpec]] = dc.Field(default_factory=list)
-    fragments: List[str] = dc.Field(default_factory=list)
+    tasks : List[TaskDef] = dc.Field(
+        default_factory=list,
+        description="List of tasks defined in the package")
+    imports : List[Union[str,PackageImportSpec]] = dc.Field(
+        default_factory=list,
+        description="List of packages to import")
+    fragments: List[str] = dc.Field(
+        default_factory=list,
+        description="List of fragments to include")
     types : List[TypeDef] = dc.Field(default_factory=list)
+    uses : str = dc.Field(
+        default=None,
+        description="Name of the package to use as a base")
+    params : List[ParamDef] = dc.Field(
+        default_factory=list, alias="with",
+        description="List of package parameters to set")
 
     fragment_l : List['FragmentDef'] = dc.Field(default_factory=list, exclude=True)
     subpkg_m : Dict[str,'PackageDef'] = dc.Field(default_factory=dict, exclude=True)

dv_flow/mgr/param_def.py

@@ -36,6 +36,7 @@ class ComplexType(BaseModel):
 class ParamDef(BaseModel):
     doc : str = None
     type : Union[str, 'ComplexType'] = None
+    derived : bool = Field(default=False)
     value : Union[Any, None] = None
     append : Union[Any, None] = None
     prepend : Union[Any, None] = None

dv_flow/mgr/std/exec.py

@@ -20,23 +20,100 @@
 #*
 #****************************************************************************
 import asyncio
+import os
 import logging
-from dv_flow.mgr import TaskDataResult
+from pydantic import BaseModel
+from dv_flow.mgr import TaskDataResult, TaskMarker
 
 _log = logging.getLogger("Exec")
 
-class Memento()
+class Memento(BaseModel):
+    timestamp_file : str
+    timestamp : float
 
 async def Exec(runner, input) -> TaskDataResult:
     _log.debug("TaskExec run: %s: cmd=%s" % (input.name, input.params.command))
 
+    # Running the command is gated by inputs to the task
+    need_to_run = input.params.when == "always" or input.changed
+
+    status = 0
+    markers = []
+
+    if need_to_run:
+        fp = open(os.path.join(input.rundir, "run.log"), "w")
+        fp.write("Command: %s\n" % input.params.command)
+        proc = await asyncio.create_subprocess_shell(
+            input.params.command,
+            stdout=fp,
+            stderr=asyncio.subprocess.STDOUT)
+
+        result = await proc.wait()
+
+        status = proc.returncode
+
+        if status != 0:
+            markers.append(TaskMarker(
+                message="Command \"%s\" failed with status %d" % (
+                    input.params.command, status), severity="error"))
+
+    changed = False
+    ex_memento = input.memento
+    memento = None
+
+    if ex_memento is not None and status == 0:
+        ex_memento_o = None
+        try:
+            ex_memento_o = Memento(**ex_memento)
+        except Exception as e:
+            _log.debug("Failed to deserialize memento: %s" % str(e))
+
+        if ex_memento_o is not None:
+            if input.params.timestamp is not None and input.params.timestamp != "":
+                # Have a file to use
+                if os.path.isfile(os.path.join(input.rundir, input.params.timestamp)):
+                    ts_file = os.path.join(input.rundir, input.params.timestamp)
+                    ts = os.path.getmtime(ts_file)
+
+                    if ts_file != ex_memento_o.timestamp_file or ts != ex_memento_o.timestamp:
+                        changed = True
+
+                    memento = Memento(timestamp_file=ts_file, timestamp=ts)
+                else:
+                    markers.append(TaskMarker(
+                        message="Timestamp file %s not found" % (
+                            os.path.join(input.rundir, input.params.timestamp)),
+                        severity="error"))
+                    status = 1
+            else:
+                memento = Memento()
+        else:
+            changed = True
+        memento = ex_memento
+    else:
+        # No existing memento, or the task failed
+        changed = (status == 0)
+
+    if input.params.when == "changed":
+        changed = input.changed
+
+    if input.params.timestamp is not None and input.params.timestamp != "":
+        # Check the timestamp file
+        try:
+            with open(input.params.timestamp_file, "r") as f:
+                timestamp = float(f.read())
+        except FileNotFoundError:
+            timestamp = 0.0
+
+        if timestamp < input.params.timestamp:
+            changed = True
 
-    proc = await asyncio.create_subprocess_shell(
-        input.params.command,
-        stdout=asyncio.subprocess.PIPE,
-        stderr=asyncio.subprocess.PIPE)
     
-    await proc.wait()
 
-    return TaskDataResult()
+    return TaskDataResult(
+        status=status,
+        changed=changed,
+        markers=markers,
+        memento=memento)
+
 

dv_flow/mgr/std/flow.dv

@@ -1,5 +1,27 @@
 
-# yaml-language-server: $schema=https://dv-flow.github.io/dv-flow.schema.json
+# yaml-language-server: $schema=https://dv-flow.github.io/flow.dv.schema.json
+
+#****************************************************************************
+#* flow.dv
+#*
+#* Copyright 2023-2025 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*  
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*  
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
 
 package:
   name: std
@@ -12,18 +34,26 @@ package:
         type: str
         value: ""
   - name: FileSet
+    doc: |
+      Creates a fileset from a list of files or glob patterns
     pytask: dv_flow.mgr.std.fileset.FileSet
     passthrough: true
     with:
       base:
+        doc: |
+          Specifies the base directory for the fileset.
         type: str
         value: ""
       type:
+        doc: |
+          Specifies the file type (eg verilogSource) for the fileset.
         type: str
         value: ""
       incdirs:
         type: list
       include:
+        doc: |
+          Specifies a list of files or a glob pattern for the files to include
         type: str
         value: ""
       exclude:

dv_flow/mgr/task.py

@@ -26,7 +26,6 @@ import logging
 from pydantic import BaseModel
 from typing import Any, Callable, ClassVar, Dict, List, Tuple
 from .task_data import TaskData, TaskDataInput, TaskDataOutput, TaskDataResult
-from .task_memento import TaskMemento
 from .task_ctor import TaskParamsCtor
 from .task_params_ctor import TaskParamsCtor
 from .task_runner import TaskRunner

dv_flow/mgr/task_ctor.py

@@ -38,7 +38,7 @@ class TaskCtor(object):
 
     _log : ClassVar = logging.getLogger("TaskCtor")
 
-    def mkTask(self, name : str, srcdir=None) -> 'Task':
+    def mkTask(self, name : str, srcdir=None) -> 'TaskNode':
         """Creates a task object"""
         if srcdir is None:
             srcdir = self.srcdir

dv_flow/mgr/task_data.py

@@ -27,14 +27,25 @@ from typing import Any, ClassVar, Dict, Set, List, Tuple
 from .fileset import FileSet
 from toposort import toposort
 
+class SeverityE(enum.Enum):
+    Info = "info"
+    Warning = "warning"
+    Error = "error"
+
 class TaskMarkerLoc(BaseModel):
+    """
+    Captures the source location of a marker
+    """
     path : str
     line : int = dc.Field(default=-1)
     pos : int = dc.Field(default=-1)
 
 class TaskMarker(BaseModel):
+    """
+    Captures marker data produced by a task.
+    """
     msg : str
-    severity : str
+    severity : SeverityE
     loc : TaskMarkerLoc = dc.Field(default=None)
 
 class TaskParameterSet(BaseModel):
@@ -43,6 +54,15 @@ class TaskParameterSet(BaseModel):
     seq : int  = -1 # Order in which the param-set must appear
 
 class TaskDataInput(BaseModel):
+    """
+    Input data to a task
+    - name - name of the task
+    - changed - indicates whether any of this task's dependencies have changed
+    - rundir - directory in which the task is to be run
+    - params - parameters to the task
+    - inputs - list of parameter sets 'consumed' by this task
+    - memento - memento data previously returned by this task. None if no memento is available
+    """
     name : str
     changed : bool
     srcdir : str
@@ -52,6 +72,14 @@ class TaskDataInput(BaseModel):
     memento : Any
 
 class TaskDataResult(BaseModel):
+    """
+    Result data from a task
+    - changed - indicates whether the task modified its result data
+    - output - list of output parameter sets
+    - memento - memento data to be passed to the next invocation of the task
+    - markers - list of markers produced by the task
+    - status - status code (0=success, non-zero=failure)
+    """
     changed : bool = dc.Field(default=True)
     output : List[Any] = dc.Field(default_factory=list)
     memento : Any = dc.Field(default=None)

dv_flow/mgr/task_def.py

@@ -20,6 +20,7 @@
 #*
 #****************************************************************************
 import pydantic.dataclasses as dc
+import enum
 from pydantic import BaseModel
 from typing import Any, Dict, List, Union, Tuple
 from .param_def import ParamDef
@@ -35,20 +36,60 @@ class NeedSpec(object):
     name : str
     block : bool = False
 
+class RundirE(enum.Enum):
+    Unique = "unique"
+    Inherit = "inherit"
+
+class StrategyDef(BaseModel):
+    matrix : Dict[str,List[Any]] = dc.Field(
+        default_factory=dict,
+        description="Matrix of parameter values to explore")
+
 class TaskDef(BaseModel):
     """Holds definition information (ie the YAML view) for a task"""
-    name : str
+    name : str = dc.Field(
+        title="Task Name",
+        description="The name of the task")
     fullname : str = dc.Field(default=None)
 #    type : Union[str,TaskSpec] = dc.Field(default_factory=list)
-    uses : str = dc.Field(default=None)
-    pytask : str = dc.Field(default=None)
-    desc : str = dc.Field(default="")
-    doc : str = dc.Field(default="")
-    needs : List[Union[str,NeedSpec,TaskSpec]] = dc.Field(default_factory=list, alias="needs")
-    params: Dict[str,Union[str,list,ParamDef]] = dc.Field(default_factory=dict, alias="with")
-    passthrough: bool = dc.Field(default=False)
-    consumes : List[Any] = dc.Field(default_factory=list)
-    tasks: List['TaskDef'] = dc.Field(default_factory=list)
+    uses : str = dc.Field(
+        default=None,
+        title="Base type",
+        description="Task from which this task is derived")
+    pytask : str = dc.Field(
+        default=None,
+        title="Python method name",
+        description="Python method to execute to implement this task")
+    strategy : StrategyDef = dc.Field(
+        default=None)
+    tasks: List['TaskDef'] = dc.Field(
+        default_factory=list,
+        description="Sub-tasks")
+    desc : str = dc.Field(
+        default="",
+        title="Task description",
+        description="Short description of the task's purpose")
+    doc : str = dc.Field(
+        default="",
+        title="Task documentation",
+        description="Full documentation of the task")
+#    needs : List[Union[str,NeedSpec,TaskSpec]] = dc.Field(
+    needs : List[Union[str]] = dc.Field(
+        default_factory=list, 
+        description="List of tasks that this task depends on")
+    params: Dict[str,Union[str,list,ParamDef]] = dc.Field(
+        default_factory=dict, 
+        alias="with",
+        description="Parameters for the task")
+    rundir : RundirE = dc.Field(
+        default=RundirE.Unique,
+        description="Specifies handling of this tasks's run directory")
+    passthrough: bool = dc.Field(
+        default=False,
+        description="Specifies whether this task should pass its inputs to its output")
+    consumes : List[Any] = dc.Field(
+        default_factory=list,
+        description="Specifies matching patterns for parameter sets that this task consumes")
 
 #    out: List[TaskOutput] = dc.Field(default_factory=list)
 

dv_flow/mgr/task_listener_log.py

@@ -56,8 +56,15 @@ class TaskListenerLog(object):
                     "error": "[red]E[/red]",
                 }
                 for m in task.result.markers:
+                    severity_s = str(m.severity)
+
+                    if severity_s in sev_pref_m.keys():
+                        sev_pref = sev_pref_m[severity_s]
+                    else:
+                        sev_pref = ""
+
                     msg = "  %s %s: %s" % (
-                        sev_pref_m[m.severity], 
+                        sev_pref,
                         task.name,
                         m.msg)
 

dv_flow/mgr/task_node.py

@@ -128,8 +128,8 @@ class TaskNode(object):
 
         # Set variables from the inputs
         for need in self.needs:
-            for name,value in {"rundir" : need.rundir}.items():
-                eval.setVar("%s.%s" % (need.name, name), value)
+            for name,value in {"rundir" : need[0].rundir}.items():
+                eval.setVar("%s.%s" % (need[0].name, name), value)
 
         # Default inputs is the list of parameter sets that match 'consumes'
         inputs = []

dv_flow/mgr/{task_impl_data.py → task_node_compound.py}

@@ -1,5 +1,5 @@
 #****************************************************************************
-#* task_impl_data.py
+#* task_node_compound.py
 #*
 #* Copyright 2023-2025 Matthew Ballance and Contributors
 #*
@@ -19,19 +19,19 @@
 #*     Author: 
 #*
 #****************************************************************************
-from pydantic import BaseModel
-from typing import Any, ClassVar, Dict, Set, List, Tuple
+import dataclasses as dc
+from .task_node import TaskNode
+from .task_data import TaskDataResult, TaskDataInput, TaskDataOutput
+from .task_runner import TaskRunner
+from typing import Any, List
 
-class TaskImplParams(BaseModel):
-    pass
-
-class TaskImplSourceData(BaseModel):
-    params : Any
-    changed : bool
-    memento : Any
-
-class TaskImplResultData(BaseModel):
-    data : List[Any]
-    changed : bool
-    memento : Any
+@dc.dataclass
+class TaskNodeCompound(TaskNode):
+    tasks : List[TaskNode] = dc.field(default_factory=list)
 
+    async def do_run(self, 
+                     runner : TaskRunner, 
+                     rundir, 
+                     memento : Any=None) -> TaskDataResult:
+        pass
+    pass

dv_flow/mgr/task_runner.py

@@ -29,7 +29,7 @@ from datetime import datetime
 from toposort import toposort
 from typing import Any, Callable, ClassVar, List, Tuple, Union
 from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
-from .task_node import TaskNode
+from .task_node import TaskNode, RundirE
 
 @dc.dataclass
 class TaskRunner(object):
@@ -140,10 +140,14 @@ class TaskSetRunner(TaskRunner):
                     memento = src_memento.get(t.name, None)
                     dirname = t.name
                     invalid_chars_pattern = r'[\/:*?"<>|#%&{}\$\\!\'`;=@+]'
-                    # Replace invalid characters with the replacement string.
-                    dirname = re.sub(invalid_chars_pattern, '_', dirname)
 
-                    rundir = os.path.join(self.rundir, dirname)
+                    if t.rundir_t == RundirE.Unique:
+                        # Replace invalid characters with the replacement string.
+                        dirname = re.sub(invalid_chars_pattern, '_', dirname)
+
+                        rundir = os.path.join(self.rundir, dirname)
+                    else:
+                        rundir = self.rundir
 
                     if not os.path.isdir(rundir):
                         os.makedirs(rundir, exist_ok=True)

{dv_flow_mgr-0.0.1.13931887892a1.dist-info → dv_flow_mgr-0.0.1.13959047687a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: dv-flow-mgr
-Version: 0.0.1.13931887892a1
+Version: 0.0.1.13959047687a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License
@@ -212,3 +212,4 @@ Requires-Dist: pyyaml
 Requires-Dist: rich
 Requires-Dist: toposort
 Requires-Dist: svdep
+Dynamic: license-file

{dv_flow_mgr-0.0.1.13931887892a1.dist-info → dv_flow_mgr-0.0.1.13959047687a1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-dv_flow/mgr/__init__.py,sha256=63pCyRCpFj4Rdc3Lz-HKaVmFAI4BAma1OEy6dDRQeXM,1134
+dv_flow/mgr/__init__.py,sha256=XWO0B0wXUyhqGzu7Pt2f-r-_k7CHPZOOxAaEPUgjaAs,1101
 dv_flow/mgr/__main__.py,sha256=yH_LoWlqPc5y9fKATUyDLNVobUnjWtvcqgqcP2kc628,2213
 dv_flow/mgr/eval_jq.py,sha256=bRsHtaN51qIOiZK1VJV52W0-vj5VH0nQ7XIFebZi5kI,1129
 dv_flow/mgr/expr_eval.py,sha256=N_8hRIgzJK9JVqhRt8F9rc4S7AAdKHMMltafqk6KhJs,3113
@@ -7,42 +7,38 @@ dv_flow/mgr/fileset.py,sha256=cfKem8w5Xw4AUQkC0gyCXof2OlnoJ1t06Iw7IvMyihs,1245
 dv_flow/mgr/fragment_def.py,sha256=nwxx3TW7UMa6uFMp0QEKgJdmOsO2iX89cwyD2beaGoU,1646
 dv_flow/mgr/out,sha256=d8GGBi3J43fhdLBlnsUbzBfRe0TD0QTP3nOTz54l2bI,200
 dv_flow/mgr/package.py,sha256=D-jH3jqoiQG3xZIYgteFLZ4vQ0ZfkbhnW1u6wlyAJMc,1684
-dv_flow/mgr/package_def.py,sha256=7WUStRgJcUY0FEKx3NYOs7hyLel0YI1lfRMutTgcblM,17193
+dv_flow/mgr/package_def.py,sha256=ZGanENWXRqYDKL8qiY4JzsGYLm2iXU85XfSAsq_7rIg,17719
 dv_flow/mgr/package_import_spec.py,sha256=aZMpnS9a5NFY76_pYXEuO3-Mkc_xFzy73fdrUe_54Dc,1760
 dv_flow/mgr/param.py,sha256=kkxMRGf6mPjSZJsjgLKH2vJL62Sn0ZESvjBLkEYOp20,1386
-dv_flow/mgr/param_def.py,sha256=ObRf289sI1xzOudjO9MzyTVuKsz3AS949XA4Deyz-kE,1549
+dv_flow/mgr/param_def.py,sha256=snLHgafVqqWcDthJUOm9zvKocFsb7-wx2WTc5ltAlhg,1591
 dv_flow/mgr/param_ref_eval.py,sha256=5yH37oIX6f2qmk7GfRgNT5qZx0jm3CJFgB9lLDZZ1yQ,1981
 dv_flow/mgr/parsetab.py,sha256=I-p3nC60t9jiNtPhKyl_sE92SiP96zJLnNdydcLy33g,3780
 dv_flow/mgr/pkg_rgy.py,sha256=d1nIjRm3ymMNJT-yiMDxCS6bFisTPvLMqh5VrfsHVKM,5404
-dv_flow/mgr/task.py,sha256=BmS0wd51l4tpTrsit3-ugCB5VGnwpMJNl0dfJfHS4DQ,5981
-dv_flow/mgr/task_ctor.py,sha256=YDYb5uW9Js7MnA8GqbYJljNCbeJFKGveZ2v-IJRgIa8,2221
-dv_flow/mgr/task_data.py,sha256=ysawiUcbQu-ImNSgU5LsfOWlBKdfBt_8juwSacnAqAQ,11437
-dv_flow/mgr/task_def.py,sha256=0rVlDjNF39PQ_0uzkSuOgvE2sZjFwpTAR8HIu_Arbog,2140
-dv_flow/mgr/task_exec_data.py,sha256=-mh9G2-iq6ioB7v7QrHiQAOEwd_hDKhnVf2Mb_IHRtg,1483
+dv_flow/mgr/task.py,sha256=464P7RAiHHiUcDzzoc1XSQj2ihwT6Lcuxr1U2qGfgLA,5943
+dv_flow/mgr/task_ctor.py,sha256=sPbDqRBMlbMAUddgtT2c7xWzbrYnZwcXf16wj-q7_5Y,2225
+dv_flow/mgr/task_data.py,sha256=IBkxdqpWDMqD1S06u4ohG9pOTdiQPCePT4ZJknPAAw0,12401
+dv_flow/mgr/task_def.py,sha256=295FqKPcZKlgJENRdPdk_io_2lvVN2c3T3CezLEt1UU,3497
 dv_flow/mgr/task_graph_builder.py,sha256=EYFsMZCDDHMEFWZOVSpPb7KBVE6N6XyL4KFzJVrPSPE,10106
-dv_flow/mgr/task_graph_runner.py,sha256=LZ5cTXLQlcK348AxTYKqfpsFFmxA3aseGfholvQW39s,2187
-dv_flow/mgr/task_graph_runner_local.py,sha256=quAxmF1go2iVhBDieksg3brAZukIvpvF-8oJ_YXiauE,4692
-dv_flow/mgr/task_impl_data.py,sha256=jpO7kQ6AjYcOOfpgFIPCEYA1pQbVwFcIY_SY3Q53eGk,1163
-dv_flow/mgr/task_listener_log.py,sha256=b7caUHERGkN6eT8IinBJTdZpwASUs8xAqDmtEymMKzQ,3529
-dv_flow/mgr/task_memento.py,sha256=CV3d1J4ADhMpqvYPfWvUI7duSFCbLEzY41y2GZBHIhM,1101
-dv_flow/mgr/task_node.py,sha256=DL7gI79uu0yoaz2J7PJhr91isL42agATbLDO0ngGqi4,15215
+dv_flow/mgr/task_listener_log.py,sha256=vhyLegEown54RY3sRqFTgqhxrRAHqp_YhomObdnZ0vs,3743
+dv_flow/mgr/task_node.py,sha256=H0OuBrridAhYutDUap-V09T-2xZNaJKkfayeqi1UrTw,15221
+dv_flow/mgr/task_node_compound.py,sha256=1NZvPh0EkyJ_1Vj4H9RNmFQKcmGavybZWubNIOyl5AA,1330
 dv_flow/mgr/task_output.py,sha256=ZwyvwnYj_gHOEFAEOH3m24Xfc4Cn77hb1j7LkX8_3C4,1086
 dv_flow/mgr/task_params_ctor.py,sha256=BPkbnoCtzhCxc1g8CJ6VimCcm5UAu92PXeDMhQ4lYsQ,1957
-dv_flow/mgr/task_runner.py,sha256=TB4feCEItxufUyyBSwNNY3OkQ0yjSO0fUfjga0u7rsg,8790
+dv_flow/mgr/task_runner.py,sha256=6yoNzAGb79vgYG95ku_4Kr90cgztdisSICFfV5Ykf9c,8936
 dv_flow/mgr/type.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dv_flow/mgr/type_def.py,sha256=NDeyplKrPnWwEv4yHkhXEMK9d9j39b5MOeLB-1Mchqo,1095
 dv_flow/mgr/util.py,sha256=BLp-25Ph9roqo1O3VHI5sKhwxQ0wUU7znFwcSKt0hpA,1442
-dv_flow/mgr/cmds/cmd_run.py,sha256=lNMvpiyKf-Ehs0GUkzcOIoJxaXgeLKv6UFNakcm9_hw,3303
+dv_flow/mgr/cmds/cmd_run.py,sha256=J6uCzzhtGHD9MYkvE0PpX4dkk5KlY3iW7yZ29iBO3js,3196
 dv_flow/mgr/share/flow.json,sha256=lNmZex9NXkYbyb2aZseQfUOkV9CMyfH0iLODEI7EPBw,5096
 dv_flow/mgr/std/create_file.py,sha256=wmn5N_mObx_wr2LPdKOVbNmdM71hhs3UXOv6Ap9l3Ts,2726
-dv_flow/mgr/std/exec.py,sha256=kU9S8mIBDvZFqcWGGe9RQIfeldDw2g2eGBBh-po0ePQ,1302
+dv_flow/mgr/std/exec.py,sha256=UChqa_tAjvdB1NjqTsvlPgFomP8hMsX2rAOPyUonNpk,3896
 dv_flow/mgr/std/fileset.py,sha256=5IxS6T-x0wzA6fdEQcfHZ9kNP8IpH9hfJ3UMhL4A6Iw,3911
-dv_flow/mgr/std/flow.dv,sha256=Ehw0rrFcHoXt3WudeJ5PyPXpVtUcvvWGm43ikUWl-lE,2602
+dv_flow/mgr/std/flow.dv,sha256=rl0VZt_H-TyIJq0iERai07aDDZHI86mNozs4gUxidDA,3759
 dv_flow/mgr/std/message.py,sha256=_Gox9oBzL9ZYG1JfJ-WYPXojVLqGNaKxZ9tpLVT0LO0,1035
 dv_flow/mgr/std/task_null.py,sha256=dw6LXBXVwth6gLPeduDvlz5znAhcVpDH8r1DticD-0w,1041
-dv_flow_mgr-0.0.1.13931887892a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dv_flow_mgr-0.0.1.13931887892a1.dist-info/METADATA,sha256=D2JLp63FgCw_yZC_bdxrl76DDLMzyI4By_K7mhYYhmM,13314
-dv_flow_mgr-0.0.1.13931887892a1.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-dv_flow_mgr-0.0.1.13931887892a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
-dv_flow_mgr-0.0.1.13931887892a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
-dv_flow_mgr-0.0.1.13931887892a1.dist-info/RECORD,,
+dv_flow_mgr-0.0.1.13959047687a1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-0.0.1.13959047687a1.dist-info/METADATA,sha256=BzJ7u3Dpcs3DIfCz-igUqT2LZi1tnsJNRuvOsOmk9Ws,13336
+dv_flow_mgr-0.0.1.13959047687a1.dist-info/WHEEL,sha256=tTnHoFhvKQHCh4jz3yCn0WPTYIy7wXx3CJtJ7SJGV7c,91
+dv_flow_mgr-0.0.1.13959047687a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
+dv_flow_mgr-0.0.1.13959047687a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
+dv_flow_mgr-0.0.1.13959047687a1.dist-info/RECORD,,

{dv_flow_mgr-0.0.1.13931887892a1.dist-info → dv_flow_mgr-0.0.1.13959047687a1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (76.1.0)
+Generator: setuptools (77.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

dv_flow/mgr/task_exec_data.py

@@ -1,43 +0,0 @@
-#****************************************************************************
-#* task_exec_data.py
-#*
-#* Copyright 2023-2025 Matthew Ballance and Contributors
-#*
-#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
-#* not use this file except in compliance with the License.  
-#* You may obtain a copy of the License at:
-#*  
-#*   http://www.apache.org/licenses/LICENSE-2.0
-#*  
-#* Unless required by applicable law or agreed to in writing, software 
-#* distributed under the License is distributed on an "AS IS" BASIS, 
-#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
-#* See the License for the specific language governing permissions and 
-#* limitations under the License.
-#*
-#* Created on:
-#*     Author: 
-#*
-#****************************************************************************
-import pydantic.dataclasses as dc
-from pydantic import BaseModel
-from typing import Any, Dict, List
-
-
-class TaskExecData(BaseModel):
-    """Data from a single exection of a task"""
-    name : str
-    start : str
-    finish : str
-    status : int
-    memento : Any
-    markers : List[Any]
-
-class FlowExecData(BaseModel):
-    """
-    Data from multiple tasks executions. 'info' holds information
-    across multiple flow invocations. 'tasks' holds the names of
-    tasks executed in the most-recent invocation.
-    """
-    info : Dict[str, TaskExecData] = dc.Field(default_factory=dict)
-    tasks : List[str] = dc.Field(default_factory=list)

dv_flow/mgr/task_graph_runner.py

@@ -1,70 +0,0 @@
-#****************************************************************************
-#* task_graph_runner.py
-#*
-#* Copyright 2023-2025 Matthew Ballance and Contributors
-#*
-#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
-#* not use this file except in compliance with the License.  
-#* You may obtain a copy of the License at:
-#*
-#*   http://www.apache.org/licenses/LICENSE-2.0
-#*
-#* Unless required by applicable law or agreed to in writing, software 
-#* distributed under the License is distributed on an "AS IS" BASIS, 
-#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
-#* See the License for the specific language governing permissions and 
-#* limitations under the License.
-#*
-#* Created on:
-#*     Author: 
-#*
-#****************************************************************************
-import asyncio
-import os
-import dataclasses as dc
-from typing import Any, Callable, ClassVar, Dict, List, Tuple
-from .task import Task
-from .task_data import TaskData
-from .task_runner import TaskRunner
-
-@dc.dataclass
-class TaskGraphRunner(TaskRunner):
-    """Session manages execution of a task graph"""
-
-    _inst : ClassVar['TaskGraphRunner'] = None
-
-
-    # Search path for .dfs files
-    create_subprocess : Callable = asyncio.create_subprocess_exec
-    _root_dir : str = None
-
-    async def exec(self, *args, **kwargs):
-        return self.create_subprocess(*args, **kwargs)
-
-    # def load(self):
-    #     if not os.path.isdir(self.srcdir):
-    #         raise Exception("Root directory %s does not exist" % self.srcdir)
-
-    #     if not os.path.isfile(os.path.join(self.srcdir, "flow.dv")):
-    #         raise Exception("No root flow file")
-
-    #     self._root_dir = os.path.dirname(self.srcdir)
-    #     self.package = PackageDef.load(os.path.join(self.srcdir, "flow.dv"), [])
-
-    #     return self.package
-
-
-    async def run(self, task : str) -> 'TaskData':
-        impl = self.mkTaskGraph(task)
-        return await impl.do_run()
-    
-    async def runTask(self, task : Task) -> 'TaskData':
-        return await task.do_run()
-    
-    def queueTask(self, task : Task):
-        """Queue a task for execution"""
-        pass
-
-
-
-

dv_flow/mgr/task_graph_runner_local.py

@@ -1,141 +0,0 @@
-#****************************************************************************
-#* task_graph_runner_local.py
-#*
-#* Copyright 2023-2025 Matthew Ballance and Contributors
-#*
-#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
-#* not use this file except in compliance with the License.  
-#* You may obtain a copy of the License at:
-#*
-#*   http://www.apache.org/licenses/LICENSE-2.0
-#*
-#* Unless required by applicable law or agreed to in writing, software 
-#* distributed under the License is distributed on an "AS IS" BASIS, 
-#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
-#* See the License for the specific language governing permissions and 
-#* limitations under the License.
-#*
-#* Created on:
-#*     Author: 
-#*
-#****************************************************************************
-import asyncio
-import os
-import yaml
-import dataclasses as dc
-import logging
-from toposort import toposort
-from typing import Any, Callable, ClassVar, Coroutine, Dict, List, Tuple, Union
-from .fragment_def import FragmentDef
-from .package import Package
-from .pkg_rgy import PkgRgy
-from .package_def import PackageDef, PackageSpec
-from .task import Task
-from .task_data import TaskData
-from .task_graph_runner import TaskGraphRunner
-
-@dc.dataclass
-class TaskGraphRunnerLocal(TaskGraphRunner):
-    """Session manages execution of a task graph"""
-
-    rundir : str
-    nproc : int = 4
-    done_task_m : Dict = dc.field(default_factory=dict)
-    _workers : List = dc.field(default_factory=list)
-
-    _inst : ClassVar['TaskGraphRunner'] = None
-
-    # Search path for .dfs files
-    create_subprocess : Callable = asyncio.create_subprocess_exec
-    _root_dir : str = None
-    _log : ClassVar = logging.getLogger("TaskGraphRunnerLocal")
-
-    def __post_init__(self):
-        if self.nproc == -1:
-            self.nproc = os.cpu_count()
-        for _ in range(self.nproc):
-            self._workers.append(LocalRunnerWorker(self))
-
-
-    async def exec(self, *args, **kwargs):
-        return await self.create_subprocess(*args, **kwargs)
-
-    async def run(self, task : Union[Task,List[Task]]) -> List['TaskData']:
-        if isinstance(task, Task):
-            unwrap = True
-            task = [task]
-        else:
-            unwrap = False
-
-        dep_m = {}
-        task_m = {}
-
-        for t in task:
-            self._mkDeps(dep_m, task_m, t)
-
-        self._log.debug("dep_m: %s" % str(dep_m))
-
-        order = list(toposort(dep_m))
-        
-        self._log.debug("order: %s" % str(order))
-
-        active_task_l : List[Tuple[Task,Coroutine]]= []
-        # Now, iterate over the concurrent sets
-        for active_s in order:
-
-            # Check to see if all tasks are complete
-            done = True
-            for t in active_s:
-                while len(active_task_l) >= self.nproc and t not in self.done_task_m.keys():
-                    # Wait for at least one job to complete
-                    done, pending = await asyncio.wait(at[1] for at in active_task_l)
-                    for d in done:
-                        for i in range(len(active_task_l)):
-                            if active_task_l[i][1] == d:
-                                tt = active_task_l[i][0]
-                                self.done_task_m[tt.name] = tt
-                                active_task_l.pop(i)
-                                break
-                if t not in self.done_task_m.keys():
-                    task_t = task_m[t]
-                    coro = asyncio.Task(task_t.do_run(self))
-                    active_task_l.append((task_t, coro))
-               
-            # Now, wait for tasks to complete
-            if len(active_task_l):
-                coros = list(at[1] for at in active_task_l)
-                res = await asyncio.gather(*coros)
-
-#        print("order: %s" % str(order))
-#        
-#        run_o = list(t.do_run() for t in task)
-
-#        ret = await asyncio.gather(*run_o)
-        ret = None
-
-        if unwrap:
-            return task[0].output
-        else:
-            return list(t.output for t in task)
-        
-    def _mkDeps(self, dep_m, task_m, task):
-        if task.name not in dep_m.keys():
-            task_m[task.name] = task
-            dep_m[task.name] = set(t.name for t in task.depends)
-            for d in task.depends:
-                self._mkDeps(dep_m, task_m, d)
-    
-    async def runTask(self, task : Task) -> 'TaskData':
-        return await task.do_run()
-    
-    def queueTask(self, task : Task):
-        """Queue a task for execution"""
-        pass
-
-@dc.dataclass
-class LocalRunnerWorker(object):
-    runner : TaskGraphRunnerLocal
-    task_s : List = dc.field(default_factory=list)
-
-
-

dv_flow/mgr/task_memento.py

@@ -1,34 +0,0 @@
-#****************************************************************************
-#* task_memento.py
-#*
-#* Copyright 2023-2025 Matthew Ballance and Contributors
-#*
-#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
-#* not use this file except in compliance with the License.  
-#* You may obtain a copy of the License at:
-#*
-#*   http://www.apache.org/licenses/LICENSE-2.0
-#*
-#* Unless required by applicable law or agreed to in writing, software 
-#* distributed under the License is distributed on an "AS IS" BASIS, 
-#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
-#* See the License for the specific language governing permissions and 
-#* limitations under the License.
-#*
-#* Created on:
-#*     Author: 
-#*
-#****************************************************************************
-import pydantic.dataclasses as dc
-from pydantic import BaseModel
-from typing import Any, Dict, List
-
-class TaskMemento(BaseModel):
-#    dep_ids : List[int] = dc.Field(default_factory=list)
-#    params : Dict[str,Any] = dc.Field(default_factory=dict)
-
-    pass
-
-
-
-