dv-flow-mgr 1.10.15961872557rc0__py3-none-any.whl → 1.11.16383233052rc0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dv_flow/mgr/__init__.py +85 -3
- dv_flow/mgr/exec_callable.py +4 -0
- dv_flow/mgr/exec_gen_callable.py +1 -0
- dv_flow/mgr/package_loader.py +107 -75
- dv_flow/mgr/param_ref_eval.py +0 -3
- dv_flow/mgr/shell_callable.py +1 -0
- dv_flow/mgr/std/flow.dv +10 -0
- dv_flow/mgr/std/set_file_type.py +18 -0
- dv_flow/mgr/task.py +1 -1
- dv_flow/mgr/task_graph_builder.py +5 -4
- dv_flow/mgr/task_node_leaf.py +6 -0
- dv_flow/mgr/task_runner.py +6 -2
- dv_flow/mgr/type.py +1 -0
- {dv_flow_mgr-1.10.15961872557rc0.dist-info → dv_flow_mgr-1.11.16383233052rc0.dist-info}/METADATA +1 -1
- {dv_flow_mgr-1.10.15961872557rc0.dist-info → dv_flow_mgr-1.11.16383233052rc0.dist-info}/RECORD +19 -18
- {dv_flow_mgr-1.10.15961872557rc0.dist-info → dv_flow_mgr-1.11.16383233052rc0.dist-info}/WHEEL +0 -0
- {dv_flow_mgr-1.10.15961872557rc0.dist-info → dv_flow_mgr-1.11.16383233052rc0.dist-info}/entry_points.txt +0 -0
- {dv_flow_mgr-1.10.15961872557rc0.dist-info → dv_flow_mgr-1.11.16383233052rc0.dist-info}/licenses/LICENSE +0 -0
- {dv_flow_mgr-1.10.15961872557rc0.dist-info → dv_flow_mgr-1.11.16383233052rc0.dist-info}/top_level.txt +0 -0
dv_flow/mgr/__init__.py
CHANGED
@@ -27,11 +27,93 @@ from .task_def import *
|
|
27
27
|
from .task_graph_builder import TaskGraphBuilder
|
28
28
|
from .task_run_ctxt import TaskRunCtxt
|
29
29
|
from .task_runner import TaskRunner
|
30
|
-
from .task_node_ctor_wrapper import task
|
31
30
|
from .task_runner import TaskSetRunner
|
32
31
|
from .task_listener_log import TaskListenerLog
|
33
32
|
|
34
|
-
VERSION="1.
|
35
|
-
SUFFIX="
|
33
|
+
VERSION="1.11.0"
|
34
|
+
SUFFIX="16383233052rc0"
|
36
35
|
__version__="%s%s" % (VERSION, SUFFIX)
|
37
36
|
|
37
|
+
def task(paramT,passthrough=PassthroughE.Unused,consumes=ConsumesE.All):
|
38
|
+
"""Decorator to wrap a task method as a TaskNodeCtor"""
|
39
|
+
|
40
|
+
def wrapper(T):
|
41
|
+
from dv_flow.mgr.task_node_leaf import TaskNodeLeaf
|
42
|
+
from .param import Param
|
43
|
+
task_mname = T.__module__
|
44
|
+
task_module = sys.modules[task_mname]
|
45
|
+
task_passthrough = passthrough
|
46
|
+
task_consumes = consumes
|
47
|
+
|
48
|
+
def mkTaskParams(params : Dict) -> Any:
|
49
|
+
obj = paramT()
|
50
|
+
|
51
|
+
# Apply user-specified params
|
52
|
+
if params is not None:
|
53
|
+
for key,value in params.items():
|
54
|
+
if not hasattr(obj, key):
|
55
|
+
raise Exception("Parameters class %s does not contain field %s" % (
|
56
|
+
str(type(obj)),
|
57
|
+
key
|
58
|
+
))
|
59
|
+
else:
|
60
|
+
if isinstance(value, Param):
|
61
|
+
if value.append is not None:
|
62
|
+
ex_value = getattr(obj, key, [])
|
63
|
+
ex_value.extend(value.append)
|
64
|
+
setattr(obj, key, ex_value)
|
65
|
+
elif value.prepend is not None:
|
66
|
+
ex_value = getattr(obj, key, [])
|
67
|
+
value = value.copy()
|
68
|
+
value.extend(ex_value)
|
69
|
+
setattr(obj, key, value)
|
70
|
+
pass
|
71
|
+
else:
|
72
|
+
raise Exception("Unhandled value spec: %s" % str(value))
|
73
|
+
else:
|
74
|
+
setattr(obj, key, value)
|
75
|
+
return obj
|
76
|
+
|
77
|
+
def ctor(
|
78
|
+
builder=None,
|
79
|
+
name=None,
|
80
|
+
srcdir=None,
|
81
|
+
params=None,
|
82
|
+
needs=None,
|
83
|
+
passthrough=None,
|
84
|
+
consumes=None,
|
85
|
+
**kwargs):
|
86
|
+
if params is None:
|
87
|
+
params = mkTaskParams(kwargs)
|
88
|
+
if passthrough is None:
|
89
|
+
passthrough = task_passthrough
|
90
|
+
if consumes is None:
|
91
|
+
consumes = task_consumes
|
92
|
+
if srcdir is None:
|
93
|
+
srcdir=os.path.dirname(os.path.abspath(task_module.__file__))
|
94
|
+
|
95
|
+
print("needs: %s" % str(needs))
|
96
|
+
|
97
|
+
task_mname = T.__module__
|
98
|
+
task_module = sys.modules[task_mname]
|
99
|
+
node = TaskNodeLeaf(
|
100
|
+
name=T.__name__,
|
101
|
+
params=params,
|
102
|
+
task=T,
|
103
|
+
srcdir=srcdir,
|
104
|
+
ctxt=None,
|
105
|
+
passthrough=passthrough,
|
106
|
+
consumes=consumes,
|
107
|
+
needs=needs)
|
108
|
+
return node
|
109
|
+
# ctor = TaskNodeCtorWrapper(
|
110
|
+
# name=T.__name__,
|
111
|
+
# srcdir=os.path.dirname(os.path.abspath(task_module.__file__)),
|
112
|
+
# paramT=paramT,
|
113
|
+
# passthrough=passthrough,
|
114
|
+
# consumes=consumes,
|
115
|
+
# needs=[],
|
116
|
+
# T=T)
|
117
|
+
return ctor
|
118
|
+
return wrapper
|
119
|
+
|
dv_flow/mgr/exec_callable.py
CHANGED
@@ -10,6 +10,7 @@ from .task_data import TaskDataResult
|
|
10
10
|
@dc.dataclass
|
11
11
|
class ExecCallable(object):
|
12
12
|
body : str
|
13
|
+
srcdir : str
|
13
14
|
shell: str = "pytask"
|
14
15
|
_log : ClassVar = logging.getLogger("ExecCallable")
|
15
16
|
|
@@ -30,6 +31,9 @@ class ExecCallable(object):
|
|
30
31
|
# have a file to load
|
31
32
|
file = self.body[:ci]
|
32
33
|
method = self.body[ci+2:]
|
34
|
+
if not os.path.isabs(file):
|
35
|
+
file = os.path.join(input.srcdir, file)
|
36
|
+
|
33
37
|
spec = importlib.util.spec_from_file_location(input.name, file)
|
34
38
|
module = importlib.util.module_from_spec(spec)
|
35
39
|
sys.modules[input.name] = module
|
dv_flow/mgr/exec_gen_callable.py
CHANGED
dv_flow/mgr/package_loader.py
CHANGED
@@ -27,7 +27,9 @@ class EmptyParams(pydantic.BaseModel):
|
|
27
27
|
class SymbolScope(object):
|
28
28
|
name : str
|
29
29
|
task_m : Dict[str,Task] = dc.field(default_factory=dict)
|
30
|
+
task_elab_m : Dict[str,bool] = dc.field(default_factory=dict)
|
30
31
|
type_m : Dict[str,Type] = dc.field(default_factory=dict)
|
32
|
+
type_elab_m : Dict[str,bool] = dc.field(default_factory=dict)
|
31
33
|
override_m : Dict[str,Any] = dc.field(default_factory=dict)
|
32
34
|
|
33
35
|
def add(self, task, name):
|
@@ -599,6 +601,8 @@ class PackageLoader(object):
|
|
599
601
|
|
600
602
|
def _loadTasks(self, pkg, taskdefs : List[TaskDef], basedir : str):
|
601
603
|
self._log.debug("--> _loadTasks %s" % pkg.name)
|
604
|
+
|
605
|
+
|
602
606
|
# Declare first
|
603
607
|
tasks = []
|
604
608
|
for taskdef in taskdefs:
|
@@ -618,7 +622,8 @@ class PackageLoader(object):
|
|
618
622
|
desc=desc,
|
619
623
|
doc=doc,
|
620
624
|
package=pkg,
|
621
|
-
srcinfo=taskdef.srcinfo
|
625
|
+
srcinfo=taskdef.srcinfo,
|
626
|
+
taskdef=taskdef)
|
622
627
|
|
623
628
|
if taskdef.iff is not None:
|
624
629
|
task.iff = taskdef.iff
|
@@ -629,81 +634,92 @@ class PackageLoader(object):
|
|
629
634
|
|
630
635
|
# Now, build out tasks
|
631
636
|
for taskdef, task in tasks:
|
637
|
+
task.taskdef = taskdef
|
638
|
+
self._elabTask(task)
|
632
639
|
|
633
|
-
|
634
|
-
task.uses = self._findTaskOrType(taskdef.uses)
|
640
|
+
self._log.debug("<-- _loadTasks %s" % pkg.name)
|
635
641
|
|
636
|
-
|
637
|
-
|
638
|
-
|
639
|
-
|
640
|
-
passthrough, consumes, rundir = self._getPTConsumesRundir(taskdef, task.uses)
|
642
|
+
def _elabTask(self, task):
|
643
|
+
self._log.debug("--> _elabTask %s" % task.name)
|
644
|
+
taskdef = task.taskdef
|
641
645
|
|
642
|
-
|
643
|
-
|
644
|
-
task.
|
646
|
+
task.taskdef = None
|
647
|
+
if taskdef.uses is not None:
|
648
|
+
task.uses = self._findTaskOrType(taskdef.uses)
|
645
649
|
|
646
|
-
task.
|
647
|
-
taskdef,
|
648
|
-
|
650
|
+
if task.uses is None:
|
651
|
+
self.error("failed to resolve task-uses %s" % taskdef.uses, taskdef.srcinfo)
|
652
|
+
return
|
649
653
|
|
650
|
-
|
651
|
-
|
654
|
+
self._eval.set("srcdir", os.path.dirname(taskdef.srcinfo.file))
|
655
|
+
|
656
|
+
passthrough, consumes, rundir = self._getPTConsumesRundir(taskdef, task.uses)
|
652
657
|
|
653
|
-
|
654
|
-
|
655
|
-
|
656
|
-
|
657
|
-
|
658
|
-
|
659
|
-
|
660
|
-
|
661
|
-
|
662
|
-
|
663
|
-
|
664
|
-
|
665
|
-
|
666
|
-
|
667
|
-
|
668
|
-
|
669
|
-
|
670
|
-
|
671
|
-
|
672
|
-
|
673
|
-
|
674
|
-
|
675
|
-
|
676
|
-
|
677
|
-
|
678
|
-
|
679
|
-
|
680
|
-
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
|
685
|
-
|
686
|
-
|
687
|
-
|
658
|
+
task.passthrough = passthrough
|
659
|
+
task.consumes = consumes
|
660
|
+
task.rundir = rundir
|
661
|
+
|
662
|
+
task.paramT = self._getParamT(
|
663
|
+
taskdef,
|
664
|
+
task.uses.paramT if task.uses is not None else None)
|
665
|
+
|
666
|
+
for need in taskdef.needs:
|
667
|
+
nt = None
|
668
|
+
|
669
|
+
need_name = None
|
670
|
+
if isinstance(need, str):
|
671
|
+
need_name = need
|
672
|
+
nt = self._findTask(need)
|
673
|
+
elif isinstance(need, TaskDef):
|
674
|
+
need_name = need.name
|
675
|
+
else:
|
676
|
+
raise Exception("Unknown need type %s" % str(type(need)))
|
677
|
+
|
678
|
+
if need_name.endswith(".needs"):
|
679
|
+
# Find the original task first
|
680
|
+
nt = self._findTask(need_name[:-len(".needs")])
|
681
|
+
if nt is None:
|
682
|
+
self.error("failed to find task %s" % need, taskdef.srcinfo)
|
683
|
+
raise Exception("Failed to find task %s" % need)
|
684
|
+
for nn in nt.needs:
|
685
|
+
task.needs.append(nn)
|
686
|
+
else:
|
687
|
+
nt = self._findTask(need_name)
|
688
|
+
|
689
|
+
if nt is None:
|
690
|
+
self.error("failed to find task %s" % need, taskdef.srcinfo)
|
691
|
+
raise Exception("Failed to find task %s" % need)
|
692
|
+
task.needs.append(nt)
|
693
|
+
|
694
|
+
if taskdef.strategy is not None:
|
695
|
+
self._log.debug("Task %s strategy: %s" % (task.name, str(taskdef.strategy)))
|
696
|
+
if taskdef.strategy.generate is not None:
|
697
|
+
shell = taskdef.strategy.generate.shell
|
698
|
+
if shell is None:
|
699
|
+
shell = "pytask"
|
700
|
+
task.strategy = Strategy(
|
701
|
+
generate=StrategyGenerate(
|
702
|
+
shell=shell,
|
703
|
+
run=taskdef.strategy.generate.run))
|
688
704
|
|
689
705
|
# task.strategy = taskdef.strategy
|
690
706
|
|
691
707
|
|
692
|
-
|
693
|
-
|
694
|
-
|
695
|
-
|
696
|
-
|
697
|
-
|
698
|
-
|
699
|
-
|
700
|
-
|
701
|
-
|
702
|
-
|
703
|
-
|
704
|
-
|
708
|
+
# Determine how to implement this task
|
709
|
+
if taskdef.body is not None and len(taskdef.body) > 0:
|
710
|
+
self._mkTaskBody(task, taskdef)
|
711
|
+
elif taskdef.run is not None:
|
712
|
+
task.run = self._eval.eval(taskdef.run)
|
713
|
+
if taskdef.shell is not None:
|
714
|
+
task.shell = taskdef.shell
|
715
|
+
elif taskdef.pytask is not None: # Deprecated case
|
716
|
+
task.run = taskdef.pytask
|
717
|
+
task.shell = "pytask"
|
718
|
+
elif task.uses is not None and isinstance(task.uses, Task) and task.uses.run is not None:
|
719
|
+
task.run = task.uses.run
|
720
|
+
task.shell = task.uses.shell
|
705
721
|
|
706
|
-
self._log.debug("<--
|
722
|
+
self._log.debug("<-- _elabTask %s" % task.name)
|
707
723
|
|
708
724
|
def _loadTypes(self, pkg, typedefs):
|
709
725
|
self._log.debug("--> _loadTypes")
|
@@ -712,25 +728,36 @@ class PackageLoader(object):
|
|
712
728
|
tt = Type(
|
713
729
|
name=self._getScopeFullname(td.name),
|
714
730
|
doc=td.doc,
|
715
|
-
srcinfo=td.srcinfo
|
731
|
+
srcinfo=td.srcinfo,
|
732
|
+
typedef=td)
|
716
733
|
pkg.type_m[tt.name] = tt
|
717
734
|
self._pkg_s[-1].addType(tt, td.name)
|
718
735
|
types.append((td, tt))
|
719
736
|
|
720
737
|
# Now, resolve 'uses' and build out
|
721
738
|
for td,tt in types:
|
722
|
-
|
723
|
-
|
724
|
-
if tt.uses is None:
|
725
|
-
raise Exception("Failed to find type %s" % td.uses)
|
726
|
-
tt.paramT = self._getParamT(
|
727
|
-
td,
|
728
|
-
tt.uses.paramT if tt.uses is not None else None,
|
729
|
-
typename=tt.name,
|
730
|
-
is_type=True)
|
739
|
+
self._elabType(tt)
|
740
|
+
|
731
741
|
self._log.debug("<-- _loadTypes")
|
732
742
|
pass
|
733
743
|
|
744
|
+
def _elabType(self, tt):
|
745
|
+
self._log.debug("--> _elabType %s" % tt.name)
|
746
|
+
td = tt.typedef
|
747
|
+
|
748
|
+
tt.typedef = None
|
749
|
+
if td.uses is not None:
|
750
|
+
tt.uses = self._findType(td.uses)
|
751
|
+
if tt.uses is None:
|
752
|
+
raise Exception("Failed to find type %s" % td.uses)
|
753
|
+
tt.paramT = self._getParamT(
|
754
|
+
td,
|
755
|
+
tt.uses.paramT if tt.uses is not None else None,
|
756
|
+
typename=tt.name,
|
757
|
+
is_type=True)
|
758
|
+
self._log.debug("<-- _elabType %s" % tt.name)
|
759
|
+
|
760
|
+
|
734
761
|
def _mkTaskBody(self, task, taskdef):
|
735
762
|
self._pkg_s[-1].push_scope(TaskScope(name=taskdef.name))
|
736
763
|
pkg = self.package_scope()
|
@@ -836,6 +863,11 @@ class PackageLoader(object):
|
|
836
863
|
|
837
864
|
if uses is None:
|
838
865
|
uses = self._findType(name)
|
866
|
+
if uses is not None and uses.typedef:
|
867
|
+
self._elabType(uses)
|
868
|
+
pass
|
869
|
+
elif uses.taskdef:
|
870
|
+
self._elabTask(uses)
|
839
871
|
|
840
872
|
self._log.debug("<-- _findTaskOrType %s (%s)" % (name, ("found" if uses is not None else "not found")))
|
841
873
|
return uses
|
dv_flow/mgr/param_ref_eval.py
CHANGED
dv_flow/mgr/shell_callable.py
CHANGED
dv_flow/mgr/std/flow.dv
CHANGED
@@ -85,6 +85,16 @@ package:
|
|
85
85
|
incdir:
|
86
86
|
type: bool
|
87
87
|
value: false
|
88
|
+
- name: SetFileType
|
89
|
+
consumes:
|
90
|
+
- type: std.FileSet
|
91
|
+
desc: Outputs all input filesets with the specified filetype
|
92
|
+
shell: pytask
|
93
|
+
run: dv_flow.mgr.std.set_file_type.SetFileType
|
94
|
+
with:
|
95
|
+
filetype:
|
96
|
+
type: str
|
97
|
+
value: ""
|
88
98
|
- name: IncDirs
|
89
99
|
shell: pytask
|
90
100
|
run: dv_flow.mgr.std.incdirs.IncDirs
|
@@ -0,0 +1,18 @@
|
|
1
|
+
from dv_flow.mgr import TaskDataResult, TaskDataInput, TaskRunCtxt
|
2
|
+
|
3
|
+
async def SetFileType(ctxt: TaskRunCtxt, inputs: TaskDataInput) -> TaskDataResult:
|
4
|
+
output = []
|
5
|
+
|
6
|
+
for inp in inputs.inputs:
|
7
|
+
if inp.type == "std.FileSet":
|
8
|
+
inp = inp.copy()
|
9
|
+
inp.filetype = inputs.params.filetype
|
10
|
+
inp.src = None
|
11
|
+
inp.seq = -1
|
12
|
+
output.append(inp)
|
13
|
+
|
14
|
+
return TaskDataResult(
|
15
|
+
changed=inputs.changed,
|
16
|
+
output=output,
|
17
|
+
status=0
|
18
|
+
)
|
dv_flow/mgr/task.py
CHANGED
@@ -2,7 +2,6 @@ import dataclasses as dc
|
|
2
2
|
from typing import Any, Callable, Dict, List, Tuple, Union
|
3
3
|
from .srcinfo import SrcInfo
|
4
4
|
from .task_def import TaskDef, RundirE, PassthroughE, ConsumesE
|
5
|
-
from .task_node_ctor import TaskNodeCtor
|
6
5
|
|
7
6
|
@dc.dataclass
|
8
7
|
class Need(object):
|
@@ -45,6 +44,7 @@ class Task(object):
|
|
45
44
|
run : str = None
|
46
45
|
shell : str = "bash"
|
47
46
|
srcinfo : SrcInfo = None
|
47
|
+
taskdef : 'TaskDef' = None
|
48
48
|
|
49
49
|
@property
|
50
50
|
def leafname(self):
|
@@ -36,7 +36,6 @@ from .task_def import RundirE
|
|
36
36
|
from .task_data import TaskMarker, TaskMarkerLoc, SeverityE
|
37
37
|
from .task_gen_ctxt import TaskGenCtxt, TaskGenInputData
|
38
38
|
from .task_node import TaskNode
|
39
|
-
from .task_node_ctor import TaskNodeCtor
|
40
39
|
from .task_node_compound import TaskNodeCompound
|
41
40
|
from .task_node_ctxt import TaskNodeCtxt
|
42
41
|
from .task_node_leaf import TaskNodeLeaf
|
@@ -76,7 +75,6 @@ class TaskGraphBuilder(object):
|
|
76
75
|
_type_m : Dict[str,Type] = dc.field(default_factory=dict)
|
77
76
|
_task_node_m : Dict['TaskSpec',TaskNode] = dc.field(default_factory=dict)
|
78
77
|
_type_node_m : Dict[str,Any] = dc.field(default_factory=dict)
|
79
|
-
_task_ctor_m : Dict[Task,TaskNodeCtor] = dc.field(default_factory=dict)
|
80
78
|
_override_m : Dict[str,str] = dc.field(default_factory=dict)
|
81
79
|
_ns_scope_s : List[TaskNamespaceScope] = dc.field(default_factory=list)
|
82
80
|
_compound_task_ctxt_s : List[CompoundTaskCtxt] = dc.field(default_factory=list)
|
@@ -574,7 +572,7 @@ class TaskGraphBuilder(object):
|
|
574
572
|
|
575
573
|
res = None
|
576
574
|
if task.strategy.generate is not None:
|
577
|
-
callable = ExecGenCallable(body=task.strategy.generate.run)
|
575
|
+
callable = ExecGenCallable(body=task.strategy.generate.run, srcdir=srcdir)
|
578
576
|
input = TaskGenInputData(params=params)
|
579
577
|
|
580
578
|
res = callable(ctxt, input)
|
@@ -702,7 +700,10 @@ class TaskGraphBuilder(object):
|
|
702
700
|
shell = task.shell if task.shell is not None else "shell"
|
703
701
|
if shell in self._shell_m.keys():
|
704
702
|
self._log.debug("Use shell implementation")
|
705
|
-
callable = self._shell_m[shell](
|
703
|
+
callable = self._shell_m[shell](
|
704
|
+
task.run,
|
705
|
+
os.path.dirname(task.srcinfo.file),
|
706
|
+
task.shell)
|
706
707
|
else:
|
707
708
|
raise Exception("Shell %s not found" % shell)
|
708
709
|
|
dv_flow/mgr/task_node_leaf.py
CHANGED
@@ -193,6 +193,10 @@ class TaskNodeLeaf(TaskNode):
|
|
193
193
|
getattr(out, "type", "<unknown>"),
|
194
194
|
getattr(out, "src", "<unknown>")))
|
195
195
|
output.append(out)
|
196
|
+
else:
|
197
|
+
self._log.debug("Skipping consumed type %s from %s" % (
|
198
|
+
getattr(out, "type", "<unknown>"),
|
199
|
+
getattr(out, "src", "<unknown>")))
|
196
200
|
else:
|
197
201
|
self._log.debug("non-passthrough: %s (only local outputs propagated) %s" % (
|
198
202
|
self.name,
|
@@ -204,9 +208,11 @@ class TaskNodeLeaf(TaskNode):
|
|
204
208
|
|
205
209
|
# Add our own output
|
206
210
|
local_out = self.result.output.copy()
|
211
|
+
self._log.debug("local_out of %s: %d" % (self.name, len(local_out)))
|
207
212
|
for i,out in enumerate(local_out):
|
208
213
|
out.src = self.name
|
209
214
|
out.seq = i
|
215
|
+
self._log.debug("Adding output of type %s" % out.type)
|
210
216
|
output.append(out)
|
211
217
|
|
212
218
|
self._log.debug("output dep_m: %s %s" % (self.name, str(dep_m)))
|
dv_flow/mgr/task_runner.py
CHANGED
@@ -123,7 +123,9 @@ class TaskSetRunner(TaskRunner):
|
|
123
123
|
for t in active_s:
|
124
124
|
while len(active_task_l) >= self.nproc and t not in done_task_s:
|
125
125
|
# Wait for at least one job to complete
|
126
|
-
done, pending = await asyncio.wait(
|
126
|
+
done, pending = await asyncio.wait(
|
127
|
+
[at[1] for at in active_task_l],
|
128
|
+
return_when=asyncio.FIRST_COMPLETED)
|
127
129
|
self._completeTasks(active_task_l, done_task_s, done, dst_memento)
|
128
130
|
|
129
131
|
if self.status == 0 and t not in done_task_s:
|
@@ -174,7 +176,9 @@ class TaskSetRunner(TaskRunner):
|
|
174
176
|
# Wait for them to all complete
|
175
177
|
while len(active_task_l):
|
176
178
|
# TODO: Shouldn't gather here -- reach to each completion
|
177
|
-
done, pending = await asyncio.wait(
|
179
|
+
done, pending = await asyncio.wait(
|
180
|
+
[at[1] for at in active_task_l],
|
181
|
+
return_when=asyncio.FIRST_COMPLETED)
|
178
182
|
self._completeTasks(active_task_l, done_task_s, done, dst_memento)
|
179
183
|
|
180
184
|
if self.status != 0:
|
dv_flow/mgr/type.py
CHANGED
{dv_flow_mgr-1.10.15961872557rc0.dist-info → dv_flow_mgr-1.11.16383233052rc0.dist-info}/RECORD
RENAMED
@@ -1,12 +1,12 @@
|
|
1
|
-
dv_flow/mgr/__init__.py,sha256=
|
1
|
+
dv_flow/mgr/__init__.py,sha256=B69GdAfXkRhts17J4aSKHnylAd1C3A1v-MQkFttNUAk,4385
|
2
2
|
dv_flow/mgr/__main__.py,sha256=yrNrvz-jYEovBGpY1E9aS1iZ9S_9MQv0lT-XUQSp0vc,4141
|
3
3
|
dv_flow/mgr/cond_def.py,sha256=2ZkzPusqVkN1fFMTvkDl9O_OJLPdD_cK3xzX9J75RMw,343
|
4
4
|
dv_flow/mgr/config.py,sha256=b2MVlVVNB0psk8x4bQRAYshkpNJrtyMtV1Ymhmx9AfM,137
|
5
5
|
dv_flow/mgr/config_def.py,sha256=N5nBc_2VPrx0pCDyIe-VPhQ7ItnbyJqrak3ZiPEAbtI,1040
|
6
6
|
dv_flow/mgr/data_callable.py,sha256=-GeMgBUdUEcdAkWsqvUYcpKqUd5BQsFgi6ARflnp2B8,687
|
7
7
|
dv_flow/mgr/eval_jq.py,sha256=bRsHtaN51qIOiZK1VJV52W0-vj5VH0nQ7XIFebZi5kI,1129
|
8
|
-
dv_flow/mgr/exec_callable.py,sha256=
|
9
|
-
dv_flow/mgr/exec_gen_callable.py,sha256=
|
8
|
+
dv_flow/mgr/exec_callable.py,sha256=rpo1N68ZZcvNcuHsH23dZCIMvZ1lylXeUO_RFfKduvc,3265
|
9
|
+
dv_flow/mgr/exec_gen_callable.py,sha256=tn14TGFlSnjpTi5q-E2pTa804zd1tEMsG-O-Yfz_g1w,3056
|
10
10
|
dv_flow/mgr/expr_eval.py,sha256=ojHo3U_QGqm0CWVSEGtFJRo3jrjiTKhy_B58yRNxN8g,4814
|
11
11
|
dv_flow/mgr/expr_parser.py,sha256=LMLe4RTbgcQVIpuZIi1vowHsKRgRmzKzomGvlJL8kJI,7110
|
12
12
|
dv_flow/mgr/ext_rgy.py,sha256=rf8lDXGbISBu4S7WuwX36b-sOiZ22d5dtjPAu2anogo,6307
|
@@ -21,22 +21,22 @@ dv_flow/mgr/out,sha256=d8GGBi3J43fhdLBlnsUbzBfRe0TD0QTP3nOTz54l2bI,200
|
|
21
21
|
dv_flow/mgr/package.py,sha256=oKzYQypF-to4OXr8BKwPz7Z8cf7AC6eF6CiYj1jCyTI,4585
|
22
22
|
dv_flow/mgr/package_def.py,sha256=h3_FIKQB0vqfcu1jRAZuB8W8Kozr62oaKNchxeNjpWk,5898
|
23
23
|
dv_flow/mgr/package_import_spec.py,sha256=aZMpnS9a5NFY76_pYXEuO3-Mkc_xFzy73fdrUe_54Dc,1760
|
24
|
-
dv_flow/mgr/package_loader.py,sha256=
|
24
|
+
dv_flow/mgr/package_loader.py,sha256=0vUMiA0LBRsEanVVYE1wq5W44z3sLLi7ModdoKDKmfE,36976
|
25
25
|
dv_flow/mgr/package_node.py,sha256=CccD2ECiIXy9JBkRR3c7qDxeBiLiPbRT9CqR-Al_niI,214
|
26
26
|
dv_flow/mgr/param.py,sha256=kkxMRGf6mPjSZJsjgLKH2vJL62Sn0ZESvjBLkEYOp20,1386
|
27
27
|
dv_flow/mgr/param_def.py,sha256=GnaolV5QOrjBpqNj9yYo9nnrwEBpp4J_4ZYe14bsfO4,1939
|
28
|
-
dv_flow/mgr/param_ref_eval.py,sha256=
|
28
|
+
dv_flow/mgr/param_ref_eval.py,sha256=n39PIQ_Ht-7lO8oYLHLfMek90mE6nqJpXck5HmWSGsQ,2176
|
29
29
|
dv_flow/mgr/parser.out,sha256=mWx6hgqORUFhM2OlxV2gp43Y7w3sX2u4ua4t-5wqmck,27566
|
30
30
|
dv_flow/mgr/parsetab.py,sha256=S2o2FfobhhVWKwZg_GA7zFg62VwT7tiRxxNZJIyqP98,4351
|
31
31
|
dv_flow/mgr/pytask_callable.py,sha256=JiFN2vq6qanI6FXJBeidwiBSn6t0oC0izPJ01c5QJTE,688
|
32
32
|
dv_flow/mgr/root_package.py,sha256=nYLOpZVQEDbY0Ui5X5UNeUHXOvqRENew7B2zpC2V2gQ,136
|
33
|
-
dv_flow/mgr/shell_callable.py,sha256
|
33
|
+
dv_flow/mgr/shell_callable.py,sha256=-3UchnpYVohqOwxs-d3Vn2xBXQQnOO64oLIxrELd-3M,1068
|
34
34
|
dv_flow/mgr/srcinfo.py,sha256=xrfp-relVr7hYNbxOjY5jqX4H0nNMm9HK5AAM6_Piyk,366
|
35
|
-
dv_flow/mgr/task.py,sha256=
|
35
|
+
dv_flow/mgr/task.py,sha256=oEaMrviXccz8IFSAHUtq2TqgHzelBga4jfq5vFnSGbM,2377
|
36
36
|
dv_flow/mgr/task_data.py,sha256=lN7Iq8YTitEMGG4rZqYQi6Ri2HuPgBQ5oGQbW-63T8c,12436
|
37
37
|
dv_flow/mgr/task_def.py,sha256=lhiOL7E_TfW7Ytg5qipE6d2cqTkfVqAOs1MZgtCqeLg,5257
|
38
38
|
dv_flow/mgr/task_gen_ctxt.py,sha256=1QQqqgaCjJSlzTV0SASpTb69vINFrDq3aqjYCKUkx3s,846
|
39
|
-
dv_flow/mgr/task_graph_builder.py,sha256=
|
39
|
+
dv_flow/mgr/task_graph_builder.py,sha256=yFdL9zGek5F8NEVFgYqL-XLZFaznusmNwIJL9FUCMkk,33718
|
40
40
|
dv_flow/mgr/task_graph_dot_writer.py,sha256=qK4Imy9o2_F1aKoU1tJ-qoBHslq2BhSMbdjAUPfpN7I,6009
|
41
41
|
dv_flow/mgr/task_listener_log.py,sha256=XcmBT-946DyrEYwNwbg7ejnB6Q3koybE7AekSWGxNGQ,4774
|
42
42
|
dv_flow/mgr/task_listener_trace.py,sha256=7B-V2YK8uXd_QSGBq_7USmbe5CleVvwumU0CpsmxZvg,6224
|
@@ -50,12 +50,12 @@ dv_flow/mgr/task_node_ctor_proxy.py,sha256=ViOFJ64JM4-CGFZNl89BghFuKSQ66kZVqSj4v
|
|
50
50
|
dv_flow/mgr/task_node_ctor_task.py,sha256=d49g90TyPCMFR8BuWWqp4ym-MW5vGSdDR0V47Ru28JY,2232
|
51
51
|
dv_flow/mgr/task_node_ctor_wrapper.py,sha256=KMSYd4ik2Si7UXhNMeXAPHBwwrFkuhpeTMeYRpWv0hw,3718
|
52
52
|
dv_flow/mgr/task_node_ctxt.py,sha256=KHt_kQch6LDq70Oor7z2xbepiiaK2jyp4S-KWZGsRjo,205
|
53
|
-
dv_flow/mgr/task_node_leaf.py,sha256=
|
53
|
+
dv_flow/mgr/task_node_leaf.py,sha256=98nVAFGuv5ZJPoc1CNUEeOc-aV69xV8Wd1hEP9QLw3Y,9517
|
54
54
|
dv_flow/mgr/task_output.py,sha256=ZwyvwnYj_gHOEFAEOH3m24Xfc4Cn77hb1j7LkX8_3C4,1086
|
55
55
|
dv_flow/mgr/task_params_ctor.py,sha256=qlrzibGAFHmbqOu88jEoh1wOSFHu68Gwdgc259-50e8,1915
|
56
56
|
dv_flow/mgr/task_run_ctxt.py,sha256=1c5dV0xd1XQOTkcqv1qBrXs-gs--nZ1AvLzFb6L5dNY,3967
|
57
|
-
dv_flow/mgr/task_runner.py,sha256=
|
58
|
-
dv_flow/mgr/type.py,sha256=
|
57
|
+
dv_flow/mgr/task_runner.py,sha256=D5XKikcR65Rk4TqMdg4URkUOzRzMOLJwRNoyBjIFdCQ,10803
|
58
|
+
dv_flow/mgr/type.py,sha256=TYDgvaNNi3KrM8k9Y_QmmTboYuOFiuIZTzkWqrt_VTY,754
|
59
59
|
dv_flow/mgr/type_def.py,sha256=4sge3PibO1jDnS0cXdX0PiurcKbDA3kT6rb4DGIKwEM,1176
|
60
60
|
dv_flow/mgr/yaml_srcinfo_loader.py,sha256=29BNRiB8Hj1FepkrLtdjHSv5U_85Q432gBeeK80nKEA,1606
|
61
61
|
dv_flow/mgr/cmds/cmd_graph.py,sha256=VQLm-zoa5xbWNfJNoCnDihKniWW8W_tmCyWZISDzwDo,3006
|
@@ -68,18 +68,19 @@ dv_flow/mgr/std/create_file.py,sha256=SEpKTQdiY32002C7b4kYfAiK9v_xajixOJU5WftW75
|
|
68
68
|
dv_flow/mgr/std/data_item.py,sha256=olKtspEVFOvr92reUl4QaQV27UVyjSGIfIWpsy0Myvo,434
|
69
69
|
dv_flow/mgr/std/exec.py,sha256=UChqa_tAjvdB1NjqTsvlPgFomP8hMsX2rAOPyUonNpk,3896
|
70
70
|
dv_flow/mgr/std/fileset.py,sha256=_9IUjZ2_doygc29Qg1hJPaFPzlaIaTdvHnYTai8mE3A,4183
|
71
|
-
dv_flow/mgr/std/flow.dv,sha256=
|
71
|
+
dv_flow/mgr/std/flow.dv,sha256=iTVpI98PBbn3oUYyQJ6yb-S7zVReM0ZL2WmodH-hZzs,4043
|
72
72
|
dv_flow/mgr/std/incdirs.py,sha256=F0wraOxkRJRkrhsyk9v2YvpquFtcK4ZsMO1G9sgmhmA,588
|
73
73
|
dv_flow/mgr/std/message.py,sha256=0JHLErg8whqMLAasG1fumZ2O7R7WNWeNQ9ibJaLDpVY,1029
|
74
|
+
dv_flow/mgr/std/set_file_type.py,sha256=t7NtTmWW3xVckoiHn3hhL2LGAPihP07n_94_4kc8_vw,504
|
74
75
|
dv_flow/mgr/std/task_null.py,sha256=dw6LXBXVwth6gLPeduDvlz5znAhcVpDH8r1DticD-0w,1041
|
75
76
|
dv_flow/mgr/util/__init__.py,sha256=6uuA6z5cKS2hcjJw6YyEM2M79g6OpXb6tZF_Gku-AGU,22
|
76
77
|
dv_flow/mgr/util/__main__.py,sha256=F0LXpCDpYTPalSo0dc1h_qZkip5v1AZYYh-vcYbh5s0,983
|
77
78
|
dv_flow/mgr/util/util.py,sha256=5wVWpkHn4U-Hcv-skaLaGC0bgoK63NaF-NH-DGpAaRM,2035
|
78
79
|
dv_flow/mgr/util/cmds/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
79
80
|
dv_flow/mgr/util/cmds/cmd_schema.py,sha256=IJzZdxCSEgIQ79LpYiM7UqJ9RJ-7yraqmBN2XVgAgXA,1752
|
80
|
-
dv_flow_mgr-1.
|
81
|
-
dv_flow_mgr-1.
|
82
|
-
dv_flow_mgr-1.
|
83
|
-
dv_flow_mgr-1.
|
84
|
-
dv_flow_mgr-1.
|
85
|
-
dv_flow_mgr-1.
|
81
|
+
dv_flow_mgr-1.11.16383233052rc0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
82
|
+
dv_flow_mgr-1.11.16383233052rc0.dist-info/METADATA,sha256=4rMZ91qdQoV84kNZh5RkW95O9ALRwiWZC3ed-Z4StKs,13336
|
83
|
+
dv_flow_mgr-1.11.16383233052rc0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
84
|
+
dv_flow_mgr-1.11.16383233052rc0.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
|
85
|
+
dv_flow_mgr-1.11.16383233052rc0.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
|
86
|
+
dv_flow_mgr-1.11.16383233052rc0.dist-info/RECORD,,
|
{dv_flow_mgr-1.10.15961872557rc0.dist-info → dv_flow_mgr-1.11.16383233052rc0.dist-info}/WHEEL
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|