dv-flow-mgr 0.0.1.13885203883a1__py3-none-any.whl → 0.0.1.13954441689a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dv_flow/mgr/pkg_rgy.py +4 -3
- dv_flow/mgr/std/exec.py +86 -7
- dv_flow/mgr/std/flow.dv +22 -1
- dv_flow/mgr/task_def.py +1 -0
- dv_flow/mgr/task_node.py +12 -0
- {dv_flow_mgr-0.0.1.13885203883a1.dist-info → dv_flow_mgr-0.0.1.13954441689a1.dist-info}/METADATA +1 -1
- {dv_flow_mgr-0.0.1.13885203883a1.dist-info → dv_flow_mgr-0.0.1.13954441689a1.dist-info}/RECORD +11 -16
- {dv_flow_mgr-0.0.1.13885203883a1.dist-info → dv_flow_mgr-0.0.1.13954441689a1.dist-info}/WHEEL +1 -1
- dv_flow/mgr/task_exec_data.py +0 -43
- dv_flow/mgr/task_graph_runner.py +0 -70
- dv_flow/mgr/task_graph_runner_local.py +0 -141
- dv_flow/mgr/task_impl_data.py +0 -37
- dv_flow/mgr/task_memento.py +0 -34
- {dv_flow_mgr-0.0.1.13885203883a1.dist-info → dv_flow_mgr-0.0.1.13954441689a1.dist-info}/LICENSE +0 -0
- {dv_flow_mgr-0.0.1.13885203883a1.dist-info → dv_flow_mgr-0.0.1.13954441689a1.dist-info}/entry_points.txt +0 -0
- {dv_flow_mgr-0.0.1.13885203883a1.dist-info → dv_flow_mgr-0.0.1.13954441689a1.dist-info}/top_level.txt +0 -0
dv_flow/mgr/pkg_rgy.py
CHANGED
@@ -120,10 +120,11 @@ class PkgRgy(object):
|
|
120
120
|
|
121
121
|
for name,path in pkg_m.items():
|
122
122
|
self._log.debug("Registering package %s: %s" % (name, path))
|
123
|
-
if name in self._pkg_m.keys():
|
124
|
-
|
123
|
+
if name in self._pkg_m.keys() and self._pkg_m[name][0] != path:
|
124
|
+
self._log.debug("Package %s already registered using path %s. Conflicting path: %s" % (
|
125
125
|
name, self._pkg_m[name][0], path))
|
126
|
-
|
126
|
+
else:
|
127
|
+
self._pkg_m[name] = (path, None)
|
127
128
|
except Exception as e:
|
128
129
|
self._log.critical("Error loading plugin %s: %s" % (p.name, str(e)))
|
129
130
|
raise e
|
dv_flow/mgr/std/exec.py
CHANGED
@@ -20,21 +20,100 @@
|
|
20
20
|
#*
|
21
21
|
#****************************************************************************
|
22
22
|
import asyncio
|
23
|
+
import os
|
23
24
|
import logging
|
24
|
-
from
|
25
|
+
from pydantic import BaseModel
|
26
|
+
from dv_flow.mgr import TaskDataResult, TaskMarker
|
25
27
|
|
26
28
|
_log = logging.getLogger("Exec")
|
27
29
|
|
30
|
+
class Memento(BaseModel):
|
31
|
+
timestamp_file : str
|
32
|
+
timestamp : float
|
33
|
+
|
28
34
|
async def Exec(runner, input) -> TaskDataResult:
|
29
35
|
_log.debug("TaskExec run: %s: cmd=%s" % (input.name, input.params.command))
|
30
36
|
|
37
|
+
# Running the command is gated by inputs to the task
|
38
|
+
need_to_run = input.params.when == "always" or input.changed
|
39
|
+
|
40
|
+
status = 0
|
41
|
+
markers = []
|
42
|
+
|
43
|
+
if need_to_run:
|
44
|
+
fp = open(os.path.join(input.rundir, "run.log"), "w")
|
45
|
+
fp.write("Command: %s\n" % input.params.command)
|
46
|
+
proc = await asyncio.create_subprocess_shell(
|
47
|
+
input.params.command,
|
48
|
+
stdout=fp,
|
49
|
+
stderr=asyncio.subprocess.STDOUT)
|
50
|
+
|
51
|
+
result = await proc.wait()
|
52
|
+
|
53
|
+
status = result.returncode
|
54
|
+
|
55
|
+
if status != 0:
|
56
|
+
markers.append(TaskMarker(
|
57
|
+
message="Command \"%s\" failed with status %d" % (
|
58
|
+
input.params.command, status), severity="error"))
|
59
|
+
|
60
|
+
changed = False
|
61
|
+
ex_memento = input.memento
|
62
|
+
memento = None
|
63
|
+
|
64
|
+
if ex_memento is not None and status == 0:
|
65
|
+
ex_memento_o = None
|
66
|
+
try:
|
67
|
+
ex_memento_o = Memento(**ex_memento)
|
68
|
+
except Exception as e:
|
69
|
+
_log.debug("Failed to deserialize memento: %s" % str(e))
|
70
|
+
|
71
|
+
if ex_memento_o is not None:
|
72
|
+
if input.params.timestamp is not None and input.params.timestamp != "":
|
73
|
+
# Have a file to use
|
74
|
+
if os.path.isfile(os.path.join(input.rundir, input.params.timestamp)):
|
75
|
+
ts_file = os.path.join(input.rundir, input.params.timestamp)
|
76
|
+
ts = os.path.getmtime(ts_file)
|
77
|
+
|
78
|
+
if ts_file != ex_memento_o.timestamp_file or ts != ex_memento_o.timestamp:
|
79
|
+
changed = True
|
80
|
+
|
81
|
+
memento = Memento(timestamp_file=ts_file, timestamp=ts)
|
82
|
+
else:
|
83
|
+
markers.append(TaskMarker(
|
84
|
+
message="Timestamp file %s not found" % (
|
85
|
+
os.path.join(input.rundir, input.params.timestamp)),
|
86
|
+
severity="error"))
|
87
|
+
status = 1
|
88
|
+
else:
|
89
|
+
memento = Memento()
|
90
|
+
else:
|
91
|
+
changed = True
|
92
|
+
memento = ex_memento
|
93
|
+
else:
|
94
|
+
# No existing memento, or the task failed
|
95
|
+
changed = (status == 0)
|
96
|
+
|
97
|
+
if input.params.when == "changed":
|
98
|
+
changed = input.changed
|
99
|
+
|
100
|
+
if input.params.timestamp is not None and input.params.timestamp_file != "":
|
101
|
+
# Check the timestamp file
|
102
|
+
try:
|
103
|
+
with open(input.params.timestamp_file, "r") as f:
|
104
|
+
timestamp = float(f.read())
|
105
|
+
except FileNotFoundError:
|
106
|
+
timestamp = 0.0
|
107
|
+
|
108
|
+
if timestamp < input.params.timestamp:
|
109
|
+
changed = True
|
31
110
|
|
32
|
-
proc = await asyncio.create_subprocess_shell(
|
33
|
-
input.params.command,
|
34
|
-
stdout=asyncio.subprocess.PIPE,
|
35
|
-
stderr=asyncio.subprocess.PIPE)
|
36
111
|
|
37
|
-
await proc.wait()
|
38
112
|
|
39
|
-
return TaskDataResult(
|
113
|
+
return TaskDataResult(
|
114
|
+
status=status,
|
115
|
+
changed=changed,
|
116
|
+
markers=markers,
|
117
|
+
memento=memento)
|
118
|
+
|
40
119
|
|
dv_flow/mgr/std/flow.dv
CHANGED
@@ -47,10 +47,31 @@ package:
|
|
47
47
|
type: str
|
48
48
|
- name: Exec
|
49
49
|
pytask: dv_flow.mgr.std.exec.Exec
|
50
|
-
|
50
|
+
desc: Executes a subprocess
|
51
|
+
with:
|
52
|
+
shell:
|
53
|
+
type: str
|
54
|
+
value: "bash"
|
55
|
+
doc: |
|
56
|
+
Shell to use for executing the command
|
51
57
|
command:
|
52
58
|
type: str
|
53
59
|
value: ""
|
60
|
+
doc: |
|
61
|
+
Command to execute
|
62
|
+
when:
|
63
|
+
type: str
|
64
|
+
value: "always"
|
65
|
+
doc: |
|
66
|
+
Specifies when the command is run.
|
67
|
+
- always -- Command is always run (default)
|
68
|
+
- changed -- If upstream tasks change
|
69
|
+
timestamp:
|
70
|
+
type: str
|
71
|
+
value: ""
|
72
|
+
doc: |
|
73
|
+
Optional timestamp file to determine if running
|
74
|
+
the command changed the output
|
54
75
|
types:
|
55
76
|
# - name: TaskDataItem
|
56
77
|
# doc: |
|
dv_flow/mgr/task_def.py
CHANGED
@@ -48,6 +48,7 @@ class TaskDef(BaseModel):
|
|
48
48
|
params: Dict[str,Union[str,list,ParamDef]] = dc.Field(default_factory=dict, alias="with")
|
49
49
|
passthrough: bool = dc.Field(default=False)
|
50
50
|
consumes : List[Any] = dc.Field(default_factory=list)
|
51
|
+
tasks: List['TaskDef'] = dc.Field(default_factory=list)
|
51
52
|
|
52
53
|
# out: List[TaskOutput] = dc.Field(default_factory=list)
|
53
54
|
|
dv_flow/mgr/task_node.py
CHANGED
@@ -19,6 +19,7 @@
|
|
19
19
|
#* Author:
|
20
20
|
#*
|
21
21
|
#****************************************************************************
|
22
|
+
import enum
|
22
23
|
import os
|
23
24
|
import sys
|
24
25
|
import dataclasses as dc
|
@@ -31,6 +32,10 @@ from .task_params_ctor import TaskParamsCtor
|
|
31
32
|
from .param_ref_eval import ParamRefEval
|
32
33
|
from .param import Param
|
33
34
|
|
35
|
+
class RundirE(enum.Enum):
|
36
|
+
Unique = enum.auto()
|
37
|
+
Inherit = enum.auto()
|
38
|
+
|
34
39
|
@dc.dataclass
|
35
40
|
class TaskNode(object):
|
36
41
|
"""Executable view of a task"""
|
@@ -48,6 +53,7 @@ class TaskNode(object):
|
|
48
53
|
consumes : List[Any] = dc.field(default_factory=list)
|
49
54
|
needs : List[Tuple['TaskNode',bool]] = dc.field(default_factory=list)
|
50
55
|
rundir : str = dc.field(default=None)
|
56
|
+
rundir_t : RundirE = dc.field(default=RundirE.Unique)
|
51
57
|
output : TaskDataOutput = dc.field(default=None)
|
52
58
|
result : TaskDataResult = dc.field(default=None)
|
53
59
|
start : float = dc.field(default=None)
|
@@ -118,6 +124,12 @@ class TaskNode(object):
|
|
118
124
|
|
119
125
|
self._log.debug("in_params[2]: %s" % ",".join(p.src for p in in_params))
|
120
126
|
eval.setVar("in", in_params)
|
127
|
+
eval.setVar("rundir", rundir)
|
128
|
+
|
129
|
+
# Set variables from the inputs
|
130
|
+
for need in self.needs:
|
131
|
+
for name,value in {"rundir" : need.rundir}.items():
|
132
|
+
eval.setVar("%s.%s" % (need.name, name), value)
|
121
133
|
|
122
134
|
# Default inputs is the list of parameter sets that match 'consumes'
|
123
135
|
inputs = []
|
{dv_flow_mgr-0.0.1.13885203883a1.dist-info → dv_flow_mgr-0.0.1.13954441689a1.dist-info}/RECORD
RENAMED
@@ -13,19 +13,14 @@ dv_flow/mgr/param.py,sha256=kkxMRGf6mPjSZJsjgLKH2vJL62Sn0ZESvjBLkEYOp20,1386
|
|
13
13
|
dv_flow/mgr/param_def.py,sha256=ObRf289sI1xzOudjO9MzyTVuKsz3AS949XA4Deyz-kE,1549
|
14
14
|
dv_flow/mgr/param_ref_eval.py,sha256=5yH37oIX6f2qmk7GfRgNT5qZx0jm3CJFgB9lLDZZ1yQ,1981
|
15
15
|
dv_flow/mgr/parsetab.py,sha256=I-p3nC60t9jiNtPhKyl_sE92SiP96zJLnNdydcLy33g,3780
|
16
|
-
dv_flow/mgr/pkg_rgy.py,sha256=
|
16
|
+
dv_flow/mgr/pkg_rgy.py,sha256=d1nIjRm3ymMNJT-yiMDxCS6bFisTPvLMqh5VrfsHVKM,5404
|
17
17
|
dv_flow/mgr/task.py,sha256=BmS0wd51l4tpTrsit3-ugCB5VGnwpMJNl0dfJfHS4DQ,5981
|
18
18
|
dv_flow/mgr/task_ctor.py,sha256=YDYb5uW9Js7MnA8GqbYJljNCbeJFKGveZ2v-IJRgIa8,2221
|
19
19
|
dv_flow/mgr/task_data.py,sha256=ysawiUcbQu-ImNSgU5LsfOWlBKdfBt_8juwSacnAqAQ,11437
|
20
|
-
dv_flow/mgr/task_def.py,sha256=
|
21
|
-
dv_flow/mgr/task_exec_data.py,sha256=-mh9G2-iq6ioB7v7QrHiQAOEwd_hDKhnVf2Mb_IHRtg,1483
|
20
|
+
dv_flow/mgr/task_def.py,sha256=0rVlDjNF39PQ_0uzkSuOgvE2sZjFwpTAR8HIu_Arbog,2140
|
22
21
|
dv_flow/mgr/task_graph_builder.py,sha256=EYFsMZCDDHMEFWZOVSpPb7KBVE6N6XyL4KFzJVrPSPE,10106
|
23
|
-
dv_flow/mgr/task_graph_runner.py,sha256=LZ5cTXLQlcK348AxTYKqfpsFFmxA3aseGfholvQW39s,2187
|
24
|
-
dv_flow/mgr/task_graph_runner_local.py,sha256=quAxmF1go2iVhBDieksg3brAZukIvpvF-8oJ_YXiauE,4692
|
25
|
-
dv_flow/mgr/task_impl_data.py,sha256=jpO7kQ6AjYcOOfpgFIPCEYA1pQbVwFcIY_SY3Q53eGk,1163
|
26
22
|
dv_flow/mgr/task_listener_log.py,sha256=b7caUHERGkN6eT8IinBJTdZpwASUs8xAqDmtEymMKzQ,3529
|
27
|
-
dv_flow/mgr/
|
28
|
-
dv_flow/mgr/task_node.py,sha256=xPCarkVSj9rZ1v12WiHtMOe-nMKHbkaUpP9-edQOXek,14828
|
23
|
+
dv_flow/mgr/task_node.py,sha256=DL7gI79uu0yoaz2J7PJhr91isL42agATbLDO0ngGqi4,15215
|
29
24
|
dv_flow/mgr/task_output.py,sha256=ZwyvwnYj_gHOEFAEOH3m24Xfc4Cn77hb1j7LkX8_3C4,1086
|
30
25
|
dv_flow/mgr/task_params_ctor.py,sha256=BPkbnoCtzhCxc1g8CJ6VimCcm5UAu92PXeDMhQ4lYsQ,1957
|
31
26
|
dv_flow/mgr/task_runner.py,sha256=TB4feCEItxufUyyBSwNNY3OkQ0yjSO0fUfjga0u7rsg,8790
|
@@ -35,14 +30,14 @@ dv_flow/mgr/util.py,sha256=BLp-25Ph9roqo1O3VHI5sKhwxQ0wUU7znFwcSKt0hpA,1442
|
|
35
30
|
dv_flow/mgr/cmds/cmd_run.py,sha256=lNMvpiyKf-Ehs0GUkzcOIoJxaXgeLKv6UFNakcm9_hw,3303
|
36
31
|
dv_flow/mgr/share/flow.json,sha256=lNmZex9NXkYbyb2aZseQfUOkV9CMyfH0iLODEI7EPBw,5096
|
37
32
|
dv_flow/mgr/std/create_file.py,sha256=wmn5N_mObx_wr2LPdKOVbNmdM71hhs3UXOv6Ap9l3Ts,2726
|
38
|
-
dv_flow/mgr/std/exec.py,sha256=
|
33
|
+
dv_flow/mgr/std/exec.py,sha256=KCPUZ0yrhsf42SxbQ7lt6XMQaSsAVw4mImchn0IbqGg,3903
|
39
34
|
dv_flow/mgr/std/fileset.py,sha256=5IxS6T-x0wzA6fdEQcfHZ9kNP8IpH9hfJ3UMhL4A6Iw,3911
|
40
|
-
dv_flow/mgr/std/flow.dv,sha256=
|
35
|
+
dv_flow/mgr/std/flow.dv,sha256=Ehw0rrFcHoXt3WudeJ5PyPXpVtUcvvWGm43ikUWl-lE,2602
|
41
36
|
dv_flow/mgr/std/message.py,sha256=_Gox9oBzL9ZYG1JfJ-WYPXojVLqGNaKxZ9tpLVT0LO0,1035
|
42
37
|
dv_flow/mgr/std/task_null.py,sha256=dw6LXBXVwth6gLPeduDvlz5znAhcVpDH8r1DticD-0w,1041
|
43
|
-
dv_flow_mgr-0.0.1.
|
44
|
-
dv_flow_mgr-0.0.1.
|
45
|
-
dv_flow_mgr-0.0.1.
|
46
|
-
dv_flow_mgr-0.0.1.
|
47
|
-
dv_flow_mgr-0.0.1.
|
48
|
-
dv_flow_mgr-0.0.1.
|
38
|
+
dv_flow_mgr-0.0.1.13954441689a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
39
|
+
dv_flow_mgr-0.0.1.13954441689a1.dist-info/METADATA,sha256=IE3eu7gxZfXq6hgYc8aTS1VccbTV6t4JNmdfj6fvLn8,13314
|
40
|
+
dv_flow_mgr-0.0.1.13954441689a1.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
|
41
|
+
dv_flow_mgr-0.0.1.13954441689a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
|
42
|
+
dv_flow_mgr-0.0.1.13954441689a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
|
43
|
+
dv_flow_mgr-0.0.1.13954441689a1.dist-info/RECORD,,
|
dv_flow/mgr/task_exec_data.py
DELETED
@@ -1,43 +0,0 @@
|
|
1
|
-
#****************************************************************************
|
2
|
-
#* task_exec_data.py
|
3
|
-
#*
|
4
|
-
#* Copyright 2023-2025 Matthew Ballance and Contributors
|
5
|
-
#*
|
6
|
-
#* Licensed under the Apache License, Version 2.0 (the "License"); you may
|
7
|
-
#* not use this file except in compliance with the License.
|
8
|
-
#* You may obtain a copy of the License at:
|
9
|
-
#*
|
10
|
-
#* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
-
#*
|
12
|
-
#* Unless required by applicable law or agreed to in writing, software
|
13
|
-
#* distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
-
#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
-
#* See the License for the specific language governing permissions and
|
16
|
-
#* limitations under the License.
|
17
|
-
#*
|
18
|
-
#* Created on:
|
19
|
-
#* Author:
|
20
|
-
#*
|
21
|
-
#****************************************************************************
|
22
|
-
import pydantic.dataclasses as dc
|
23
|
-
from pydantic import BaseModel
|
24
|
-
from typing import Any, Dict, List
|
25
|
-
|
26
|
-
|
27
|
-
class TaskExecData(BaseModel):
|
28
|
-
"""Data from a single exection of a task"""
|
29
|
-
name : str
|
30
|
-
start : str
|
31
|
-
finish : str
|
32
|
-
status : int
|
33
|
-
memento : Any
|
34
|
-
markers : List[Any]
|
35
|
-
|
36
|
-
class FlowExecData(BaseModel):
|
37
|
-
"""
|
38
|
-
Data from multiple tasks executions. 'info' holds information
|
39
|
-
across multiple flow invocations. 'tasks' holds the names of
|
40
|
-
tasks executed in the most-recent invocation.
|
41
|
-
"""
|
42
|
-
info : Dict[str, TaskExecData] = dc.Field(default_factory=dict)
|
43
|
-
tasks : List[str] = dc.Field(default_factory=list)
|
dv_flow/mgr/task_graph_runner.py
DELETED
@@ -1,70 +0,0 @@
|
|
1
|
-
#****************************************************************************
|
2
|
-
#* task_graph_runner.py
|
3
|
-
#*
|
4
|
-
#* Copyright 2023-2025 Matthew Ballance and Contributors
|
5
|
-
#*
|
6
|
-
#* Licensed under the Apache License, Version 2.0 (the "License"); you may
|
7
|
-
#* not use this file except in compliance with the License.
|
8
|
-
#* You may obtain a copy of the License at:
|
9
|
-
#*
|
10
|
-
#* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
-
#*
|
12
|
-
#* Unless required by applicable law or agreed to in writing, software
|
13
|
-
#* distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
-
#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
-
#* See the License for the specific language governing permissions and
|
16
|
-
#* limitations under the License.
|
17
|
-
#*
|
18
|
-
#* Created on:
|
19
|
-
#* Author:
|
20
|
-
#*
|
21
|
-
#****************************************************************************
|
22
|
-
import asyncio
|
23
|
-
import os
|
24
|
-
import dataclasses as dc
|
25
|
-
from typing import Any, Callable, ClassVar, Dict, List, Tuple
|
26
|
-
from .task import Task
|
27
|
-
from .task_data import TaskData
|
28
|
-
from .task_runner import TaskRunner
|
29
|
-
|
30
|
-
@dc.dataclass
|
31
|
-
class TaskGraphRunner(TaskRunner):
|
32
|
-
"""Session manages execution of a task graph"""
|
33
|
-
|
34
|
-
_inst : ClassVar['TaskGraphRunner'] = None
|
35
|
-
|
36
|
-
|
37
|
-
# Search path for .dfs files
|
38
|
-
create_subprocess : Callable = asyncio.create_subprocess_exec
|
39
|
-
_root_dir : str = None
|
40
|
-
|
41
|
-
async def exec(self, *args, **kwargs):
|
42
|
-
return self.create_subprocess(*args, **kwargs)
|
43
|
-
|
44
|
-
# def load(self):
|
45
|
-
# if not os.path.isdir(self.srcdir):
|
46
|
-
# raise Exception("Root directory %s does not exist" % self.srcdir)
|
47
|
-
|
48
|
-
# if not os.path.isfile(os.path.join(self.srcdir, "flow.dv")):
|
49
|
-
# raise Exception("No root flow file")
|
50
|
-
|
51
|
-
# self._root_dir = os.path.dirname(self.srcdir)
|
52
|
-
# self.package = PackageDef.load(os.path.join(self.srcdir, "flow.dv"), [])
|
53
|
-
|
54
|
-
# return self.package
|
55
|
-
|
56
|
-
|
57
|
-
async def run(self, task : str) -> 'TaskData':
|
58
|
-
impl = self.mkTaskGraph(task)
|
59
|
-
return await impl.do_run()
|
60
|
-
|
61
|
-
async def runTask(self, task : Task) -> 'TaskData':
|
62
|
-
return await task.do_run()
|
63
|
-
|
64
|
-
def queueTask(self, task : Task):
|
65
|
-
"""Queue a task for execution"""
|
66
|
-
pass
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
@@ -1,141 +0,0 @@
|
|
1
|
-
#****************************************************************************
|
2
|
-
#* task_graph_runner_local.py
|
3
|
-
#*
|
4
|
-
#* Copyright 2023-2025 Matthew Ballance and Contributors
|
5
|
-
#*
|
6
|
-
#* Licensed under the Apache License, Version 2.0 (the "License"); you may
|
7
|
-
#* not use this file except in compliance with the License.
|
8
|
-
#* You may obtain a copy of the License at:
|
9
|
-
#*
|
10
|
-
#* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
-
#*
|
12
|
-
#* Unless required by applicable law or agreed to in writing, software
|
13
|
-
#* distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
-
#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
-
#* See the License for the specific language governing permissions and
|
16
|
-
#* limitations under the License.
|
17
|
-
#*
|
18
|
-
#* Created on:
|
19
|
-
#* Author:
|
20
|
-
#*
|
21
|
-
#****************************************************************************
|
22
|
-
import asyncio
|
23
|
-
import os
|
24
|
-
import yaml
|
25
|
-
import dataclasses as dc
|
26
|
-
import logging
|
27
|
-
from toposort import toposort
|
28
|
-
from typing import Any, Callable, ClassVar, Coroutine, Dict, List, Tuple, Union
|
29
|
-
from .fragment_def import FragmentDef
|
30
|
-
from .package import Package
|
31
|
-
from .pkg_rgy import PkgRgy
|
32
|
-
from .package_def import PackageDef, PackageSpec
|
33
|
-
from .task import Task
|
34
|
-
from .task_data import TaskData
|
35
|
-
from .task_graph_runner import TaskGraphRunner
|
36
|
-
|
37
|
-
@dc.dataclass
|
38
|
-
class TaskGraphRunnerLocal(TaskGraphRunner):
|
39
|
-
"""Session manages execution of a task graph"""
|
40
|
-
|
41
|
-
rundir : str
|
42
|
-
nproc : int = 4
|
43
|
-
done_task_m : Dict = dc.field(default_factory=dict)
|
44
|
-
_workers : List = dc.field(default_factory=list)
|
45
|
-
|
46
|
-
_inst : ClassVar['TaskGraphRunner'] = None
|
47
|
-
|
48
|
-
# Search path for .dfs files
|
49
|
-
create_subprocess : Callable = asyncio.create_subprocess_exec
|
50
|
-
_root_dir : str = None
|
51
|
-
_log : ClassVar = logging.getLogger("TaskGraphRunnerLocal")
|
52
|
-
|
53
|
-
def __post_init__(self):
|
54
|
-
if self.nproc == -1:
|
55
|
-
self.nproc = os.cpu_count()
|
56
|
-
for _ in range(self.nproc):
|
57
|
-
self._workers.append(LocalRunnerWorker(self))
|
58
|
-
|
59
|
-
|
60
|
-
async def exec(self, *args, **kwargs):
|
61
|
-
return await self.create_subprocess(*args, **kwargs)
|
62
|
-
|
63
|
-
async def run(self, task : Union[Task,List[Task]]) -> List['TaskData']:
|
64
|
-
if isinstance(task, Task):
|
65
|
-
unwrap = True
|
66
|
-
task = [task]
|
67
|
-
else:
|
68
|
-
unwrap = False
|
69
|
-
|
70
|
-
dep_m = {}
|
71
|
-
task_m = {}
|
72
|
-
|
73
|
-
for t in task:
|
74
|
-
self._mkDeps(dep_m, task_m, t)
|
75
|
-
|
76
|
-
self._log.debug("dep_m: %s" % str(dep_m))
|
77
|
-
|
78
|
-
order = list(toposort(dep_m))
|
79
|
-
|
80
|
-
self._log.debug("order: %s" % str(order))
|
81
|
-
|
82
|
-
active_task_l : List[Tuple[Task,Coroutine]]= []
|
83
|
-
# Now, iterate over the concurrent sets
|
84
|
-
for active_s in order:
|
85
|
-
|
86
|
-
# Check to see if all tasks are complete
|
87
|
-
done = True
|
88
|
-
for t in active_s:
|
89
|
-
while len(active_task_l) >= self.nproc and t not in self.done_task_m.keys():
|
90
|
-
# Wait for at least one job to complete
|
91
|
-
done, pending = await asyncio.wait(at[1] for at in active_task_l)
|
92
|
-
for d in done:
|
93
|
-
for i in range(len(active_task_l)):
|
94
|
-
if active_task_l[i][1] == d:
|
95
|
-
tt = active_task_l[i][0]
|
96
|
-
self.done_task_m[tt.name] = tt
|
97
|
-
active_task_l.pop(i)
|
98
|
-
break
|
99
|
-
if t not in self.done_task_m.keys():
|
100
|
-
task_t = task_m[t]
|
101
|
-
coro = asyncio.Task(task_t.do_run(self))
|
102
|
-
active_task_l.append((task_t, coro))
|
103
|
-
|
104
|
-
# Now, wait for tasks to complete
|
105
|
-
if len(active_task_l):
|
106
|
-
coros = list(at[1] for at in active_task_l)
|
107
|
-
res = await asyncio.gather(*coros)
|
108
|
-
|
109
|
-
# print("order: %s" % str(order))
|
110
|
-
#
|
111
|
-
# run_o = list(t.do_run() for t in task)
|
112
|
-
|
113
|
-
# ret = await asyncio.gather(*run_o)
|
114
|
-
ret = None
|
115
|
-
|
116
|
-
if unwrap:
|
117
|
-
return task[0].output
|
118
|
-
else:
|
119
|
-
return list(t.output for t in task)
|
120
|
-
|
121
|
-
def _mkDeps(self, dep_m, task_m, task):
|
122
|
-
if task.name not in dep_m.keys():
|
123
|
-
task_m[task.name] = task
|
124
|
-
dep_m[task.name] = set(t.name for t in task.depends)
|
125
|
-
for d in task.depends:
|
126
|
-
self._mkDeps(dep_m, task_m, d)
|
127
|
-
|
128
|
-
async def runTask(self, task : Task) -> 'TaskData':
|
129
|
-
return await task.do_run()
|
130
|
-
|
131
|
-
def queueTask(self, task : Task):
|
132
|
-
"""Queue a task for execution"""
|
133
|
-
pass
|
134
|
-
|
135
|
-
@dc.dataclass
|
136
|
-
class LocalRunnerWorker(object):
|
137
|
-
runner : TaskGraphRunnerLocal
|
138
|
-
task_s : List = dc.field(default_factory=list)
|
139
|
-
|
140
|
-
|
141
|
-
|
dv_flow/mgr/task_impl_data.py
DELETED
@@ -1,37 +0,0 @@
|
|
1
|
-
#****************************************************************************
|
2
|
-
#* task_impl_data.py
|
3
|
-
#*
|
4
|
-
#* Copyright 2023-2025 Matthew Ballance and Contributors
|
5
|
-
#*
|
6
|
-
#* Licensed under the Apache License, Version 2.0 (the "License"); you may
|
7
|
-
#* not use this file except in compliance with the License.
|
8
|
-
#* You may obtain a copy of the License at:
|
9
|
-
#*
|
10
|
-
#* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
-
#*
|
12
|
-
#* Unless required by applicable law or agreed to in writing, software
|
13
|
-
#* distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
-
#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
-
#* See the License for the specific language governing permissions and
|
16
|
-
#* limitations under the License.
|
17
|
-
#*
|
18
|
-
#* Created on:
|
19
|
-
#* Author:
|
20
|
-
#*
|
21
|
-
#****************************************************************************
|
22
|
-
from pydantic import BaseModel
|
23
|
-
from typing import Any, ClassVar, Dict, Set, List, Tuple
|
24
|
-
|
25
|
-
class TaskImplParams(BaseModel):
|
26
|
-
pass
|
27
|
-
|
28
|
-
class TaskImplSourceData(BaseModel):
|
29
|
-
params : Any
|
30
|
-
changed : bool
|
31
|
-
memento : Any
|
32
|
-
|
33
|
-
class TaskImplResultData(BaseModel):
|
34
|
-
data : List[Any]
|
35
|
-
changed : bool
|
36
|
-
memento : Any
|
37
|
-
|
dv_flow/mgr/task_memento.py
DELETED
@@ -1,34 +0,0 @@
|
|
1
|
-
#****************************************************************************
|
2
|
-
#* task_memento.py
|
3
|
-
#*
|
4
|
-
#* Copyright 2023-2025 Matthew Ballance and Contributors
|
5
|
-
#*
|
6
|
-
#* Licensed under the Apache License, Version 2.0 (the "License"); you may
|
7
|
-
#* not use this file except in compliance with the License.
|
8
|
-
#* You may obtain a copy of the License at:
|
9
|
-
#*
|
10
|
-
#* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
-
#*
|
12
|
-
#* Unless required by applicable law or agreed to in writing, software
|
13
|
-
#* distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
-
#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
-
#* See the License for the specific language governing permissions and
|
16
|
-
#* limitations under the License.
|
17
|
-
#*
|
18
|
-
#* Created on:
|
19
|
-
#* Author:
|
20
|
-
#*
|
21
|
-
#****************************************************************************
|
22
|
-
import pydantic.dataclasses as dc
|
23
|
-
from pydantic import BaseModel
|
24
|
-
from typing import Any, Dict, List
|
25
|
-
|
26
|
-
class TaskMemento(BaseModel):
|
27
|
-
# dep_ids : List[int] = dc.Field(default_factory=list)
|
28
|
-
# params : Dict[str,Any] = dc.Field(default_factory=dict)
|
29
|
-
|
30
|
-
pass
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
{dv_flow_mgr-0.0.1.13885203883a1.dist-info → dv_flow_mgr-0.0.1.13954441689a1.dist-info}/LICENSE
RENAMED
File without changes
|
File without changes
|
File without changes
|