flowweave 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowweave/__init__.py +6 -0
- flowweave/base.py +52 -0
- flowweave/cli.py +81 -0
- flowweave/flowweave.py +346 -0
- flowweave/message.py +90 -0
- flowweave/schema/flow.json +72 -0
- flowweave/schema/op_code.json +20 -0
- flowweave-1.0.0.dist-info/METADATA +26 -0
- flowweave-1.0.0.dist-info/RECORD +13 -0
- flowweave-1.0.0.dist-info/WHEEL +5 -0
- flowweave-1.0.0.dist-info/entry_points.txt +2 -0
- flowweave-1.0.0.dist-info/licenses/LICENSE +24 -0
- flowweave-1.0.0.dist-info/top_level.txt +1 -0
flowweave/__init__.py
ADDED
flowweave/base.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Standard library
|
|
2
|
+
from enum import IntEnum
|
|
3
|
+
from typing import IO, Optional
|
|
4
|
+
|
|
5
|
+
class Result(IntEnum):
|
|
6
|
+
FAIL = 0
|
|
7
|
+
SUCCESS = 1
|
|
8
|
+
IGNORE = 2
|
|
9
|
+
|
|
10
|
+
class TaskData:
|
|
11
|
+
def __init__(self,
|
|
12
|
+
name,
|
|
13
|
+
task_class,
|
|
14
|
+
option: dict,
|
|
15
|
+
stage_name: str,
|
|
16
|
+
flow_part: int,
|
|
17
|
+
flow_all: int,
|
|
18
|
+
do_only: str = None,
|
|
19
|
+
show_log: bool = False):
|
|
20
|
+
self.name = name
|
|
21
|
+
self.task_class = task_class
|
|
22
|
+
self.option = option
|
|
23
|
+
|
|
24
|
+
self.stage_name = stage_name
|
|
25
|
+
self.flow_part = flow_part
|
|
26
|
+
self.flow_all = flow_all
|
|
27
|
+
|
|
28
|
+
self.do_only = do_only
|
|
29
|
+
self.show_log = show_log
|
|
30
|
+
|
|
31
|
+
class FlowWeaveTaskRunner:
|
|
32
|
+
def set_task_data(self, task_data: TaskData) -> None:
|
|
33
|
+
self.task_data = task_data
|
|
34
|
+
|
|
35
|
+
def message(self,
|
|
36
|
+
*args: object,
|
|
37
|
+
sep: str = " ",
|
|
38
|
+
end: str = "\n",
|
|
39
|
+
file: Optional[IO[str]] = None,
|
|
40
|
+
flush: bool = False) -> None:
|
|
41
|
+
if not self.task_data:
|
|
42
|
+
raise Exception("task_data is 'None'")
|
|
43
|
+
|
|
44
|
+
part_num = self.task_data.flow_part
|
|
45
|
+
all_num = self.task_data.flow_all
|
|
46
|
+
stage = self.task_data.stage_name
|
|
47
|
+
task = self.task_data.name
|
|
48
|
+
head = f"[Flow {part_num} / {all_num}] {stage}/{task}"
|
|
49
|
+
|
|
50
|
+
args = [f"{head}: {str(a)}" for a in args]
|
|
51
|
+
|
|
52
|
+
print(*args, sep=sep, end=end, file=file, flush=flush)
|
flowweave/cli.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# Standard library
|
|
2
|
+
import argparse
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
# Third-party
|
|
7
|
+
import colorama
|
|
8
|
+
|
|
9
|
+
# Local application / relative imports
|
|
10
|
+
from .flowweave import FlowWeave
|
|
11
|
+
|
|
12
|
+
def get_setting_path(args):
|
|
13
|
+
setting_path = None
|
|
14
|
+
|
|
15
|
+
if args.flow_file:
|
|
16
|
+
setting_path = str(Path("flow") / f"{args.flow_file}.yml")
|
|
17
|
+
|
|
18
|
+
return setting_path
|
|
19
|
+
|
|
20
|
+
def serialize(obj) -> str:
|
|
21
|
+
if isinstance(obj, type):
|
|
22
|
+
return obj.__name__
|
|
23
|
+
raise TypeError(f"Type {type(obj)} not serializable")
|
|
24
|
+
|
|
25
|
+
def show_flow_op(setting_path: str, flow_name: str) -> None:
|
|
26
|
+
flow_data = FlowWeave.load_and_validate_schema(file=setting_path, schema="flow")
|
|
27
|
+
op_dic = FlowWeave.get_op_dic(flow_data)
|
|
28
|
+
print_op_dic(op_dic, flow_name)
|
|
29
|
+
|
|
30
|
+
def show_available_op() -> None:
|
|
31
|
+
op_dic = FlowWeave.get_available_op_dic()
|
|
32
|
+
print_op_dic(op_dic)
|
|
33
|
+
|
|
34
|
+
def print_op_dic(op_dic: dict, flow_name: str = "") -> None:
|
|
35
|
+
flow_text = ""
|
|
36
|
+
if "" != flow_name:
|
|
37
|
+
flow_text = f"in {flow_name} "
|
|
38
|
+
print(f"Operation code available {flow_text}(<op> : <class>)")
|
|
39
|
+
print(json.dumps(op_dic, indent=2, default=serialize))
|
|
40
|
+
|
|
41
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
42
|
+
parser = argparse.ArgumentParser(prog="flowweave")
|
|
43
|
+
|
|
44
|
+
subparsers = parser.add_subparsers(dest="command")
|
|
45
|
+
run_parser = subparsers.add_parser("run", help="Run a flow file")
|
|
46
|
+
run_parser.add_argument("flow_file", help="Path to YAML flow file")
|
|
47
|
+
run_parser.add_argument("-l", "--log", help="Show flow log", action='store_true')
|
|
48
|
+
run_parser.add_argument("-p", "--parallel", help="Run flow parallel", action='store_true')
|
|
49
|
+
|
|
50
|
+
info_parser = subparsers.add_parser("info", help="Show info")
|
|
51
|
+
info_parser.add_argument(
|
|
52
|
+
"flow_file",
|
|
53
|
+
nargs="?",
|
|
54
|
+
default=None,
|
|
55
|
+
help="Path to YAML flow file"
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
return parser
|
|
59
|
+
|
|
60
|
+
def main() -> None:
|
|
61
|
+
colorama.init(autoreset=True)
|
|
62
|
+
|
|
63
|
+
parser = build_parser()
|
|
64
|
+
args = parser.parse_args()
|
|
65
|
+
|
|
66
|
+
setting_path = get_setting_path(args)
|
|
67
|
+
|
|
68
|
+
if args.command == "run":
|
|
69
|
+
if not setting_path:
|
|
70
|
+
parser.error("run requires flow_file")
|
|
71
|
+
FlowWeave.run(setting_file=setting_path, parallel=args.parallel, show_log = args.log)
|
|
72
|
+
return
|
|
73
|
+
|
|
74
|
+
if args.command == "info":
|
|
75
|
+
if args.flow_file:
|
|
76
|
+
show_flow_op(setting_path, args.flow_file)
|
|
77
|
+
else:
|
|
78
|
+
show_available_op()
|
|
79
|
+
return
|
|
80
|
+
|
|
81
|
+
parser.print_help()
|
flowweave/flowweave.py
ADDED
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
# Standard library
|
|
2
|
+
import copy
|
|
3
|
+
import importlib
|
|
4
|
+
import itertools
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from importlib.resources import files
|
|
9
|
+
|
|
10
|
+
# Third-party
|
|
11
|
+
import jsonschema
|
|
12
|
+
import yaml
|
|
13
|
+
from prefect import flow, task, get_run_logger
|
|
14
|
+
|
|
15
|
+
# Local application / relative imports
|
|
16
|
+
from .base import Result, TaskData
|
|
17
|
+
from .message import FlowMessage
|
|
18
|
+
|
|
19
|
+
class StageData():
|
|
20
|
+
def __init__(self, name: str, stage_info: dict, default_option: dict, global_option: dict, op_dic: dict, flow_part: int, flow_all: int):
|
|
21
|
+
self.name = name
|
|
22
|
+
self.stage_info = stage_info
|
|
23
|
+
self.default_option = default_option
|
|
24
|
+
self.global_option = global_option
|
|
25
|
+
self.op_dic = op_dic
|
|
26
|
+
|
|
27
|
+
self.flow_part = flow_part
|
|
28
|
+
self.flow_all = flow_all
|
|
29
|
+
|
|
30
|
+
def __str__(self):
|
|
31
|
+
text = "== Stage ==\n"
|
|
32
|
+
text += f"Name: {self.name}\n"
|
|
33
|
+
text += f"Stage Info: {self.stage_info}\n"
|
|
34
|
+
text += f"Default: {self.default_option}\n"
|
|
35
|
+
text += f"Global: {self.global_option}\n"
|
|
36
|
+
text += f"Operation: {self.op_dic}\n"
|
|
37
|
+
text += "==========="
|
|
38
|
+
return text
|
|
39
|
+
|
|
40
|
+
class FlowWeaveTask():
|
|
41
|
+
task_class = None
|
|
42
|
+
|
|
43
|
+
@task
|
|
44
|
+
def start(prev_future, task_data: TaskData):
|
|
45
|
+
try:
|
|
46
|
+
task_instance = task_data.task_class.runner()
|
|
47
|
+
except AttributeError:
|
|
48
|
+
raise TypeError(f"{task_data.task_class} must define runner")
|
|
49
|
+
|
|
50
|
+
# set task member variables
|
|
51
|
+
setattr(task_instance, "task_data", task_data)
|
|
52
|
+
# options
|
|
53
|
+
for key, value in task_data.option.items():
|
|
54
|
+
if hasattr(task_instance, key):
|
|
55
|
+
setattr(task_instance, key, value)
|
|
56
|
+
if task_data.show_log:
|
|
57
|
+
FlowWeave._print_log(f"Task option {key} found: store {value}")
|
|
58
|
+
else:
|
|
59
|
+
if task_data.show_log:
|
|
60
|
+
FlowWeave._print_log(f"Task option {key} not found: ignore")
|
|
61
|
+
|
|
62
|
+
run_task = True
|
|
63
|
+
if prev_future:
|
|
64
|
+
if "pre_success" == task_data.do_only:
|
|
65
|
+
run_task = True if (Result.SUCCESS == prev_future.get("result")) else False
|
|
66
|
+
elif "pre_fail" == task_data.do_only:
|
|
67
|
+
run_task = True if (Result.FAIL == prev_future.get("result")) else False
|
|
68
|
+
|
|
69
|
+
if run_task:
|
|
70
|
+
FlowWeaveTask.message_task_start(prev_future, task_data)
|
|
71
|
+
|
|
72
|
+
try:
|
|
73
|
+
task_result = task_instance()
|
|
74
|
+
except Exception as e:
|
|
75
|
+
FlowMessage.error(e)
|
|
76
|
+
task_result = Result.FAIL
|
|
77
|
+
|
|
78
|
+
FlowMessage.task_end(task_data, task_result)
|
|
79
|
+
else:
|
|
80
|
+
FlowWeaveTask.message_task_ignore(prev_future, task_data)
|
|
81
|
+
task_result = Result.IGNORE
|
|
82
|
+
|
|
83
|
+
return {"name" : task_data.name, "option" : task_data.option, "result" : task_result}
|
|
84
|
+
|
|
85
|
+
def message_task_start(prev_future, task_data: TaskData):
|
|
86
|
+
if prev_future:
|
|
87
|
+
prev_task_name = prev_future.get("name")
|
|
88
|
+
FlowMessage.task_start_link(prev_task_name, task_data)
|
|
89
|
+
else:
|
|
90
|
+
FlowMessage.task_start(task_data)
|
|
91
|
+
|
|
92
|
+
def message_task_ignore(prev_future, task_data: TaskData):
|
|
93
|
+
if prev_future:
|
|
94
|
+
prev_task_name = prev_future.get("name")
|
|
95
|
+
FlowMessage.task_ignore_link(task_data, prev_task_name)
|
|
96
|
+
else:
|
|
97
|
+
FlowMessage.task_ignore(task_data)
|
|
98
|
+
|
|
99
|
+
class FlowWeave():
|
|
100
|
+
@flow
|
|
101
|
+
def run(setting_file: str, parallel: bool = False, show_log: bool = False) -> list[str]:
|
|
102
|
+
if not show_log:
|
|
103
|
+
logging.getLogger("prefect").setLevel(logging.CRITICAL)
|
|
104
|
+
|
|
105
|
+
flow_data = FlowWeave.load_and_validate_schema(setting_file, "flow")
|
|
106
|
+
|
|
107
|
+
op_dic = FlowWeave.get_op_dic(flow_data)
|
|
108
|
+
|
|
109
|
+
global_option = flow_data.get("global_option")
|
|
110
|
+
comb_list = list()
|
|
111
|
+
if global_option:
|
|
112
|
+
comb_list = FlowWeave._get_global_option_comb(global_option)
|
|
113
|
+
else:
|
|
114
|
+
comb_list = [{}]
|
|
115
|
+
|
|
116
|
+
comb_count = 0
|
|
117
|
+
all_count = len(comb_list)
|
|
118
|
+
futures = []
|
|
119
|
+
results = []
|
|
120
|
+
for comb in comb_list:
|
|
121
|
+
comb_count += 1
|
|
122
|
+
FlowMessage.flow_start(comb_count, all_count)
|
|
123
|
+
FlowMessage.flow_message(comb_count, all_count, comb)
|
|
124
|
+
if parallel:
|
|
125
|
+
futures.append(FlowWeave.run_flow.submit(flow_data=flow_data,
|
|
126
|
+
global_cmb=comb,
|
|
127
|
+
op_dic=op_dic,
|
|
128
|
+
part=comb_count,
|
|
129
|
+
all=all_count,
|
|
130
|
+
show_log=show_log))
|
|
131
|
+
else:
|
|
132
|
+
result = FlowWeave.run_flow(flow_data=flow_data,
|
|
133
|
+
global_cmb=comb,
|
|
134
|
+
op_dic=op_dic,
|
|
135
|
+
part=comb_count,
|
|
136
|
+
all=all_count,
|
|
137
|
+
show_log=show_log)
|
|
138
|
+
results.append(result)
|
|
139
|
+
FlowMessage.flow_end(comb_count, all_count, result)
|
|
140
|
+
|
|
141
|
+
if parallel:
|
|
142
|
+
comb_count = 0
|
|
143
|
+
for f in futures:
|
|
144
|
+
comb_count += 1
|
|
145
|
+
result = f.result()
|
|
146
|
+
results.append(result)
|
|
147
|
+
FlowMessage.flow_end(comb_count, all_count, result)
|
|
148
|
+
|
|
149
|
+
return results
|
|
150
|
+
|
|
151
|
+
def load_and_validate_schema(file: str, schema: str) -> dict:
|
|
152
|
+
data = FlowWeave._load_yaml(file)
|
|
153
|
+
schema = FlowWeave._load_schema(schema)
|
|
154
|
+
jsonschema.validate(instance=data, schema=schema)
|
|
155
|
+
|
|
156
|
+
return data
|
|
157
|
+
|
|
158
|
+
def _load_yaml(path: str) -> dict:
|
|
159
|
+
file_path = Path(path)
|
|
160
|
+
if not file_path.exists() or not file_path.is_file():
|
|
161
|
+
raise FileNotFoundError(f"{file_path.resolve()} does not exist or not file")
|
|
162
|
+
|
|
163
|
+
with open(Path(path), "r", encoding="utf-8") as f:
|
|
164
|
+
data = yaml.safe_load(f)
|
|
165
|
+
return data
|
|
166
|
+
|
|
167
|
+
def _load_schema(schema: str) -> dict:
|
|
168
|
+
schema_path = files("flowweave")/ "schema" / f"{schema}.json"
|
|
169
|
+
with schema_path.open("r", encoding="utf-8") as f:
|
|
170
|
+
data = json.load(f)
|
|
171
|
+
|
|
172
|
+
return data
|
|
173
|
+
|
|
174
|
+
def get_op_dic(flow_data: dict):
|
|
175
|
+
return_dic = dict()
|
|
176
|
+
|
|
177
|
+
op_source = flow_data.get("op_source")
|
|
178
|
+
op_source_list = op_source if isinstance(op_source, list) else [op_source]
|
|
179
|
+
for source in op_source_list:
|
|
180
|
+
source_name = f"task/{source}"
|
|
181
|
+
setting_file = f"{source_name.replace('.', '/')}/op_code.yml"
|
|
182
|
+
return_dic |= FlowWeave._get_op_dic_from_setting_file(setting_file)
|
|
183
|
+
|
|
184
|
+
return return_dic
|
|
185
|
+
|
|
186
|
+
def get_available_op_dic():
|
|
187
|
+
return_dic = dict()
|
|
188
|
+
|
|
189
|
+
base_path = Path("task")
|
|
190
|
+
avaliable_settings = [str(f) for f in base_path.rglob("op_code.yml")]
|
|
191
|
+
for setting in avaliable_settings:
|
|
192
|
+
place = setting.replace("\\", ".").removeprefix("task.").removesuffix(".op_code.yml")
|
|
193
|
+
return_dic[place] = FlowWeave._get_op_dic_from_setting_file(setting.replace("\\", "/"))
|
|
194
|
+
|
|
195
|
+
return return_dic
|
|
196
|
+
|
|
197
|
+
def _get_op_dic_from_setting_file(setting_file: str):
|
|
198
|
+
return_dic = dict()
|
|
199
|
+
|
|
200
|
+
setting = FlowWeave.load_and_validate_schema(setting_file, "op_code")
|
|
201
|
+
source_name = setting_file.removesuffix("/op_code.yml").replace("/", ".")
|
|
202
|
+
|
|
203
|
+
op_dic = setting.get("op", {})
|
|
204
|
+
for op, op_info in op_dic.items():
|
|
205
|
+
script_name = op_info.get('script')
|
|
206
|
+
op_class = FlowWeave._get_op_class(source_name, script_name)
|
|
207
|
+
|
|
208
|
+
return_dic[str(op)] = op_class
|
|
209
|
+
|
|
210
|
+
return return_dic
|
|
211
|
+
|
|
212
|
+
def _get_op_class(source_name: str, script_name: str):
|
|
213
|
+
file_path = Path(f"{source_name.replace('.', '/')}/{script_name}.py").resolve()
|
|
214
|
+
spec = importlib.util.spec_from_file_location(file_path.stem, file_path)
|
|
215
|
+
module = importlib.util.module_from_spec(spec)
|
|
216
|
+
spec.loader.exec_module(module)
|
|
217
|
+
op_class = getattr(module, "Task")
|
|
218
|
+
|
|
219
|
+
return op_class
|
|
220
|
+
|
|
221
|
+
def _get_global_option_comb(global_option: dict) -> list:
|
|
222
|
+
keys = list(global_option.keys())
|
|
223
|
+
value_lists = []
|
|
224
|
+
|
|
225
|
+
for key in keys:
|
|
226
|
+
inner_dict = global_option[key]
|
|
227
|
+
value_lists.append([dict(zip(inner_dict.keys(), v))
|
|
228
|
+
for v in itertools.product(*inner_dict.values())])
|
|
229
|
+
|
|
230
|
+
all_combinations = []
|
|
231
|
+
for combo in itertools.product(*value_lists):
|
|
232
|
+
combined = dict(zip(keys, combo))
|
|
233
|
+
all_combinations.append(combined)
|
|
234
|
+
|
|
235
|
+
return all_combinations
|
|
236
|
+
|
|
237
|
+
@task
|
|
238
|
+
def run_flow(flow_data: dict, global_cmb: dict, op_dic: dict, part: int, all: int, show_log: bool = False) -> list[str]:
|
|
239
|
+
flow_result = Result.SUCCESS
|
|
240
|
+
|
|
241
|
+
if show_log:
|
|
242
|
+
text = "= Flow =\n"
|
|
243
|
+
text += f"Stage: {flow_data.get('flow')}\n"
|
|
244
|
+
text += "========"
|
|
245
|
+
FlowWeave._print_log(text)
|
|
246
|
+
|
|
247
|
+
default_option = flow_data.get("default_option")
|
|
248
|
+
|
|
249
|
+
stage_list = flow_data.get("flow")
|
|
250
|
+
for stage in stage_list:
|
|
251
|
+
FlowMessage.stage_start(stage, part, all)
|
|
252
|
+
|
|
253
|
+
stage_info = flow_data.get("stage", {}).get(stage)
|
|
254
|
+
stage_global_option = FlowWeave._get_stage_global_option(global_cmb, stage)
|
|
255
|
+
|
|
256
|
+
stage_data = StageData(stage, stage_info, default_option, stage_global_option, op_dic, part, all)
|
|
257
|
+
if show_log:
|
|
258
|
+
FlowWeave._print_log(str(stage_data))
|
|
259
|
+
|
|
260
|
+
result = FlowWeave._run_stage(stage_data, show_log)
|
|
261
|
+
if Result.FAIL == result:
|
|
262
|
+
flow_result = Result.FAIL
|
|
263
|
+
|
|
264
|
+
FlowMessage.stage_end(stage, part, all, flow_result)
|
|
265
|
+
|
|
266
|
+
return flow_result
|
|
267
|
+
|
|
268
|
+
def _get_stage_global_option(global_cmb: dict, stage: str) -> dict:
|
|
269
|
+
stage_global_option = dict()
|
|
270
|
+
|
|
271
|
+
for stages, option in global_cmb.items():
|
|
272
|
+
stage_list = [x.strip() for x in stages.split(",")]
|
|
273
|
+
if stage in stage_list:
|
|
274
|
+
stage_global_option |= option
|
|
275
|
+
|
|
276
|
+
return stage_global_option
|
|
277
|
+
|
|
278
|
+
def _print_log(text: str):
|
|
279
|
+
logger = get_run_logger()
|
|
280
|
+
stage_text_list = text.split("\n")
|
|
281
|
+
for text in stage_text_list:
|
|
282
|
+
logger.info(f"{text}")
|
|
283
|
+
|
|
284
|
+
def _run_stage(stage_data: StageData, show_log: bool = False):
|
|
285
|
+
stage_result = Result.SUCCESS
|
|
286
|
+
|
|
287
|
+
all_futures = []
|
|
288
|
+
|
|
289
|
+
for task_name, task_dic in stage_data.stage_info.items():
|
|
290
|
+
part = task_dic.get("chain", {}).get("part", "head")
|
|
291
|
+
if "head" == part:
|
|
292
|
+
all_futures.extend(
|
|
293
|
+
FlowWeave._run_task(stage_data, task_name, None, None, show_log)
|
|
294
|
+
)
|
|
295
|
+
|
|
296
|
+
for f in all_futures:
|
|
297
|
+
result = f.result()
|
|
298
|
+
if Result.FAIL == result.get("result"):
|
|
299
|
+
stage_result = Result.FAIL
|
|
300
|
+
|
|
301
|
+
return stage_result
|
|
302
|
+
|
|
303
|
+
def _run_task(stage_data: dict, task_name: str, prev_future = None, visited = None, show_log: bool = False):
|
|
304
|
+
if visited is None:
|
|
305
|
+
visited = set()
|
|
306
|
+
if task_name in visited:
|
|
307
|
+
raise Exception(f"Cycle detected at task '{task_name}' in {visited}")
|
|
308
|
+
visited.add(task_name)
|
|
309
|
+
|
|
310
|
+
task_dic = stage_data.stage_info.get(task_name)
|
|
311
|
+
|
|
312
|
+
task_module = stage_data.op_dic.get(task_dic.get('op'))
|
|
313
|
+
if not task_module:
|
|
314
|
+
raise Exception(f"module of op '{task_dic.get('op')}' for '{task_name}' not found")
|
|
315
|
+
|
|
316
|
+
default_option = stage_data.default_option or {}
|
|
317
|
+
global_option = stage_data.global_option or {}
|
|
318
|
+
task_option = copy.deepcopy(
|
|
319
|
+
default_option
|
|
320
|
+
| global_option
|
|
321
|
+
| task_dic.get("option", {})
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
task_data = TaskData(name=task_name,
|
|
325
|
+
task_class=task_module,
|
|
326
|
+
option=task_option,
|
|
327
|
+
stage_name=stage_data.name,
|
|
328
|
+
flow_part=stage_data.flow_part,
|
|
329
|
+
flow_all=stage_data.flow_all,
|
|
330
|
+
do_only=task_dic.get("do_only"),
|
|
331
|
+
show_log=show_log)
|
|
332
|
+
if prev_future is None:
|
|
333
|
+
future = task_module.start.submit(None, task_data)
|
|
334
|
+
else:
|
|
335
|
+
future = task_module.start.submit(prev_future, task_data)
|
|
336
|
+
|
|
337
|
+
links = task_dic.get("chain", {}).get("next", [])
|
|
338
|
+
links = links if isinstance(links, list) else [links]
|
|
339
|
+
|
|
340
|
+
futures = [future]
|
|
341
|
+
for link in links:
|
|
342
|
+
futures.extend(
|
|
343
|
+
FlowWeave._run_task(stage_data, link, future, visited.copy(), show_log)
|
|
344
|
+
)
|
|
345
|
+
|
|
346
|
+
return futures
|
flowweave/message.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# Standard library
|
|
2
|
+
from typing import IO, Optional
|
|
3
|
+
|
|
4
|
+
# Third-party
|
|
5
|
+
from colorama import Fore
|
|
6
|
+
|
|
7
|
+
# Local application / relative imports
|
|
8
|
+
from .base import Result, TaskData
|
|
9
|
+
|
|
10
|
+
class FlowMessage:
|
|
11
|
+
@staticmethod
|
|
12
|
+
def _print(text: str) -> None:
|
|
13
|
+
print(text)
|
|
14
|
+
|
|
15
|
+
@staticmethod
|
|
16
|
+
def get_result_text(result: Result) -> str:
|
|
17
|
+
text = ""
|
|
18
|
+
|
|
19
|
+
if Result.SUCCESS == result:
|
|
20
|
+
text = f"{Fore.GREEN}SUCCESS"
|
|
21
|
+
elif Result.IGNORE == result:
|
|
22
|
+
text = f"{Fore.CYAN}IGNORE"
|
|
23
|
+
elif Result.FAIL == result:
|
|
24
|
+
text = f"{Fore.CYAN}FAIL"
|
|
25
|
+
else:
|
|
26
|
+
text = f"{Fore.RED}UNKNOWN: {result.name}({result.value})"
|
|
27
|
+
|
|
28
|
+
return text
|
|
29
|
+
|
|
30
|
+
@staticmethod
|
|
31
|
+
def flow_start(part: int, all: int) -> None:
|
|
32
|
+
text = f"{Fore.YELLOW}[Flow {part} / {all}] Start"
|
|
33
|
+
FlowMessage._print(text)
|
|
34
|
+
|
|
35
|
+
@staticmethod
|
|
36
|
+
def flow_message(part: int, all: int, global_option: dict) -> None:
|
|
37
|
+
text = f"[Flow {part} / {all}] {global_option}"
|
|
38
|
+
FlowMessage._print(text)
|
|
39
|
+
|
|
40
|
+
@staticmethod
|
|
41
|
+
def flow_end(part: int, all: int, result: Result) -> None:
|
|
42
|
+
result_text = FlowMessage.get_result_text(result)
|
|
43
|
+
text = f"{Fore.YELLOW}[Flow {part} / {all}] Finish - {result_text}"
|
|
44
|
+
FlowMessage._print(text)
|
|
45
|
+
|
|
46
|
+
@staticmethod
|
|
47
|
+
def stage_start(stage: str, part: int, all: int) -> None:
|
|
48
|
+
text = f"{Fore.MAGENTA}[Flow {part} / {all}] Start Stage {stage}"
|
|
49
|
+
FlowMessage._print(text)
|
|
50
|
+
|
|
51
|
+
@staticmethod
|
|
52
|
+
def stage_end(stage: str, part: int, all: int, result: Result) -> None:
|
|
53
|
+
result_text = FlowMessage.get_result_text(result)
|
|
54
|
+
text = f"{Fore.MAGENTA}[Flow {part} / {all}] Finish Stage {stage} - {result_text}"
|
|
55
|
+
FlowMessage._print(text)
|
|
56
|
+
|
|
57
|
+
@staticmethod
|
|
58
|
+
def task_start(task_data: TaskData) -> None:
|
|
59
|
+
text = f"{Fore.CYAN}[Flow {task_data.flow_part} / {task_data.flow_all}] Start Task {task_data.stage_name}/{task_data.name}"
|
|
60
|
+
FlowMessage._print(text)
|
|
61
|
+
|
|
62
|
+
@staticmethod
|
|
63
|
+
def task_start_link(prev_task: str, task_data: TaskData) -> None:
|
|
64
|
+
text = f"{Fore.CYAN}[Flow {task_data.flow_part} / {task_data.flow_all}] Start Link Task {task_data.stage_name}/{prev_task} -> {task_data.name}"
|
|
65
|
+
FlowMessage._print(text)
|
|
66
|
+
|
|
67
|
+
@staticmethod
|
|
68
|
+
def task_ignore(task_data: TaskData) -> None:
|
|
69
|
+
text = f"{Fore.CYAN}[Flow {task_data.flow_part} / {task_data.flow_all}] Ignore {task_data.name} (do_only : {task_data.do_only})"
|
|
70
|
+
FlowMessage._print(text)
|
|
71
|
+
|
|
72
|
+
@staticmethod
|
|
73
|
+
def task_ignore_link(task_data: TaskData, prev_task_name: str) -> None:
|
|
74
|
+
text = f"{Fore.CYAN}[Flow {task_data.flow_part} / {task_data.flow_all}] Ignore {prev_task_name} -> {task_data.name} (do_only : {task_data.do_only})"
|
|
75
|
+
FlowMessage._print(text)
|
|
76
|
+
|
|
77
|
+
@staticmethod
|
|
78
|
+
def task_end(task_data: TaskData, result: Result) -> None:
|
|
79
|
+
result_text = FlowMessage.get_result_text(result)
|
|
80
|
+
text = f"{Fore.CYAN}[Flow {task_data.flow_part} / {task_data.flow_all}] Finish Task {task_data.stage_name}/{task_data.name} - {result_text}"
|
|
81
|
+
FlowMessage._print(text)
|
|
82
|
+
|
|
83
|
+
@staticmethod
|
|
84
|
+
def error(*args: object,
|
|
85
|
+
sep: str = " ",
|
|
86
|
+
end: str = "\n",
|
|
87
|
+
file: Optional[IO[str]] = None,
|
|
88
|
+
flush: bool = False) -> None:
|
|
89
|
+
args = [f"{Fore.RED}{str(a)}" for a in args]
|
|
90
|
+
FlowMessage._print(*args, sep=sep, end=end, file=file, flush=flush)
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
{
|
|
2
|
+
"type": "object",
|
|
3
|
+
"properties": {
|
|
4
|
+
"flow": {
|
|
5
|
+
"type": ["array", "string"],
|
|
6
|
+
"items": { "type": "string" }
|
|
7
|
+
},
|
|
8
|
+
|
|
9
|
+
"op_source": {
|
|
10
|
+
"type": ["array", "string"],
|
|
11
|
+
"items": { "type": "string" }
|
|
12
|
+
},
|
|
13
|
+
|
|
14
|
+
"global_option": {
|
|
15
|
+
"type": "object",
|
|
16
|
+
"additionalProperties": {
|
|
17
|
+
"type": "object",
|
|
18
|
+
"additionalProperties": {
|
|
19
|
+
"type": "array",
|
|
20
|
+
"items": {}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
},
|
|
24
|
+
|
|
25
|
+
"default_option": {
|
|
26
|
+
"type": "object",
|
|
27
|
+
"additionalProperties": true
|
|
28
|
+
},
|
|
29
|
+
|
|
30
|
+
"stage": {
|
|
31
|
+
"type": "object",
|
|
32
|
+
"patternProperties": {
|
|
33
|
+
".*": {
|
|
34
|
+
"type": "object",
|
|
35
|
+
"patternProperties": {
|
|
36
|
+
".*": {
|
|
37
|
+
"type": "object",
|
|
38
|
+
"properties": {
|
|
39
|
+
"op": { "type": "string" },
|
|
40
|
+
|
|
41
|
+
"do_only": {
|
|
42
|
+
"type": "string",
|
|
43
|
+
"enum": ["pre_success", "pre_fail"]
|
|
44
|
+
},
|
|
45
|
+
"chain": {
|
|
46
|
+
"type": "object",
|
|
47
|
+
"properties": {
|
|
48
|
+
"part": { "type": "string" },
|
|
49
|
+
"next": {
|
|
50
|
+
"type": ["array", "string"],
|
|
51
|
+
"items": { "type": "string" }
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
"additionalProperties": false
|
|
55
|
+
},
|
|
56
|
+
"option": {
|
|
57
|
+
"type": "object",
|
|
58
|
+
"additionalProperties": true
|
|
59
|
+
}
|
|
60
|
+
},
|
|
61
|
+
"required": ["op"],
|
|
62
|
+
"additionalProperties": false
|
|
63
|
+
}
|
|
64
|
+
},
|
|
65
|
+
"additionalProperties": true
|
|
66
|
+
}
|
|
67
|
+
},
|
|
68
|
+
"additionalProperties": true
|
|
69
|
+
}
|
|
70
|
+
},
|
|
71
|
+
"required": ["flow", "op_source", "stage"]
|
|
72
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{
|
|
2
|
+
"type": "object",
|
|
3
|
+
"properties": {
|
|
4
|
+
"op": {
|
|
5
|
+
"type": "object",
|
|
6
|
+
"patternProperties": {
|
|
7
|
+
".*": {
|
|
8
|
+
"type": "object",
|
|
9
|
+
"properties": {
|
|
10
|
+
"script": { "type": "string" }
|
|
11
|
+
},
|
|
12
|
+
"required": ["script"],
|
|
13
|
+
"additionalProperties": false
|
|
14
|
+
}
|
|
15
|
+
},
|
|
16
|
+
"additionalProperties": false
|
|
17
|
+
}
|
|
18
|
+
},
|
|
19
|
+
"required": ["op"]
|
|
20
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: flowweave
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: YAML-based workflow runner for task orchestration
|
|
5
|
+
Author: syatch
|
|
6
|
+
Requires-Python: >=3.9
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: colorama>=0.4.6
|
|
10
|
+
Requires-Dist: jsonschema>=4.25.0
|
|
11
|
+
Requires-Dist: prefect<3,>=2
|
|
12
|
+
Requires-Dist: pyyaml<7,>=6
|
|
13
|
+
Dynamic: license-file
|
|
14
|
+
|
|
15
|
+
# FlowWeave
|
|
16
|
+
YAML-based workflow runner for task orchestration
|
|
17
|
+
|
|
18
|
+
This project is in early development.
|
|
19
|
+
|
|
20
|
+
## Installation
|
|
21
|
+
|
|
22
|
+
Install FlowWeave using pip:
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install flowweave
|
|
26
|
+
```
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
flowweave/__init__.py,sha256=FA0lhFSH-_LhDGGJKyazusNtboi79-iDOAngsCFrlPM,163
|
|
2
|
+
flowweave/base.py,sha256=tw8yMrN1KQ3f65GAT60L0k1cX2Iz4KixYqhcgRjrF_Q,1517
|
|
3
|
+
flowweave/cli.py,sha256=djDcW0XN7YN_DRmdKEXZC1Ob2JPfjSrXvgOdGVtE_kc,2453
|
|
4
|
+
flowweave/flowweave.py,sha256=GtTNcnvHQNIg3qO6aHQusn0htAV3ja80SB8uHspHMsk,12964
|
|
5
|
+
flowweave/message.py,sha256=duU4KMMS3LTGubTHO9iAghoRImtgBdxQJcgzVecRsFQ,3506
|
|
6
|
+
flowweave/schema/flow.json,sha256=K7EAcH2KqniAPdNJWJdLiOYC77D2SGg50SOU2frC7LU,1810
|
|
7
|
+
flowweave/schema/op_code.json,sha256=C9JeyBvSC6fA8Ss0exk3nsPjTRq_2XpK2gshvAcavbQ,414
|
|
8
|
+
flowweave-1.0.0.dist-info/licenses/LICENSE,sha256=iN7x3Cz45_nCPCn23daVPqs0m_ZWPvTdayE2jg2OSmY,1203
|
|
9
|
+
flowweave-1.0.0.dist-info/METADATA,sha256=PrX3pSQgb8FMJtHSHNbApOwGcXu9BB3-kUVSreV944o,566
|
|
10
|
+
flowweave-1.0.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
11
|
+
flowweave-1.0.0.dist-info/entry_points.txt,sha256=qV4hkGzamuf-N1eNz-h_tRQ89Wa_py2jIdOSTeGAE1M,49
|
|
12
|
+
flowweave-1.0.0.dist-info/top_level.txt,sha256=7fLs0F6CROwWhmjP9CyKODB25HJohoougiOCCL1YD_Y,10
|
|
13
|
+
flowweave-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
This project depends on third-party open source software.
|
|
2
|
+
Their licenses are governed by their respective authors.
|
|
3
|
+
|
|
4
|
+
MIT License
|
|
5
|
+
|
|
6
|
+
Copyright (c) 2026 syatch
|
|
7
|
+
|
|
8
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
9
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
10
|
+
in the Software without restriction, including without limitation the rights
|
|
11
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
12
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
13
|
+
furnished to do so, subject to the following conditions:
|
|
14
|
+
|
|
15
|
+
The above copyright notice and this permission notice shall be included in all
|
|
16
|
+
copies or substantial portions of the Software.
|
|
17
|
+
|
|
18
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
19
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
20
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
21
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
22
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
23
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
24
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
flowweave
|