flowweave 1.0.0__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flowweave-1.0.0 → flowweave-2.0.0}/PKG-INFO +2 -1
- {flowweave-1.0.0 → flowweave-2.0.0}/flowweave/__init__.py +2 -2
- {flowweave-1.0.0 → flowweave-2.0.0}/flowweave/base.py +33 -1
- {flowweave-1.0.0 → flowweave-2.0.0}/flowweave/cli.py +9 -6
- {flowweave-1.0.0 → flowweave-2.0.0}/flowweave/flowweave.py +27 -16
- {flowweave-1.0.0 → flowweave-2.0.0}/flowweave/message.py +9 -5
- {flowweave-1.0.0 → flowweave-2.0.0}/flowweave.egg-info/PKG-INFO +2 -1
- {flowweave-1.0.0 → flowweave-2.0.0}/pyproject.toml +2 -1
- {flowweave-1.0.0 → flowweave-2.0.0}/LICENSE +0 -0
- {flowweave-1.0.0 → flowweave-2.0.0}/README.md +0 -0
- {flowweave-1.0.0 → flowweave-2.0.0}/flowweave/schema/flow.json +0 -0
- {flowweave-1.0.0 → flowweave-2.0.0}/flowweave/schema/op_code.json +0 -0
- {flowweave-1.0.0 → flowweave-2.0.0}/flowweave.egg-info/SOURCES.txt +0 -0
- {flowweave-1.0.0 → flowweave-2.0.0}/flowweave.egg-info/dependency_links.txt +0 -0
- {flowweave-1.0.0 → flowweave-2.0.0}/flowweave.egg-info/entry_points.txt +0 -0
- {flowweave-1.0.0 → flowweave-2.0.0}/flowweave.egg-info/requires.txt +0 -0
- {flowweave-1.0.0 → flowweave-2.0.0}/flowweave.egg-info/top_level.txt +0 -0
- {flowweave-1.0.0 → flowweave-2.0.0}/setup.cfg +0 -0
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
from enum import IntEnum
|
|
3
3
|
from typing import IO, Optional
|
|
4
4
|
|
|
5
|
+
from colorama import Fore
|
|
6
|
+
|
|
5
7
|
class Result(IntEnum):
|
|
6
8
|
FAIL = 0
|
|
7
9
|
SUCCESS = 1
|
|
@@ -29,6 +31,17 @@ class TaskData:
|
|
|
29
31
|
self.show_log = show_log
|
|
30
32
|
|
|
31
33
|
class FlowWeaveTaskRunner:
|
|
34
|
+
def __init__(self, prev_future):
|
|
35
|
+
self.prev_future = prev_future
|
|
36
|
+
self.return_data = None
|
|
37
|
+
|
|
38
|
+
def __call__(self) -> Result:
|
|
39
|
+
result = self.run()
|
|
40
|
+
return result
|
|
41
|
+
|
|
42
|
+
def run(self) -> Result:
|
|
43
|
+
return Result.SUCCESS
|
|
44
|
+
|
|
32
45
|
def set_task_data(self, task_data: TaskData) -> None:
|
|
33
46
|
self.task_data = task_data
|
|
34
47
|
|
|
@@ -49,4 +62,23 @@ class FlowWeaveTaskRunner:
|
|
|
49
62
|
|
|
50
63
|
args = [f"{head}: {str(a)}" for a in args]
|
|
51
64
|
|
|
52
|
-
print(*args, sep=sep, end=end, file=file, flush=flush)
|
|
65
|
+
print(*args, sep=sep, end=end, file=file, flush=flush)
|
|
66
|
+
|
|
67
|
+
def error(self,
|
|
68
|
+
*args: object,
|
|
69
|
+
sep: str = " ",
|
|
70
|
+
end: str = "\n",
|
|
71
|
+
file: Optional[IO[str]] = None,
|
|
72
|
+
flush: bool = False) -> None:
|
|
73
|
+
if not self.task_data:
|
|
74
|
+
raise Exception("task_data is 'None'")
|
|
75
|
+
|
|
76
|
+
part_num = self.task_data.flow_part
|
|
77
|
+
all_num = self.task_data.flow_all
|
|
78
|
+
stage = self.task_data.stage_name
|
|
79
|
+
task = self.task_data.name
|
|
80
|
+
head = f"[Flow {part_num} / {all_num}] {stage}/{task}"
|
|
81
|
+
|
|
82
|
+
args = [f"{head}: {Fore.RED}{str(a)}" for a in args]
|
|
83
|
+
|
|
84
|
+
print(*args, sep=sep, end=end, file=file, flush=flush)
|
|
@@ -8,6 +8,7 @@ import colorama
|
|
|
8
8
|
|
|
9
9
|
# Local application / relative imports
|
|
10
10
|
from .flowweave import FlowWeave
|
|
11
|
+
from .base import Result
|
|
11
12
|
|
|
12
13
|
def get_setting_path(args):
|
|
13
14
|
setting_path = None
|
|
@@ -58,6 +59,8 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
58
59
|
return parser
|
|
59
60
|
|
|
60
61
|
def main() -> None:
|
|
62
|
+
result = Result.SUCCESS
|
|
63
|
+
|
|
61
64
|
colorama.init(autoreset=True)
|
|
62
65
|
|
|
63
66
|
parser = build_parser()
|
|
@@ -68,14 +71,14 @@ def main() -> None:
|
|
|
68
71
|
if args.command == "run":
|
|
69
72
|
if not setting_path:
|
|
70
73
|
parser.error("run requires flow_file")
|
|
71
|
-
FlowWeave.run(setting_file=setting_path, parallel=args.parallel, show_log = args.log)
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
if args.command == "info":
|
|
74
|
+
results = FlowWeave.run(setting_file=setting_path, parallel=args.parallel, show_log = args.log)
|
|
75
|
+
result = all(x == Result.SUCCESS for x in results)
|
|
76
|
+
elif args.command == "info":
|
|
75
77
|
if args.flow_file:
|
|
76
78
|
show_flow_op(setting_path, args.flow_file)
|
|
77
79
|
else:
|
|
78
80
|
show_available_op()
|
|
79
|
-
|
|
81
|
+
else:
|
|
82
|
+
parser.print_help()
|
|
80
83
|
|
|
81
|
-
|
|
84
|
+
return result
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
# Standard library
|
|
2
2
|
import copy
|
|
3
3
|
import importlib
|
|
4
|
+
from importlib.resources import files
|
|
4
5
|
import itertools
|
|
5
6
|
import json
|
|
6
7
|
import logging
|
|
7
8
|
from pathlib import Path
|
|
8
|
-
|
|
9
|
+
import sys
|
|
9
10
|
|
|
10
11
|
# Third-party
|
|
11
12
|
import jsonschema
|
|
@@ -43,7 +44,7 @@ class FlowWeaveTask():
|
|
|
43
44
|
@task
|
|
44
45
|
def start(prev_future, task_data: TaskData):
|
|
45
46
|
try:
|
|
46
|
-
task_instance = task_data.task_class.runner()
|
|
47
|
+
task_instance = task_data.task_class.runner(prev_future)
|
|
47
48
|
except AttributeError:
|
|
48
49
|
raise TypeError(f"{task_data.task_class} must define runner")
|
|
49
50
|
|
|
@@ -58,7 +59,6 @@ class FlowWeaveTask():
|
|
|
58
59
|
else:
|
|
59
60
|
if task_data.show_log:
|
|
60
61
|
FlowWeave._print_log(f"Task option {key} not found: ignore")
|
|
61
|
-
|
|
62
62
|
run_task = True
|
|
63
63
|
if prev_future:
|
|
64
64
|
if "pre_success" == task_data.do_only:
|
|
@@ -70,7 +70,7 @@ class FlowWeaveTask():
|
|
|
70
70
|
FlowWeaveTask.message_task_start(prev_future, task_data)
|
|
71
71
|
|
|
72
72
|
try:
|
|
73
|
-
task_result = task_instance()
|
|
73
|
+
task_result, return_data = task_instance()
|
|
74
74
|
except Exception as e:
|
|
75
75
|
FlowMessage.error(e)
|
|
76
76
|
task_result = Result.FAIL
|
|
@@ -80,7 +80,7 @@ class FlowWeaveTask():
|
|
|
80
80
|
FlowWeaveTask.message_task_ignore(prev_future, task_data)
|
|
81
81
|
task_result = Result.IGNORE
|
|
82
82
|
|
|
83
|
-
return {"name" : task_data.name, "option" : task_data.option, "result" : task_result}
|
|
83
|
+
return {"name" : task_data.name, "option" : task_data.option, "data" : return_data, "result" : task_result}
|
|
84
84
|
|
|
85
85
|
def message_task_start(prev_future, task_data: TaskData):
|
|
86
86
|
if prev_future:
|
|
@@ -190,33 +190,44 @@ class FlowWeave():
|
|
|
190
190
|
avaliable_settings = [str(f) for f in base_path.rglob("op_code.yml")]
|
|
191
191
|
for setting in avaliable_settings:
|
|
192
192
|
place = setting.replace("\\", ".").removeprefix("task.").removesuffix(".op_code.yml")
|
|
193
|
-
return_dic[place] = FlowWeave._get_op_dic_from_setting_file(setting.replace("\\", "/"))
|
|
193
|
+
return_dic[place] = FlowWeave._get_op_dic_from_setting_file(setting.replace("\\", "/"), info=True)
|
|
194
194
|
|
|
195
195
|
return return_dic
|
|
196
196
|
|
|
197
|
-
def _get_op_dic_from_setting_file(setting_file: str):
|
|
197
|
+
def _get_op_dic_from_setting_file(setting_file: str, info: bool = False):
|
|
198
198
|
return_dic = dict()
|
|
199
199
|
|
|
200
200
|
setting = FlowWeave.load_and_validate_schema(setting_file, "op_code")
|
|
201
201
|
source_name = setting_file.removesuffix("/op_code.yml").replace("/", ".")
|
|
202
202
|
|
|
203
|
+
task_root = Path("task").resolve()
|
|
204
|
+
if str(task_root.parent) not in sys.path:
|
|
205
|
+
sys.path.insert(0, str(task_root.parent))
|
|
206
|
+
|
|
203
207
|
op_dic = setting.get("op", {})
|
|
204
208
|
for op, op_info in op_dic.items():
|
|
205
209
|
script_name = op_info.get('script')
|
|
206
|
-
op_class = FlowWeave._get_op_class(source_name, script_name)
|
|
210
|
+
op_class = FlowWeave._get_op_class(source_name, script_name, info)
|
|
207
211
|
|
|
208
212
|
return_dic[str(op)] = op_class
|
|
209
213
|
|
|
210
214
|
return return_dic
|
|
211
215
|
|
|
212
|
-
def _get_op_class(source_name: str, script_name: str):
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
216
|
+
def _get_op_class(source_name: str, script_name: str, info: bool = False):
|
|
217
|
+
module_name = f"{source_name}.{script_name}"
|
|
218
|
+
try:
|
|
219
|
+
module = importlib.import_module(module_name)
|
|
220
|
+
except Exception as e:
|
|
221
|
+
raise RuntimeError(f"Failed to import {module_name}: {e}")
|
|
222
|
+
|
|
223
|
+
if not hasattr(module, "Task"):
|
|
224
|
+
raise RuntimeError(f"'Task' class not found in {module_name}")
|
|
225
|
+
|
|
226
|
+
return_module = module.Task
|
|
227
|
+
if info:
|
|
228
|
+
return_module = module.Task.runner
|
|
218
229
|
|
|
219
|
-
return
|
|
230
|
+
return return_module
|
|
220
231
|
|
|
221
232
|
def _get_global_option_comb(global_option: dict) -> list:
|
|
222
233
|
keys = list(global_option.keys())
|
|
@@ -343,4 +354,4 @@ class FlowWeave():
|
|
|
343
354
|
FlowWeave._run_task(stage_data, link, future, visited.copy(), show_log)
|
|
344
355
|
)
|
|
345
356
|
|
|
346
|
-
return futures
|
|
357
|
+
return futures
|
|
@@ -9,8 +9,12 @@ from .base import Result, TaskData
|
|
|
9
9
|
|
|
10
10
|
class FlowMessage:
|
|
11
11
|
@staticmethod
|
|
12
|
-
def _print(
|
|
13
|
-
|
|
12
|
+
def _print(*args: object,
|
|
13
|
+
sep: str = " ",
|
|
14
|
+
end: str = "\n",
|
|
15
|
+
file: Optional[IO[str]] = None,
|
|
16
|
+
flush: bool = False) -> None:
|
|
17
|
+
print(*args, sep=sep, end=end, file=file, flush=flush)
|
|
14
18
|
|
|
15
19
|
@staticmethod
|
|
16
20
|
def get_result_text(result: Result) -> str:
|
|
@@ -21,9 +25,9 @@ class FlowMessage:
|
|
|
21
25
|
elif Result.IGNORE == result:
|
|
22
26
|
text = f"{Fore.CYAN}IGNORE"
|
|
23
27
|
elif Result.FAIL == result:
|
|
24
|
-
text = f"{Fore.
|
|
28
|
+
text = f"{Fore.RED}FAIL"
|
|
25
29
|
else:
|
|
26
|
-
text = f"{Fore.
|
|
30
|
+
text = f"{Fore.MAGENTA}UNKNOWN: {result.name}({result.value})"
|
|
27
31
|
|
|
28
32
|
return text
|
|
29
33
|
|
|
@@ -87,4 +91,4 @@ class FlowMessage:
|
|
|
87
91
|
file: Optional[IO[str]] = None,
|
|
88
92
|
flush: bool = False) -> None:
|
|
89
93
|
args = [f"{Fore.RED}{str(a)}" for a in args]
|
|
90
|
-
FlowMessage._print(*args, sep=sep, end=end, file=file, flush=flush)
|
|
94
|
+
FlowMessage._print(*args, sep=sep, end=end, file=file, flush=flush)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "flowweave"
|
|
3
|
-
version = "
|
|
3
|
+
version = "2.0.0"
|
|
4
4
|
description = "YAML-based workflow runner for task orchestration"
|
|
5
5
|
authors = [{name = "syatch"}]
|
|
6
6
|
readme = "README.md"
|
|
@@ -11,6 +11,7 @@ dependencies = [
|
|
|
11
11
|
"prefect>=2,<3",
|
|
12
12
|
"pyyaml>=6,<7"
|
|
13
13
|
]
|
|
14
|
+
license = { text = "MIT" }
|
|
14
15
|
|
|
15
16
|
[tool.setuptools.packages.find]
|
|
16
17
|
include = ["flowweave", "flowweave.*"]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|