flowweave 2.0.1__tar.gz → 3.0.2__tar.gz

{flowweave-2.0.1 → flowweave-3.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowweave
-Version: 2.0.1
+Version: 3.0.2
 Summary: YAML-based workflow runner for task orchestration
 Author: syatch
 License: MIT
@@ -16,6 +16,10 @@ Dynamic: license-file
 # FlowWeave
 YAML-based workflow runner for task orchestration
 
+Although this version is more stable, it takes a few seconds to start up.
+
+Therefore, [Lite version](https://github.com/syatch/flowweave-lite) is recommended.
+
 This project is in early development.
 
 ## Installation

flowweave-3.0.2/README.md

@@ -0,0 +1,16 @@
+# FlowWeave
+YAML-based workflow runner for task orchestration
+
+Although this version is more stable, it takes a few seconds to start up.
+
+Therefore, [Lite version](https://github.com/syatch/flowweave-lite) is recommended.
+
+This project is in early development.
+
+## Installation
+
+Install FlowWeave using pip:
+
+```bash
+pip install flowweave
+```

flowweave-3.0.2/flowweave/__init__.py

@@ -0,0 +1,6 @@
+__version__ = "3.0.2"
+__author__ = "syatch"
+__license__ = "MIT"
+
+from .flowweave import FlowWeave
+from .base import FlowWeaveResult, FlowWeaveTask

{flowweave-2.0.1 → flowweave-3.0.2}/flowweave/base.py

@@ -1,15 +1,32 @@
 # Standard library
 from enum import IntEnum
+import importlib
 from typing import IO, Optional
 
 from colorama import Fore
 
-class Result(IntEnum):
+class FlowWeaveResult(IntEnum):
     FAIL = 0
     SUCCESS = 1
     IGNORE = 2
 
 class TaskData:
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        state["task_class_path"] = (
+            self.task_class.__module__,
+            self.task_class.__name__,
+        )
+        del state["task_class"]
+        return state
+
+    def __setstate__(self, state):
+        module_name, class_name = state["task_class_path"]
+        module = importlib.import_module(module_name)
+        state["task_class"] = getattr(module, class_name)
+        del state["task_class_path"]
+        self.__dict__.update(state)
+
     def __init__(self,
                  name,
                  task_class,
@@ -30,17 +47,17 @@ class TaskData:
         self.do_only = do_only
         self.show_log = show_log
 
-class FlowWeaveTaskRunner:
+class FlowWeaveTask:
     def __init__(self, prev_future):
         self.prev_future = prev_future
         self.return_data = None
 
-    def __call__(self) -> Result:
-        result = self.run()
-        return result
+    def __call__(self):
+        result, return_data = self.run()
+        return result, return_data
 
-    def run(self) -> Result:
-        return Result.SUCCESS
+    def run(self):
+        return FlowWeaveResult.SUCCESS, self.return_data
 
     def set_task_data(self, task_data: TaskData) -> None:
         self.task_data = task_data

{flowweave-2.0.1 → flowweave-3.0.2}/flowweave/cli.py

@@ -8,7 +8,7 @@ import colorama
 
 # Local application / relative imports
 from .flowweave import FlowWeave
-from .base import Result
+from .base import FlowWeaveResult
 
 def get_setting_path(args):
     setting_path = None
@@ -59,7 +59,7 @@ def build_parser() -> argparse.ArgumentParser:
     return parser
 
 def main() -> None:
-    result = Result.SUCCESS
+    result = FlowWeaveResult.SUCCESS
 
     colorama.init(autoreset=True)
 
@@ -72,7 +72,7 @@ def main() -> None:
         if not setting_path:
             parser.error("run requires flow_file")
         results = FlowWeave.run(setting_file=setting_path, parallel=args.parallel, show_log = args.log)
-        result = all(x == Result.SUCCESS for x in results)
+        result = all(x == FlowWeaveResult.SUCCESS for x in results)
     elif args.command == "info":
         if args.flow_file:
             show_flow_op(setting_path, args.flow_file, info=True)

{flowweave-2.0.1 → flowweave-3.0.2}/flowweave/flowweave.py

@@ -1,11 +1,13 @@
 # Standard library
-import copy
+from functools import reduce
 import importlib
 from importlib.resources import files
+import inspect
 import itertools
 import json
 import logging
 from pathlib import Path
+import pickle
 import sys
 
 # Third-party
@@ -14,7 +16,7 @@ import yaml
 from prefect import flow, task, get_run_logger
 
 # Local application / relative imports
-from .base import Result, TaskData
+from .base import FlowWeaveResult, TaskData, FlowWeaveTask
 from .message import FlowMessage
 
 class StageData():
@@ -38,15 +40,17 @@ class StageData():
         text += "==========="
         return text
 
-class FlowWeaveTask():
-    task_class = None
-
+class TaskRunner():
     @task
     def start(prev_future, task_data: TaskData):
+        if hasattr(prev_future, "result"):
+            prev_future = prev_future.result()
+
+        return_data = None
         try:
-            task_instance = task_data.task_class.runner(prev_future)
+            task_instance = task_data.task_class(prev_future)
         except AttributeError:
-            raise TypeError(f"{task_data.task_class} must define runner")
+            raise TypeError(f"Failed to get instance of '{task_data.task_class}'")
 
         # set task member variables
         setattr(task_instance, "task_data", task_data)
@@ -60,37 +64,44 @@ class FlowWeaveTask():
                 if task_data.show_log:
                     FlowWeave._print_log(f"Task option {key} not found: ignore")
         run_task = True
-        if prev_future:
+
+        if prev_future is not None:
             if "pre_success" == task_data.do_only:
-                run_task = True if (Result.SUCCESS == prev_future.get("result")) else False
+                run_task = True if (FlowWeaveResult.SUCCESS == prev_future.get("result")) else False
             elif "pre_fail" == task_data.do_only:
-                run_task = True if (Result.FAIL == prev_future.get("result")) else False
+                run_task = True if (FlowWeaveResult.FAIL == prev_future.get("result")) else False
 
         if run_task:
-            FlowWeaveTask.message_task_start(prev_future, task_data)
+            TaskRunner.message_task_start(prev_future, task_data)
 
             try:
                 task_result, return_data = task_instance()
             except Exception as e:
                 FlowMessage.error(e)
-                task_result = Result.FAIL
+                task_result = FlowWeaveResult.FAIL
 
             FlowMessage.task_end(task_data, task_result)
         else:
-            FlowWeaveTask.message_task_ignore(prev_future, task_data)
-            task_result = Result.IGNORE
+            TaskRunner.message_task_ignore(prev_future, task_data)
+            task_result = FlowWeaveResult.IGNORE
+
+        try:
+            if return_data is not None:
+                pickle.dumps(return_data)
+        except Exception:
+            raise TypeError(f"Task '{task_data.name}' return_data is not serializable")
 
         return {"name" : task_data.name, "option" : task_data.option, "data" : return_data, "result" : task_result}
 
     def message_task_start(prev_future, task_data: TaskData):
-        if prev_future:
+        if prev_future is not None:
             prev_task_name = prev_future.get("name")
             FlowMessage.task_start_link(prev_task_name, task_data)
         else:
             FlowMessage.task_start(task_data)
 
     def message_task_ignore(prev_future, task_data: TaskData):
-        if prev_future:
+        if prev_future is not None:
             prev_task_name = prev_future.get("name")
             FlowMessage.task_ignore_link(task_data, prev_task_name)
         else:
@@ -98,7 +109,7 @@ class FlowWeaveTask():
 
 class FlowWeave():
     @flow
-    def run(setting_file: str, parallel: bool = False, show_log: bool = False) -> list[str]:
+    def run(setting_file: str, parallel: bool = False, show_log: bool = False) -> list[FlowWeaveResult]:
         if not show_log:
             logging.getLogger("prefect").setLevel(logging.CRITICAL)
 
@@ -177,7 +188,7 @@ class FlowWeave():
         op_source = flow_data.get("op_source")
         op_source_list = op_source if isinstance(op_source, list) else [op_source]
         for source in op_source_list:
-            source_name = f"task/{source}"
+            source_name = f"task.{source}"
             setting_file = f"{source_name.replace('.', '/')}/op_code.yml"
             return_dic |= FlowWeave._get_op_dic_from_setting_file(setting_file, info=info)
 
@@ -207,27 +218,43 @@ class FlowWeave():
         op_dic = setting.get("op", {})
         for op, op_info in op_dic.items():
             script_name = op_info.get('script')
-            op_class = FlowWeave._get_op_class(source_name, script_name, info)
+            op_class = FlowWeave._get_op_class(source_name, script_name, FlowWeaveTask)
 
             return_dic[str(op)] = op_class
 
         return return_dic
 
-    def _get_op_class(source_name: str, script_name: str, info: bool = False):
+    def _get_op_class(source_name: str, script_name: str, base_class):
         module_name = f"{source_name}.{script_name}"
+
         try:
             module = importlib.import_module(module_name)
         except Exception as e:
             raise RuntimeError(f"Failed to import {module_name}: {e}")
 
-        if not hasattr(module, "Task"):
-            raise RuntimeError(f"'Task' class not found in {module_name}")
+        candidates = []
 
-        return_module = module.Task
-        if info:
-            return_module = module.Task.runner
+        for _, obj in inspect.getmembers(module, inspect.isclass):
+            if obj.__module__ != module.__name__:
+                continue
+
+            if issubclass(obj, base_class) and obj is not base_class:
+                candidates.append(obj)
+
+        if len(candidates) == 0:
+            raise RuntimeError(
+                f"No subclass of {base_class.__name__} found in {module_name}"
+            )
+
+        if len(candidates) > 1:
+            names = ", ".join(c.__name__ for c in candidates)
+            raise RuntimeError(
+                f"Multiple subclasses of {base_class.__name__} found in {module_name}: {names}"
+            )
 
-        return return_module
+        cls = candidates[0]
+
+        return cls
 
     def _get_global_option_comb(global_option: dict) -> list:
         keys = list(global_option.keys())
@@ -246,8 +273,8 @@ class FlowWeave():
         return all_combinations
 
     @task
-    def run_flow(flow_data: dict, global_cmb: dict, op_dic: dict, part: int, all: int, show_log: bool = False) -> list[str]:
-        flow_result = Result.SUCCESS
+    def run_flow(flow_data: dict, global_cmb: dict, op_dic: dict, part: int, all: int, show_log: bool = False) -> FlowWeaveResult:
+        flow_result = FlowWeaveResult.SUCCESS
 
         if show_log:
             text = "= Flow =\n"
@@ -255,7 +282,7 @@ class FlowWeave():
             text += "========"
             FlowWeave._print_log(text)
 
-        default_option = flow_data.get("default_option")
+        default_option = flow_data.get("default_option", {})
 
         stage_list = flow_data.get("flow")
         for stage in stage_list:
@@ -269,10 +296,10 @@ class FlowWeave():
                 FlowWeave._print_log(str(stage_data))
 
             result = FlowWeave._run_stage(stage_data, show_log)
-            if Result.FAIL == result:
-                flow_result = Result.FAIL
+            if FlowWeaveResult.FAIL == result:
+                flow_result = FlowWeaveResult.FAIL
 
-            FlowMessage.stage_end(stage, part, all, flow_result)
+            FlowMessage.stage_end(stage, part, all, result)
 
         return flow_result
 
@@ -293,7 +320,7 @@ class FlowWeave():
             logger.info(f"{text}")
 
     def _run_stage(stage_data: StageData, show_log: bool = False):
-        stage_result = Result.SUCCESS
+        stage_result = FlowWeaveResult.SUCCESS
 
         all_futures = []
 
@@ -306,11 +333,23 @@ class FlowWeave():
 
         for f in all_futures:
             result = f.result()
-            if Result.FAIL == result.get("result"):
-                stage_result = Result.FAIL
+            if FlowWeaveResult.FAIL == result.get("result"):
+                stage_result = FlowWeaveResult.FAIL
 
         return stage_result
 
+    def _deep_merge(a: dict, b: dict) -> dict:
+        result = copy.deepcopy(a)
+        for k, v in b.items():
+            if k in result and isinstance(result[k], dict) and isinstance(v, dict):
+                result[k] = FlowWeave._deep_merge(result[k], v)
+            else:
+                result[k] = v
+        return result
+
+    def _deep_merge_many(*dicts):
+        return reduce(FlowWeave._deep_merge, dicts)
+
     def _run_task(stage_data: dict, task_name: str, prev_future = None, visited = None, show_log: bool = False):
         if visited is None:
             visited = set()
@@ -318,40 +357,42 @@ class FlowWeave():
             raise Exception(f"Cycle detected at task '{task_name}' in {visited}")
         visited.add(task_name)
 
-        task_dic = stage_data.stage_info.get(task_name)
-
-        task_module = stage_data.op_dic.get(task_dic.get('op'))
-        if not task_module:
-            raise Exception(f"module of op '{task_dic.get('op')}' for '{task_name}' not found")
-
-        default_option = stage_data.default_option or {}
-        global_option = stage_data.global_option or {}
-        task_option = copy.deepcopy(
-            default_option
-            | global_option
-            | task_dic.get("option", {})
-        )
-
-        task_data = TaskData(name=task_name,
-                             task_class=task_module,
-                             option=task_option,
-                             stage_name=stage_data.name,
-                             flow_part=stage_data.flow_part,
-                             flow_all=stage_data.flow_all,
-                             do_only=task_dic.get("do_only"),
-                             show_log=show_log)
-        if prev_future is None:
-            future = task_module.start.submit(None, task_data)
-        else:
-            future = task_module.start.submit(prev_future, task_data)
+        try:
+            task_dic = stage_data.stage_info.get(task_name)
+            if task_dic is None:
+                raise KeyError(f"Task '{task_name}' not found in stage '{stage_data.name}'")
+
+            task_module = stage_data.op_dic.get(task_dic.get('op'))
+            if not task_module:
+                raise Exception(f"module of op '{task_dic.get('op')}' for '{task_name}' not found")
+
+            default_option = stage_data.default_option or {}
+            global_option = stage_data.global_option or {}
+            task_option = FlowWeave._deep_merge_many(default_option, global_option, task_dic.get("option", {}))
+
+            task_data = TaskData(name=task_name,
+                                task_class=task_module,
+                                option=task_option,
+                                stage_name=stage_data.name,
+                                flow_part=stage_data.flow_part,
+                                flow_all=stage_data.flow_all,
+                                do_only=task_dic.get("do_only"),
+                                show_log=show_log)
+            if prev_future is None:
+                future = TaskRunner.start.submit(None, task_data)
+            else:
+                future = TaskRunner.start.submit(prev_future, task_data)
 
-        links = task_dic.get("chain", {}).get("next", [])
-        links = links if isinstance(links, list) else [links]
+            links = task_dic.get("chain", {}).get("next", [])
+            links = links if isinstance(links, list) else [links]
 
-        futures = [future]
-        for link in links:
-            futures.extend(
-                FlowWeave._run_task(stage_data, link, future, visited.copy(), show_log)
-            )
+            futures = [future]
+            for link in links:
+                futures.extend(
+                    FlowWeave._run_task(stage_data, link, future, visited.copy(), show_log)
+                )
+
+            return futures
 
-        return futures
+        finally:
+            visited.remove(task_name)

{flowweave-2.0.1 → flowweave-3.0.2}/flowweave/message.py

@@ -5,7 +5,7 @@ from typing import IO, Optional
 from colorama import Fore
 
 # Local application / relative imports
-from .base import Result, TaskData
+from .base import FlowWeaveResult, TaskData
 
 class FlowMessage:
     @staticmethod
@@ -17,14 +17,14 @@ class FlowMessage:
         print(*args, sep=sep, end=end, file=file, flush=flush)
 
     @staticmethod
-    def get_result_text(result: Result) -> str:
+    def get_result_text(result: FlowWeaveResult) -> str:
         text = ""
 
-        if Result.SUCCESS == result:
+        if FlowWeaveResult.SUCCESS == result:
             text = f"{Fore.GREEN}SUCCESS"
-        elif Result.IGNORE == result:
+        elif FlowWeaveResult.IGNORE == result:
             text = f"{Fore.CYAN}IGNORE"
-        elif Result.FAIL == result:
+        elif FlowWeaveResult.FAIL == result:
             text = f"{Fore.RED}FAIL"
         else:
             text = f"{Fore.MAGENTA}UNKNOWN: {result.name}({result.value})"
@@ -42,7 +42,7 @@ class FlowMessage:
         FlowMessage._print(text)
 
     @staticmethod
-    def flow_end(part: int, all: int, result: Result) -> None:
+    def flow_end(part: int, all: int, result: FlowWeaveResult) -> None:
         result_text = FlowMessage.get_result_text(result)
         text = f"{Fore.YELLOW}[Flow {part} / {all}] Finish - {result_text}"
         FlowMessage._print(text)
@@ -53,7 +53,7 @@ class FlowMessage:
         FlowMessage._print(text)
 
     @staticmethod
-    def stage_end(stage: str, part: int, all: int, result: Result) -> None:
+    def stage_end(stage: str, part: int, all: int, result: FlowWeaveResult) -> None:
         result_text = FlowMessage.get_result_text(result)
         text = f"{Fore.MAGENTA}[Flow {part} / {all}] Finish Stage {stage} - {result_text}"
         FlowMessage._print(text)
@@ -79,7 +79,7 @@ class FlowMessage:
         FlowMessage._print(text)
 
     @staticmethod
-    def task_end(task_data: TaskData, result: Result) -> None:
+    def task_end(task_data: TaskData, result: FlowWeaveResult) -> None:
         result_text = FlowMessage.get_result_text(result)
         text = f"{Fore.CYAN}[Flow {task_data.flow_part} / {task_data.flow_all}] Finish Task {task_data.stage_name}/{task_data.name} - {result_text}"
         FlowMessage._print(text)

{flowweave-2.0.1 → flowweave-3.0.2}/flowweave.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowweave
-Version: 2.0.1
+Version: 3.0.2
 Summary: YAML-based workflow runner for task orchestration
 Author: syatch
 License: MIT
@@ -16,6 +16,10 @@ Dynamic: license-file
 # FlowWeave
 YAML-based workflow runner for task orchestration
 
+Although this version is more stable, it takes a few seconds to start up.
+
+Therefore, [Lite version](https://github.com/syatch/flowweave-lite) is recommended.
+
 This project is in early development.
 
 ## Installation

{flowweave-2.0.1 → flowweave-3.0.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "flowweave"
-version = "2.0.1"
+version = "3.0.2"
 description = "YAML-based workflow runner for task orchestration"
 authors = [{name = "syatch"}]
 readme = "README.md"

flowweave-2.0.1/README.md

@@ -1,12 +0,0 @@
-# FlowWeave
-YAML-based workflow runner for task orchestration
-
-This project is in early development.
-
-## Installation
-
-Install FlowWeave using pip:
-
-```bash
-pip install flowweave
-```

flowweave-2.0.1/flowweave/__init__.py

@@ -1,6 +0,0 @@
-__version__ = "2.0.1"
-__author__ = "syatch"
-__license__ = "MIT"
-
-from .flowweave import FlowWeave, FlowWeaveTask
-from .base import Result, TaskData, FlowWeaveTaskRunner