flowweave 3.0.0__py3-none-any.whl → 3.0.2__py3-none-any.whl

flowweave/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "3.0.0"
+__version__ = "3.0.2"
 __author__ = "syatch"
 __license__ = "MIT"
 

flowweave/base.py

@@ -1,5 +1,6 @@
 # Standard library
 from enum import IntEnum
+import importlib
 from typing import IO, Optional
 
 from colorama import Fore
@@ -10,6 +11,22 @@ class FlowWeaveResult(IntEnum):
     IGNORE = 2
 
 class TaskData:
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        state["task_class_path"] = (
+            self.task_class.__module__,
+            self.task_class.__name__,
+        )
+        del state["task_class"]
+        return state
+
+    def __setstate__(self, state):
+        module_name, class_name = state["task_class_path"]
+        module = importlib.import_module(module_name)
+        state["task_class"] = getattr(module, class_name)
+        del state["task_class_path"]
+        self.__dict__.update(state)
+
     def __init__(self,
                  name,
                  task_class,

flowweave/flowweave.py

@@ -1,5 +1,4 @@
 # Standard library
-import copy
 from functools import reduce
 import importlib
 from importlib.resources import files
@@ -8,6 +7,7 @@ import itertools
 import json
 import logging
 from pathlib import Path
+import pickle
 import sys
 
 # Third-party
@@ -43,6 +43,10 @@ class StageData():
 class TaskRunner():
     @task
     def start(prev_future, task_data: TaskData):
+        if hasattr(prev_future, "result"):
+            prev_future = prev_future.result()
+
+        return_data = None
         try:
             task_instance = task_data.task_class(prev_future)
         except AttributeError:
@@ -60,7 +64,8 @@ class TaskRunner():
                 if task_data.show_log:
                     FlowWeave._print_log(f"Task option {key} not found: ignore")
         run_task = True
-        if prev_future:
+
+        if prev_future is not None:
             if "pre_success" == task_data.do_only:
                 run_task = True if (FlowWeaveResult.SUCCESS == prev_future.get("result")) else False
             elif "pre_fail" == task_data.do_only:
@@ -80,17 +85,23 @@ class TaskRunner():
             TaskRunner.message_task_ignore(prev_future, task_data)
             task_result = FlowWeaveResult.IGNORE
 
+        try:
+            if return_data is not None:
+                pickle.dumps(return_data)
+        except Exception:
+            raise TypeError(f"Task '{task_data.name}' return_data is not serializable")
+
         return {"name" : task_data.name, "option" : task_data.option, "data" : return_data, "result" : task_result}
 
     def message_task_start(prev_future, task_data: TaskData):
-        if prev_future:
+        if prev_future is not None:
             prev_task_name = prev_future.get("name")
             FlowMessage.task_start_link(prev_task_name, task_data)
         else:
             FlowMessage.task_start(task_data)
 
     def message_task_ignore(prev_future, task_data: TaskData):
-        if prev_future:
+        if prev_future is not None:
             prev_task_name = prev_future.get("name")
             FlowMessage.task_ignore_link(task_data, prev_task_name)
         else:
@@ -98,7 +109,7 @@ class TaskRunner():
 
 class FlowWeave():
     @flow
-    def run(setting_file: str, parallel: bool = False, show_log: bool = False) -> list[str]:
+    def run(setting_file: str, parallel: bool = False, show_log: bool = False) -> list[FlowWeaveResult]:
         if not show_log:
             logging.getLogger("prefect").setLevel(logging.CRITICAL)
 
@@ -177,7 +188,7 @@ class FlowWeave():
         op_source = flow_data.get("op_source")
         op_source_list = op_source if isinstance(op_source, list) else [op_source]
         for source in op_source_list:
-            source_name = f"task/{source}"
+            source_name = f"task.{source}"
             setting_file = f"{source_name.replace('.', '/')}/op_code.yml"
             return_dic |= FlowWeave._get_op_dic_from_setting_file(setting_file, info=info)
 
@@ -262,7 +273,7 @@ class FlowWeave():
         return all_combinations
 
     @task
-    def run_flow(flow_data: dict, global_cmb: dict, op_dic: dict, part: int, all: int, show_log: bool = False) -> list[str]:
+    def run_flow(flow_data: dict, global_cmb: dict, op_dic: dict, part: int, all: int, show_log: bool = False) -> FlowWeaveResult:
         flow_result = FlowWeaveResult.SUCCESS
 
         if show_log:
@@ -271,7 +282,7 @@ class FlowWeave():
             text += "========"
             FlowWeave._print_log(text)
 
-        default_option = flow_data.get("default_option")
+        default_option = flow_data.get("default_option", {})
 
         stage_list = flow_data.get("flow")
         for stage in stage_list:
@@ -288,7 +299,7 @@ class FlowWeave():
             if FlowWeaveResult.FAIL == result:
                 flow_result = FlowWeaveResult.FAIL
 
-            FlowMessage.stage_end(stage, part, all, flow_result)
+            FlowMessage.stage_end(stage, part, all, result)
 
         return flow_result
 
@@ -328,7 +339,7 @@ class FlowWeave():
         return stage_result
 
     def _deep_merge(a: dict, b: dict) -> dict:
-        result = a.copy()
+        result = copy.deepcopy(a)
         for k, v in b.items():
             if k in result and isinstance(result[k], dict) and isinstance(v, dict):
                 result[k] = FlowWeave._deep_merge(result[k], v)
@@ -346,36 +357,42 @@ class FlowWeave():
             raise Exception(f"Cycle detected at task '{task_name}' in {visited}")
         visited.add(task_name)
 
-        task_dic = stage_data.stage_info.get(task_name)
-
-        task_module = stage_data.op_dic.get(task_dic.get('op'))
-        if not task_module:
-            raise Exception(f"module of op '{task_dic.get('op')}' for '{task_name}' not found")
-
-        default_option = stage_data.default_option or {}
-        global_option = stage_data.global_option or {}
-        task_option = FlowWeave._deep_merge_many(default_option, global_option, task_dic.get("option", {}))
-
-        task_data = TaskData(name=task_name,
-                             task_class=task_module,
-                             option=task_option,
-                             stage_name=stage_data.name,
-                             flow_part=stage_data.flow_part,
-                             flow_all=stage_data.flow_all,
-                             do_only=task_dic.get("do_only"),
-                             show_log=show_log)
-        if prev_future is None:
-            future = TaskRunner.start.submit(None, task_data)
-        else:
-            future = TaskRunner.start.submit(prev_future, task_data)
+        try:
+            task_dic = stage_data.stage_info.get(task_name)
+            if task_dic is None:
+                raise KeyError(f"Task '{task_name}' not found in stage '{stage_data.name}'")
+
+            task_module = stage_data.op_dic.get(task_dic.get('op'))
+            if not task_module:
+                raise Exception(f"module of op '{task_dic.get('op')}' for '{task_name}' not found")
+
+            default_option = stage_data.default_option or {}
+            global_option = stage_data.global_option or {}
+            task_option = FlowWeave._deep_merge_many(default_option, global_option, task_dic.get("option", {}))
+
+            task_data = TaskData(name=task_name,
+                                task_class=task_module,
+                                option=task_option,
+                                stage_name=stage_data.name,
+                                flow_part=stage_data.flow_part,
+                                flow_all=stage_data.flow_all,
+                                do_only=task_dic.get("do_only"),
+                                show_log=show_log)
+            if prev_future is None:
+                future = TaskRunner.start.submit(None, task_data)
+            else:
+                future = TaskRunner.start.submit(prev_future, task_data)
 
-        links = task_dic.get("chain", {}).get("next", [])
-        links = links if isinstance(links, list) else [links]
+            links = task_dic.get("chain", {}).get("next", [])
+            links = links if isinstance(links, list) else [links]
 
-        futures = [future]
-        for link in links:
-            futures.extend(
-                FlowWeave._run_task(stage_data, link, future, visited.copy(), show_log)
-            )
+            futures = [future]
+            for link in links:
+                futures.extend(
+                    FlowWeave._run_task(stage_data, link, future, visited.copy(), show_log)
+                )
+
+            return futures
 
-        return futures
+        finally:
+            visited.remove(task_name)

{flowweave-3.0.0.dist-info → flowweave-3.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowweave
-Version: 3.0.0
+Version: 3.0.2
 Summary: YAML-based workflow runner for task orchestration
 Author: syatch
 License: MIT
@@ -16,6 +16,10 @@ Dynamic: license-file
 # FlowWeave
 YAML-based workflow runner for task orchestration
 
+Although this version is more stable, it takes a few seconds to start up.
+
+Therefore, [Lite version](https://github.com/syatch/flowweave-lite) is recommended.
+
 This project is in early development.
 
 ## Installation

flowweave-3.0.2.dist-info/RECORD

@@ -0,0 +1,13 @@
+flowweave/__init__.py,sha256=mO0zI_6wJsS2Gd4hE20UjTNiiNBSRWuwURV02UQwTgo,151
+flowweave/base.py,sha256=Z9nctdS2Uin0MDMRxhQooJP-XuxkgWXBd83Qls0GQjE,3072
+flowweave/cli.py,sha256=xO3hWlMlK9RLniXwQGHHq21IpX9PGki1n3UCR9VAvmQ,2651
+flowweave/flowweave.py,sha256=KhyTD20KOiZLeUX2knggQ5LyNb2awCOrwp19AE5SYRA,15116
+flowweave/message.py,sha256=2tvWW0rEQFjJFwb4Zu7ht2zb6iTRVLIux3jX3P2KzIU,3780
+flowweave/schema/flow.json,sha256=K7EAcH2KqniAPdNJWJdLiOYC77D2SGg50SOU2frC7LU,1810
+flowweave/schema/op_code.json,sha256=C9JeyBvSC6fA8Ss0exk3nsPjTRq_2XpK2gshvAcavbQ,414
+flowweave-3.0.2.dist-info/licenses/LICENSE,sha256=iN7x3Cz45_nCPCn23daVPqs0m_ZWPvTdayE2jg2OSmY,1203
+flowweave-3.0.2.dist-info/METADATA,sha256=IhKSliT_YX1ZEbD2cUaasuD-4hnTqRh17oKwbedhfZ0,744
+flowweave-3.0.2.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+flowweave-3.0.2.dist-info/entry_points.txt,sha256=qV4hkGzamuf-N1eNz-h_tRQ89Wa_py2jIdOSTeGAE1M,49
+flowweave-3.0.2.dist-info/top_level.txt,sha256=7fLs0F6CROwWhmjP9CyKODB25HJohoougiOCCL1YD_Y,10
+flowweave-3.0.2.dist-info/RECORD,,

flowweave-3.0.0.dist-info/RECORD

@@ -1,13 +0,0 @@
-flowweave/__init__.py,sha256=hFKSVXLXkc5VTBybQshM0N9uvhA02lAiCsImk-bp3K8,151
-flowweave/base.py,sha256=o5spY8XxfoP585Qxw02o7xIz9phJUXnVjKXCLUmhsFk,2514
-flowweave/cli.py,sha256=xO3hWlMlK9RLniXwQGHHq21IpX9PGki1n3UCR9VAvmQ,2651
-flowweave/flowweave.py,sha256=5bnVxfrQ2NzW8pssHyz_OLwIRaB_3GcijyXmX3dKWAQ,14413
-flowweave/message.py,sha256=2tvWW0rEQFjJFwb4Zu7ht2zb6iTRVLIux3jX3P2KzIU,3780
-flowweave/schema/flow.json,sha256=K7EAcH2KqniAPdNJWJdLiOYC77D2SGg50SOU2frC7LU,1810
-flowweave/schema/op_code.json,sha256=C9JeyBvSC6fA8Ss0exk3nsPjTRq_2XpK2gshvAcavbQ,414
-flowweave-3.0.0.dist-info/licenses/LICENSE,sha256=iN7x3Cz45_nCPCn23daVPqs0m_ZWPvTdayE2jg2OSmY,1203
-flowweave-3.0.0.dist-info/METADATA,sha256=sSjhmn-nA2LXYMT_ZMO8R6xWyhQDgvmxXZO04W-n5I8,580
-flowweave-3.0.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-flowweave-3.0.0.dist-info/entry_points.txt,sha256=qV4hkGzamuf-N1eNz-h_tRQ89Wa_py2jIdOSTeGAE1M,49
-flowweave-3.0.0.dist-info/top_level.txt,sha256=7fLs0F6CROwWhmjP9CyKODB25HJohoougiOCCL1YD_Y,10
-flowweave-3.0.0.dist-info/RECORD,,