alayaflow 0.1.0__py3-none-any.whl

alayaflow/execution/executors/naive_executor.py

@@ -0,0 +1,121 @@
+import asyncio
+import traceback
+import queue
+import threading
+from typing import Generator, Dict, Optional, Any
+
+from alayaflow.execution.executors.base_executor import BaseExecutor
+from alayaflow.workflow.workflow_manager import WorkflowManager
+from alayaflow.workflow.runnable import BaseRunnableWorkflow, StateGraphRunnableWorkflow
+from alayaflow.common.config import settings
+from alayaflow.execution.langfuse_tracing import get_tracing
+
+_SENTINEL = object()
+
+class NaiveExecutor(BaseExecutor):
+    def __init__(self, workflow_manager: WorkflowManager):
+        self.workflow_manager = workflow_manager
+
+    def _to_jsonable(self, obj: Any) -> Any:
+        """Recursively convert object to JSON-serializable format."""
+        if obj is None or isinstance(obj, (bool, int, float, str)):
+            return obj
+        if isinstance(obj, dict):
+            return {k: self._to_jsonable(v) for k, v in obj.items()}
+        if isinstance(obj, (list, tuple)):
+            return [self._to_jsonable(item) for item in obj]
+        if hasattr(obj, 'model_dump') and callable(obj.model_dump):
+            return obj.model_dump()
+        return str(obj)
+    
+    def _serialize_event(self, event: Dict) -> Dict:
+        # Convert event to JSON-serializable format
+        return self._to_jsonable(event)
+
+    def execute_stream(
+            self, 
+            workflow_id: str, 
+            version: str, 
+            input_data: dict, 
+            user_config: dict
+    ) -> Generator[Dict, None, None]:
+
+        # 1) resolve workflow
+        try:
+            runnable = self.workflow_manager.get_workflow_runnable(workflow_id, version)
+        except ValueError as e:
+            yield {"error": str(e), "workflow_id": workflow_id, "version": version}
+            return
+
+        print(f"NaiveExecutor execute_stream: {workflow_id} {version} {input_data} {user_config}")
+
+        # TODO: Support langflow workflow
+        # Only support StateGraphRunnableWorkflow now. 
+        # As in _produce_events_to_queue, we relay on the "configurable" (StateGraph only) to use langfuse tracing.
+        if not isinstance(runnable, StateGraphRunnableWorkflow):
+            raise ValueError(f"NaiveExecutor only supports StateGraphRunnableWorkflow, but got {type(runnable)}")
+
+        # 3) async -> sync bridge
+        event_queue = queue.Queue(maxsize=1000)
+
+        def run_async_producer():
+            try:
+                asyncio.run(self._produce_events_to_queue(runnable, input_data, user_config, event_queue))
+            except Exception as e:
+                event_queue.put({"error": str(e), "traceback": traceback.format_exc(), "workflow_id": workflow_id, "version": version})
+            finally:
+                event_queue.put(_SENTINEL)
+
+        producer_thread = threading.Thread(target=run_async_producer, daemon=True)
+        producer_thread.start()
+
+        # 4) stream
+        while True:
+            try:
+                item = event_queue.get(timeout=1.0)
+            except queue.Empty:
+                if not producer_thread.is_alive():
+                    saw_sentinel = False
+                    while True:
+                        try:
+                            item = event_queue.get_nowait()
+                            if item is _SENTINEL:
+                                saw_sentinel = True
+                                break
+                            yield self._serialize_event(item)
+                        except queue.Empty:
+                            break
+                    if saw_sentinel:
+                        break
+                    yield {"error": "producer thread exited unexpectedly", "workflow_id": workflow_id, "version": version}
+                    break
+                continue
+            if item is _SENTINEL:
+                break
+            yield self._serialize_event(item)
+
+    
+    async def _produce_events_to_queue(self, runnable: BaseRunnableWorkflow, input_data: dict, user_config: dict, event_queue: queue.Queue):
+        try:
+            # Setup tracing
+            tracing = get_tracing(settings)
+            langfuse_cb = tracing.build_callback()
+            
+            # Merge user_config and tracing config
+            merged_config = {
+                "configurable": user_config
+            }
+            if langfuse_cb:
+                merged_config.update(tracing.build_config(input_data, runnable.info, langfuse_cb))
+            
+            async for chunk in runnable.astream_events(input_data, merged_config):
+                event_queue.put(chunk)  # Put each event immediately (real-time)
+        except Exception as e:
+            # If execution fails, put error event in queue
+            event_queue.put({
+                "error": str(e),
+                "traceback": traceback.format_exc(),
+                "workflow_id": runnable.info.id,
+                "version": runnable.info.version
+            })
+

alayaflow/execution/executors/uv_executor.py

@@ -0,0 +1,125 @@
+import socket
+import os
+import struct
+from typing import Generator, Dict, Optional
+from pathlib import Path
+
+from alayaflow.execution.executors.base_executor import BaseExecutor
+from alayaflow.execution.env_manager import EnvManager
+from alayaflow.workflow.workflow_manager import WorkflowManager
+from alayaflow.common.config import settings
+
+
+class UvExecutor(BaseExecutor):
+    def __init__(
+        self,
+        workflow_manager: WorkflowManager,
+        env_manager: Optional[EnvManager] = None
+    ):
+        self.workflow_manager = workflow_manager
+        self.env_mgr = env_manager if env_manager is not None else EnvManager()
+    
+    def _get_free_port(self):
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            s.bind(('', 0))
+            return s.getsockname()[1]
+    
+    def execute_stream(
+        self,
+        workflow_id: str,
+        version: str,
+        input_data: dict,
+        user_config: dict,
+    ) -> Generator[Dict, None, None]:
+        raise NotImplementedError("uv executor not supported yet")
+
+        # # 1. 确定工作流目录和版本
+        # workflow_info = self.workflow_manager.get_workflow_info(workflow_id, version)
+        # if not workflow_info:
+        #     yield {"error": f"工作流 {workflow_id}_{version} 不存在"}
+        #     return
+        
+        # wf_dir = str(workflow_info.wf_dir)
+        
+        # # 2. 读取工作流依赖
+        # requirements = self.workflow_manager.get_workflow_requirements(workflow_id, version)
+        
+        # # 3. 使用 uv 准备环境 (自动管理 python 版本)
+        # python_exe = self.env_mgr.ensure_env(workflow_id, version, requirements, use_venv=True)
+
+        # 4. 准备 Socket Runner
+        # if settings.is_dev():
+        #     # dev 模式：从项目根目录找
+        #     runner_script_path = os.path.join(settings.alayaflow_root, 'src/alayaflow/execution/workflow_runner.py')
+        # else:
+        #     # dev-uneditable/prod 模式：从安装包中找
+        #     runner_script_path = str(Path(__file__).parent.parent / 'workflow_runner.py')
+        
+        # # 5. 启动 Server Socket
+        # server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        # port = self._get_free_port()
+        # server_socket.bind(('127.0.0.1', port))
+        # server_socket.listen(1)
+        
+        # # 6. 启动子进程
+        # import subprocess
+        # import json
+        # process = subprocess.Popen(
+        #     [python_exe, runner_script_path, str(port), wf_dir],
+        #     stdin=subprocess.PIPE,
+        #     stdout=subprocess.PIPE,
+        #     stderr=subprocess.PIPE,
+        #     text=True
+        # )
+        
+        # process.stdin.write(json.dumps(init_args))
+        # process.stdin.write('\n')
+        # process.stdin.write(json.dumps(config))
+        # process.stdin.write('\n')
+        # process.stdin.write(json.dumps(input_data))
+        # process.stdin.write('\n')
+        # process.stdin.close()
+
+        # # 7. 等待连接
+        # server_socket.settimeout(10.0)  # 首次 uv 下载 python 可能稍慢，增加超时
+        # conn = None
+        # try:
+        #     conn, addr = server_socket.accept()
+            
+        #     # 9. 读取数据流
+        #     while True:
+        #         header = conn.recv(4)
+        #         if not header:
+        #             break
+        #         msg_len = struct.unpack('!I', header)[0]
+                
+        #         chunks = []
+        #         bytes_recd = 0
+        #         while bytes_recd < msg_len:
+        #             chunk = conn.recv(min(msg_len - bytes_recd, 4096))
+        #             if not chunk:
+        #                 raise RuntimeError("Socket connection broken")
+        #             chunks.append(chunk)
+        #             bytes_recd += len(chunk)
+                
+        #         full_msg = b''.join(chunks)
+        #         full_msg_json = json.loads(full_msg.decode('utf-8'))
+        #         yield full_msg_json
+
+        # except socket.timeout:
+        #     yield {"error": "Runner connection timeout (Maybe UV is downloading python?)"}
+        # except Exception as e:
+        #     yield {"error": f"Socket Error: {e}"}
+        # finally:
+        #     if conn:
+        #         conn.close()
+        #     server_socket.close()
+            
+        #     logs = process.stdout.read()
+        #     if logs:
+        #         print(f"--- [Clean Runner Logs] ---\n{logs}---------------------------")
+            
+        #     stderr = process.stderr.read()
+        #     if stderr:
+        #         print(f"[Stderr] {stderr}")
+

alayaflow/execution/executors/worker_executor.py

@@ -0,0 +1,12 @@
+from typing import Generator, Dict
+
+from alayaflow.execution.executors.base_executor import BaseExecutor
+from alayaflow.workflow.workflow_manager import WorkflowManager
+
+class WorkerExecutor(BaseExecutor):
+    def __init__(self, workflow_manager: WorkflowManager):
+        self.workflow_manager = workflow_manager
+    
+    def execute_stream(self, workflow_id: str, version: str, input_data: dict, user_config: dict) -> Generator[Dict, None, None]:
+        raise NotImplementedError("worker executor not supported yet")
+

alayaflow/execution/langfuse_tracing.py

@@ -0,0 +1,104 @@
+from typing import Any, Optional
+
+from alayaflow.workflow.workflow_info import WorkflowInfo
+
+
+class LangfuseTracing:
+    def __init__(self, settings: Any) -> None:
+        self._settings = settings
+        self._client_inited = False
+
+    def _normalize_value(self, value: Optional[str]) -> Optional[str]:
+        return value.strip() if isinstance(value, str) and value.strip() else None
+
+    def _merge_tags(self, wf_tags: Optional[list], input_tags: Optional[list]) -> list:
+        tags = []
+        if isinstance(wf_tags, list):
+            tags.extend(wf_tags)
+        if isinstance(input_tags, list):
+            tags.extend(input_tags)
+        if tags:
+            # 去重并保持顺序
+            seen = set()
+            tags = [t for t in tags if not (t in seen or seen.add(t))]
+        return tags
+
+    def _init_langfuse_client(self, public_key: str, secret_key: str, host: Optional[str]) -> None:
+        if self._client_inited:
+            return
+        from langfuse import Langfuse
+        if host is None:
+            Langfuse(public_key=public_key, secret_key=secret_key)
+        else:
+            Langfuse(public_key=public_key, secret_key=secret_key, host=host)
+        self._client_inited = True
+
+    def _load_langfuse_handler_class(self) -> Optional[Any]:
+        try:
+            from langfuse.langchain import CallbackHandler
+            return CallbackHandler
+        except Exception:
+            try:
+                from langfuse.callback import CallbackHandler
+                return CallbackHandler
+            except Exception:
+                return None
+
+    def build_callback(self) -> Optional[Any]:
+        if not self._settings.langfuse_enabled:
+            return None
+
+        public_key = self._normalize_value(self._settings.langfuse_public_key)
+        secret_key = self._normalize_value(self._settings.langfuse_secret_key)
+        host = self._normalize_value(self._settings.langfuse_url)
+        if not (public_key and secret_key):
+            return None
+
+        handler_cls = self._load_langfuse_handler_class()
+        if handler_cls is None:
+            print("[Langfuse] SDK 未安装，跳过追踪。请在工作流 requirements.txt 中添加 langfuse。")
+            return None
+
+        try:
+            self._init_langfuse_client(public_key, secret_key, host)
+            return handler_cls()
+        except Exception as e:
+            print(f"[Langfuse] 初始化失败，跳过追踪: {e}")
+            return None
+
+    def build_config(self, input_data: dict, wf_info: Optional[WorkflowInfo], cb: Any) -> dict:
+        metadata = {
+            "workflow_id": getattr(wf_info, "id", None) if wf_info is not None else None,
+            "workflow_version": getattr(wf_info, "version", None) if wf_info is not None else None,
+            "workflow_name": getattr(wf_info, "name", None) if wf_info is not None else None,
+        }
+        input_metadata = input_data.get("metadata")
+        if isinstance(input_metadata, dict):
+            metadata.update(input_metadata)
+
+        wf_tags = getattr(wf_info, "tags", None) if wf_info is not None else None
+        tags = self._merge_tags(wf_tags, input_data.get("tags"))
+
+        metadata.update({
+            "langfuse_user_id": input_data.get("user_id"),
+            "langfuse_session_id": input_data.get("session_id"),
+            "langfuse_tags": tags or None,
+        })
+        metadata = {k: v for k, v in metadata.items() if v is not None}
+
+        return {
+            "callbacks": [cb],
+            "run_name": getattr(wf_info, "name", None) or getattr(wf_info, "id", None),
+            "metadata": metadata,
+        }
+
+
+_GLOBAL_TRACING: Optional[LangfuseTracing] = None
+
+
+def get_tracing(settings: Any) -> LangfuseTracing:
+    global _GLOBAL_TRACING
+    if _GLOBAL_TRACING is None:
+        _GLOBAL_TRACING = LangfuseTracing(settings)
+    return _GLOBAL_TRACING
+

alayaflow/execution/workflow_runner.py

@@ -0,0 +1,98 @@
+import sys
+import json
+import importlib.util
+import socket
+import struct
+import traceback
+import asyncio
+from pathlib import Path
+
+# 尝试导入 langchain 的序列化工具 (如果环境中有)
+try:
+    from langchain_core.messages import BaseMessage
+    from pydantic import BaseModel
+except ImportError as e:
+    raise ImportError("请安装 langchain_core 和 pydantic 库以支持 LangChain/LangGraph 序列化") from e
+
+try:
+    import alayaflow
+except ImportError as e:
+    raise ImportError("alayaflow没有在worker内安装") from e
+
+try:
+    from alayaflow.workflow.workflow_loader import WorkflowLoader
+    from alayaflow.common.config import settings
+    from alayaflow.execution.langfuse_tracing import get_tracing
+except ImportError as e:
+    raise e
+
+port = int(sys.argv[1])
+wf_dir = sys.argv[2]
+
+s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+try:
+    s.connect(('127.0.0.1', port))
+except ConnectionRefusedError:
+    sys.exit(1)
+
+def default_encoder(obj):
+    if hasattr(obj, 'model_dump') and callable(obj.model_dump):
+        return obj.model_dump()
+    return str(obj)
+
+def send_chunk(data_dict):
+    try:
+        json_bytes = json.dumps(data_dict, default=default_encoder).encode('utf-8')
+        header = struct.pack('!I', len(json_bytes))
+        s.sendall(header + json_bytes)
+    except Exception as e:
+        err_msg = json.dumps({"error": f"Serialization Error: {str(e)}"}).encode('utf-8')
+        s.sendall(struct.pack('!I', len(err_msg)) + err_msg)
+
+async def run():
+    try:
+        input_str = sys.stdin.read()
+        lines = input_str.strip().split('\n')
+        init_args = json.loads(lines[0]) if len(lines) > 0 and lines[0].strip() else {}
+        config = json.loads(lines[1]) if len(lines) > 1 and lines[1].strip() else {}
+        input_data = json.loads(lines[2]) if len(lines) > 2 and lines[2].strip() else {}
+
+        print("Input data for workflow:", input_data)
+
+        # 加载工作流元信息和创建器
+        # wf_dir 格式: {workflows_dir}/{workflow_id}/{version}
+        wf_dir_path = Path(wf_dir)
+        workflows_dir = str(wf_dir_path.parent.parent)  # 工作流根目录
+        workflow_id = wf_dir_path.parent.name  # 工作流ID
+        version = wf_dir_path.name  # 版本号
+        
+        loader = WorkflowLoader(workflows_dir, workflow_id, version)
+        result = loader.load_workflow()
+        wf_info = result.info
+        creator = result.creator
+        graph = creator(init_args)
+
+        tracing = get_tracing(settings)
+        langfuse_cb = tracing.build_callback()
+        
+        # 合并用户提供的 config 和 tracing config
+        merged_config = dict(config) if config else {}
+        if langfuse_cb:
+            merged_config.update(tracing.build_config(input_data, wf_info, langfuse_cb))
+        
+        async for chunk in graph.astream_events(input_data, config=merged_config, version="v2"):
+            send_chunk(chunk)
+
+    except Exception as e:
+        error_info = {
+            "error": str(e),
+            "traceback": traceback.format_exc()
+        }
+        send_chunk(error_info)
+        traceback.print_exc()
+        print(f"[Runner Critical] {{error_info}}", file=sys.stderr)
+
+    finally:
+        s.close()
+
+asyncio.run(run())

alayaflow/utils/singleton.py

@@ -0,0 +1,14 @@
+import threading
+
+class SingletonMeta(type):
+    _instances = {}
+    _lock = threading.RLock()
+
+    def __call__(cls, *args, **kwargs):
+        # Double-Checked Locking
+        if cls not in cls._instances:
+            with cls._lock:
+                if cls not in cls._instances:
+                    instance = super().__call__(*args, **kwargs)
+                    cls._instances[cls] = instance
+        return cls._instances[cls]

alayaflow/workflow/__init__.py

@@ -0,0 +1,6 @@
+from alayaflow.workflow.workflow_info import WorkflowInfo
+from alayaflow.workflow.workflow_loader import WorkflowLoader
+from alayaflow.workflow.workflow_manager import WorkflowManager
+
+__all__ = ["WorkflowInfo", "WorkflowLoader", "WorkflowManager"]
+

alayaflow/workflow/runnable/__init__.py

@@ -0,0 +1,7 @@
+from alayaflow.workflow.runnable.base_runnable_workflow import BaseRunnableWorkflow
+from alayaflow.workflow.runnable.state_graph_runnable_workflow import StateGraphRunnableWorkflow
+
+__all__ = [
+    "BaseRunnableWorkflow",
+    "StateGraphRunnableWorkflow"
+]

alayaflow/workflow/runnable/base_runnable_workflow.py

@@ -0,0 +1,19 @@
+from abc import ABC
+from typing import Dict, Generator
+
+from alayaflow.workflow.workflow_info import WorkflowInfo
+
+class BaseRunnableWorkflow(ABC):
+    def __init__(self, info: WorkflowInfo) -> None:
+        self._info = info
+
+    @property
+    def info(self) -> WorkflowInfo:
+        return self._info
+
+    def invoke(self, input_data: dict, user_config: dict) -> dict:
+        raise NotImplementedError("invoke method must be implemented in derived classes")
+    
+    def astream_events(self, input_data: dict, user_config: dict) -> Generator[Dict, None, None]:
+        raise NotImplementedError("invoke method must be implemented in derived classes")
+

alayaflow/workflow/runnable/state_graph_runnable_workflow.py

@@ -0,0 +1,23 @@
+from functools import cached_property
+from typing import Dict, Generator
+
+from langgraph.graph.state import CompiledStateGraph
+
+from alayaflow.workflow.runnable.base_runnable_workflow import BaseRunnableWorkflow
+from alayaflow.workflow.workflow_info import WorkflowInfo
+
+class StateGraphRunnableWorkflow(BaseRunnableWorkflow):
+    def __init__(self, info: WorkflowInfo, graph: CompiledStateGraph) -> None:
+        super().__init__(info)
+        self._graph = graph
+
+    def invoke(self, input_data: dict, user_config: dict) -> dict:
+        return self._graph.invoke(input_data, {
+            "configurable": user_config
+        })
+    
+    def astream_events(self, input_data: dict, user_config: dict) -> Generator[Dict, None, None]:
+        return self._graph.astream_events(input_data, {
+            "configurable": user_config
+        })
+

alayaflow/workflow/workflow_info.py

@@ -0,0 +1,50 @@
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Callable, Any, NamedTuple
+
+
+class WorkflowKey(NamedTuple):
+    id: str
+    version: str
+
+
+@dataclass
+class WorkflowInfo:
+    """工作流元信息"""
+    id: str
+    name: str
+    description: str
+    version: str
+    tags: List[str]
+    entry_file: str
+    entry_point: str
+    wf_dir: Path
+
+    @property
+    def key(self) -> WorkflowKey:
+        return WorkflowKey(self.id, self.version)
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "name": self.name,
+            "description": self.description,
+            "version": self.version,
+            "tags": self.tags,
+            "entry_file": self.entry_file,
+            "entry_point": self.entry_point,
+            "path": str(self.wf_dir),
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "WorkflowInfo":
+        return cls(
+            id=data["id"],
+            name=data["name"],
+            description=data["description"],
+            version=data["version"],
+            tags=data["tags"],
+            entry_file=data["entry_file"],
+            entry_point=data["entry_point"],
+            wf_dir=Path(data.get("path", data.get("wf_dir", ""))),
+        )