service-forge 0.1.11__py3-none-any.whl

service_forge/sft/util/yaml_utils.py

@@ -0,0 +1,57 @@
+import yaml
+from pathlib import Path
+from typing import Any, Dict
+
+
+def extract_yaml_content_without_comments(yaml_file: Path) -> str:
+    """
+    读取 YAML 文件，去除注释，返回格式化的 YAML 字符串
+
+    Args:
+        yaml_file: YAML 文件路径
+
+    Returns:
+        去除注释后的 YAML 字符串
+    """
+    with open(yaml_file, 'r', encoding='utf-8') as f:
+        content = f.read()
+
+    # 移除注释行（以 # 开头的行，但不包含字符串中的 #）
+    lines = content.split('\n')
+    filtered_lines = []
+
+    for line in lines:
+        stripped = line.strip()
+        # 跳过空行和纯注释行
+        if not stripped or stripped.startswith('#'):
+            continue
+        filtered_lines.append(line)
+
+    # 重新组合并解析为 Python 对象，然后重新序列化
+    yaml_content = '\n'.join(filtered_lines)
+
+    try:
+        # 使用 yaml.safe_load 解析，然后重新序列化确保格式正确
+        data = yaml.safe_load(yaml_content)
+        if data is None:
+            return ""
+        return yaml.dump(data, default_flow_style=False, allow_unicode=True, sort_keys=False)
+    except yaml.YAMLError:
+        # 如果解析失败，返回原始过滤后的内容
+        return yaml_content
+
+
+def load_sf_metadata_as_string(metadata_file: Path) -> str:
+    """
+    加载 sf-meta.yaml 文件，去除注释后返回字符串格式
+
+    Args:
+        metadata_file: sf-meta.yaml 文件路径
+
+    Returns:
+        去除注释后的 YAML 内容字符串
+    """
+    if not metadata_file.exists():
+        raise FileNotFoundError(f"Metadata file not found: {metadata_file}")
+
+    return extract_yaml_content_without_comments(metadata_file)

service_forge/utils/default_type_converter.py

@@ -0,0 +1,12 @@
+from ..utils.type_converter import TypeConverter
+from ..workflow.workflow import Workflow
+from ..api.http_api import fastapi_app
+from ..api.kafka_api import KafkaApp, kafka_app
+from fastapi import FastAPI
+from ..workflow.workflow_type import WorkflowType, workflow_type_register
+
+type_converter = TypeConverter()
+type_converter.register(str, Workflow, lambda s, node: node.sub_workflows.get_workflow(s))
+type_converter.register(str, FastAPI, lambda s, node: fastapi_app)
+type_converter.register(str, KafkaApp, lambda s, node: kafka_app)
+type_converter.register(str, type, lambda s, node: workflow_type_register.items[s].type)

service_forge/utils/register.py

@@ -0,0 +1,39 @@
+from typing import TypeVar, Generic, Any
+from loguru import logger
+
+T = TypeVar('T')
+
+class Register(Generic[T]):
+  def __init__(self) -> None:
+    self.items:dict[str, T] = {}
+
+  def register(
+      self,
+      name: str,
+      item: T,
+      show_info_log: bool = False,
+  ) -> None:
+    if name not in self.items:
+      self.items[name] = item
+      if show_info_log:
+        logger.info(f'Register {name}.')
+    else:
+      logger.warning(f'{name} has been registered.')
+
+  def instance(
+      self,
+      name: str,
+      kwargs: dict[str, Any] = {},
+      ignore_keys: list[str] = []
+  ) -> None:
+    for key in ignore_keys:
+      try:
+        kwargs.pop(key)
+      except:
+        pass
+    if name not in self.items:
+      logger.error(f'{name} has not been registered.')
+    return self.items[name](**kwargs)
+  
+  def __len__(self) -> int:
+    return len(self.items)

service_forge/utils/type_converter.py

@@ -0,0 +1,99 @@
+from typing import Any, Callable, Type, Dict, Tuple, Set, List
+from collections import deque
+import inspect
+from typing_extensions import get_origin, get_args
+
+def is_type(value, dst_type):
+    origin = get_origin(dst_type)
+    if origin is None:
+        return isinstance(value, dst_type)
+    
+    if not isinstance(value, origin):
+        return False
+    
+    args = get_args(dst_type)
+    if not args:
+        return True
+    
+    if origin is list:
+        elem_type = args[0]
+        return all(is_type(item, elem_type) for item in value)
+    elif origin is dict:
+        key_type, value_type = args
+        return all(
+            is_type(k, key_type) and is_type(v, value_type)
+            for k, v in value.items()
+        )
+    
+    return True
+
+class TypeConverter:
+    def __init__(self):
+        self._registry: Dict[Tuple[Type, Type], Callable[..., Any]] = {}
+
+    def register(self, src_type: Type, dst_type: Type, func: Callable[..., Any]):
+        self._registry[(src_type, dst_type)] = func
+
+    def can_convert(self, src_type: Type, dst_type: Type) -> bool:
+        return self._find_path(src_type, dst_type) is not None
+
+    def convert(self, value: Any, dst_type: Type, **kwargs) -> Any:
+        if value is None:
+            return None
+        
+        if dst_type == Any:
+            return value
+
+        src_type = type(value)
+
+        if is_type(value, dst_type):
+            return value
+
+        if (src_type, dst_type) in self._registry:
+            return self._call_func(self._registry[(src_type, dst_type)], value, **kwargs)
+
+        try:
+            return dst_type(value)
+        except Exception:
+            pass
+
+        path = self._find_path(src_type, dst_type)
+        if not path:
+            raise TypeError(f"No conversion path found from {src_type.__name__} to {dst_type.__name__}.")
+
+        result = value
+        for i in range(len(path) - 1):
+            func = self._registry[(path[i], path[i + 1])]
+            result = self._call_func(func, result, **kwargs)
+        return result
+
+    def _call_func(self, func: Callable[..., Any], value: Any, **kwargs) -> Any:
+        sig = inspect.signature(func)
+        if len(sig.parameters) == 1:
+            return func(value)
+        else:
+            return func(value, **kwargs)
+
+    def _find_path(self, src_type: Type, dst_type: Type) -> List[Type] | None:
+        if src_type == dst_type:
+            return [src_type]
+
+        graph: Dict[Type, Set[Type]] = {}
+        for (s, d) in self._registry.keys():
+            graph.setdefault(s, set()).add(d)
+
+        queue = deque([[src_type]])
+        visited = {src_type}
+
+        while queue:
+            path = queue.popleft()
+            current = path[-1]
+            for neighbor in graph.get(current, []):
+                if neighbor in visited:
+                    continue
+                new_path = path + [neighbor]
+                if neighbor == dst_type:
+                    return new_path
+                queue.append(new_path)
+                visited.add(neighbor)
+        return None

service_forge/utils/workflow_clone.py

@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+import uuid
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from service_forge.workflow.node import Node
+    from service_forge.workflow.port import Port
+    from service_forge.workflow.edge import Edge
+    from service_forge.workflow.workflow import Workflow
+    from service_forge.workflow.trigger import Trigger
+    from service_forge.workflow.context import Context
+
+def workflow_clone(self: Workflow, task_id: uuid.UUID, trigger_node: Trigger) -> Workflow:
+    from service_forge.workflow.workflow import Workflow
+
+    if self.nodes is not None and len(self.nodes) > 0:
+        context = self.nodes[0].context._clone()
+    else:
+        context = Context(variables={})
+
+    node_map: dict[Node, Node] = {node: node._clone(context) for node in self.nodes}
+
+    port_map: dict[Port, Port] = {}
+    port_map.update({port: port._clone(node_map) for port in self.input_ports})
+    port_map.update({port: port._clone(node_map) for port in self.output_ports})
+    for node in self.nodes:
+        for port in node.input_ports:
+            if port not in port_map:
+                port_map[port] = port._clone(node_map)
+        for port in node.output_ports:
+            if port not in port_map:
+                port_map[port] = port._clone(node_map)
+
+    edge_map: dict[Edge, Edge] = {}
+    for node in self.nodes:
+        for edge in node.input_edges:
+            if edge not in edge_map:
+                edge_map[edge] = edge._clone(node_map, port_map)
+        for edge in node.output_edges:
+            if edge not in edge_map:
+                edge_map[edge] = edge._clone(node_map, port_map)
+
+    # fill port.port
+    for old_port, new_port in port_map.items():
+        if old_port.port is not None:
+            new_port.port = port_map[old_port.port]
+
+
+    # fill ports and edges in nodes
+    for old_node, new_node in node_map.items():
+        new_node.input_edges = [edge_map[edge] for edge in old_node.input_edges]
+        new_node.output_edges = [edge_map[edge] for edge in old_node.output_edges]
+        new_node.input_ports = [port_map[port] for port in old_node.input_ports]
+        new_node.output_ports = [port_map[port] for port in old_node.output_ports]
+        new_node.input_variables = {
+            port_map[port]: value for port, value in old_node.input_variables.items()
+        }
+
+    workflow = Workflow(
+        name=self.name,
+        description=self.description,
+        nodes=[node_map[node] for node in self.nodes],
+        input_ports=[port_map[port] for port in self.input_ports],
+        output_ports=[port_map[port] for port in self.output_ports],
+        _handle_stream_output=self._handle_stream_output,
+        _handle_query_user=self._handle_query_user,
+        database_manager=self.database_manager,
+        max_concurrent_runs=self.max_concurrent_runs,
+        callbacks=self.callbacks,
+        task_id=task_id,
+        real_trigger_node=trigger_node,
+    )
+
+    for node in workflow.nodes:
+        node.workflow = workflow
+
+    return workflow
+
+def port_clone(self: Port, node_map: dict[Node, Node]) -> Port:
+    from service_forge.workflow.port import Port
+    node = node_map[self.node] if self.node is not None else None
+    port = Port(
+        name=self.name,
+        type=self.type,
+        node=node,
+        port=None,
+        value=self.value,
+        default=self.default,
+        is_extended=self.is_extended,
+        is_extended_generated=self.is_extended_generated,
+    )
+    port.is_prepared = self.is_prepared
+    return port
+
+def node_clone(self: Node, context: Context) -> Node:
+    node = self.__class__(
+        name=self.name
+    )
+    node.context = context
+    node.input_edges = []
+    node.output_edges = []
+    node.input_ports = []
+    node.output_ports = []
+    node.query_user = self.query_user
+    node.workflow = None
+
+    if self.sub_workflows is not None:
+        raise ValueError("Sub workflows are not supported in node clone.")
+    node.sub_workflows = None
+    node.input_variables = {}
+    node.num_activated_input_edges = self.num_activated_input_edges
+
+    return node
+
+def edge_clone(self: Edge, node_map: dict[Node, Node], port_map: dict[Port, Port]) -> Edge:
+    from service_forge.workflow.edge import Edge
+    start_node = node_map[self.start_node] if self.start_node is not None else None
+    end_node = node_map[self.end_node] if self.end_node is not None else None
+    return Edge(
+        start_node=start_node,
+        end_node=end_node,
+        start_port=port_map[self.start_port],
+        end_port=port_map[self.end_port],
+    )

service_forge/workflow/__init__.py

@@ -0,0 +1 @@
+from .workflow_type import workflow_type_register

service_forge/workflow/context.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+from typing import Any
+
+class Context():
+    def __init__(
+        self,
+        variables: dict[Any, Any] = dict(),
+    ) -> None:
+        self.variables = variables
+
+    def _clone(self) -> Context:
+        return Context(
+            variables={key: value for key, value in self.variables.items()},
+        )

service_forge/workflow/edge.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from ..utils.workflow_clone import edge_clone
+
+if TYPE_CHECKING:
+    from .node import Node
+    from .port import Port
+
+class Edge:
+    def __init__(
+        self,
+        start_node: Node,
+        end_node: Node,
+        start_port: Port,
+        end_port: Port,
+    ) -> None:
+        self.start_node = start_node
+        self.end_node = end_node
+        self.start_port = start_port
+        self.end_port = end_port
+
+    def _clone(self, node_map: dict[Node, Node], port_map: dict[Port, Port]) -> Edge:
+        return edge_clone(self, node_map, port_map)

service_forge/workflow/node.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+from typing import Any, AsyncIterator, Union, TYPE_CHECKING, Callable, Awaitable
+from abc import ABC, abstractmethod
+import uuid
+from loguru import logger
+from .edge import Edge
+from .port import Port
+from .context import Context
+from ..utils.register import Register
+from ..db.database import DatabaseManager, PostgresDatabase, MongoDatabase, RedisDatabase
+from ..utils.workflow_clone import node_clone
+
+if TYPE_CHECKING:
+    from .workflow import Workflow
+
+class Node(ABC):
+    DEFAULT_INPUT_PORTS: list[Port] = []
+    DEFAULT_OUTPUT_PORTS: list[Port] = []
+
+    CLASS_NOT_REQUIRED_TO_REGISTER = ['Node']
+    AUTO_FILL_INPUT_PORTS = []
+
+    def __init__(
+        self,
+        name: str,
+        context: Context = None,
+        input_edges: list[Edge] = None,
+        output_edges: list[Edge] = None,
+        input_ports: list[Port] = DEFAULT_INPUT_PORTS,
+        output_ports: list[Port] = DEFAULT_OUTPUT_PORTS,
+        query_user: Callable[[str, str], Awaitable[str]] = None,
+    ) -> None:
+        from .workflow_group import WorkflowGroup
+        self.name = name
+        self.input_edges = [] if input_edges is None else input_edges
+        self.output_edges = [] if output_edges is None else output_edges
+        self.input_ports = input_ports
+        self.output_ports = output_ports
+        self.workflow: Workflow = None
+        self.query_user = query_user
+        self.sub_workflows: WorkflowGroup = None
+
+        # runtime variables
+        self.context = context
+        self.input_variables: dict[Port, Any] = {}
+        self.num_activated_input_edges = 0
+
+    @property
+    def default_postgres_database(self) -> PostgresDatabase | None:
+        return self.database_manager.get_default_postgres_database()
+
+    @property
+    def default_mongo_database(self) -> MongoDatabase | None:
+        return self.database_manager.get_default_mongo_database()
+
+    @property
+    def default_redis_database(self) -> RedisDatabase | None:
+        return self.database_manager.get_default_redis_database()
+    
+    @property
+    def database_manager(self) -> DatabaseManager:
+        return self.workflow.database_manager
+
+    def backup(self) -> None:
+        # do NOT use deepcopy here
+        # self.bak_context = deepcopy(self.context)
+        # TODO: what if the value changes after backup?
+        self.bak_input_variables = {port: value for port, value in self.input_variables.items()}
+        self.bak_num_activated_input_edges = self.num_activated_input_edges
+
+    def reset(self) -> None:
+        # self.context = deepcopy(self.bak_context)
+        self.input_variables = {port: value for port, value in self.bak_input_variables.items()}
+        self.num_activated_input_edges = self.bak_num_activated_input_edges
+
+    def __init_subclass__(cls) -> None:
+        if cls.__name__ not in Node.CLASS_NOT_REQUIRED_TO_REGISTER:
+            node_register.register(cls.__name__, cls)
+        return super().__init_subclass__()
+
+    def _query_user(self, prompt: str) -> Callable[[str, str], Awaitable[str]]:
+        return self.query_user(self.name, prompt)
+    
+    def variables_to_params(self) -> dict[str, Any]:
+        params = {port.name: self.input_variables[port] for port in self.input_variables.keys() if not port.is_extended_generated}
+        for port in self.input_variables.keys():
+            if port.is_extended_generated:
+                if port.get_extended_name() not in params:
+                    params[port.get_extended_name()] = []
+                params[port.get_extended_name()].append((port.get_extended_index(), self.input_variables[port]))
+                params[port.get_extended_name()].sort()
+        return params
+
+    def is_trigger(self) -> bool:
+        from .trigger import Trigger
+        return isinstance(self, Trigger)
+
+    # TODO: maybe add a function before the run function?
+
+    @abstractmethod
+    async def _run(self, **kwargs) -> Union[None, AsyncIterator]:
+        ...
+    
+    def run(self) -> Union[None, AsyncIterator]:
+        for key in list(self.input_variables.keys()):
+            if key and key.name[0].isupper():
+                del self.input_variables[key]
+        params = self.variables_to_params()
+        return self._run(**params)
+
+    def get_input_port_by_name(self, name: str) -> Port:
+        # TODO: add warning if port is extended
+        for port in self.input_ports:
+            if port.name == name:
+                return port
+        return None
+
+    def get_output_port_by_name(self, name: str) -> Port:
+        # TODO: add warning if port is extended
+        for port in self.output_ports:
+            if port.name == name:
+                return port
+        return None
+
+    def try_create_extended_input_port(self, name: str) -> None:
+        for port in self.input_ports:
+            if port.is_extended and name.startswith(port.name + '_') and name[len(port.name + '_'):].isdigit():
+                self.input_ports.append(Port(name=name, type=port.type, node=port.node, port=port.port, value=port.value, default=port.default, is_extended=False, is_extended_generated=True))
+
+    def try_create_extended_output_port(self, name: str) -> None:
+        for port in self.output_ports:
+            if port.is_extended and name.startswith(port.name + '_') and name[len(port.name + '_'):].isdigit():
+                self.output_ports.append(Port(name=name, type=port.type, node=port.node, port=port.port, value=port.value, default=port.default, is_extended=False, is_extended_generated=True))
+
+    def num_input_ports(self) -> int:
+        return sum(1 for port in self.input_ports if not port.is_extended)
+    
+    def is_ready(self) -> bool:
+        return self.num_activated_input_edges == self.num_input_ports()
+
+    def fill_input_by_name(self, port_name: str, value: Any) -> None:
+        self.try_create_extended_input_port(port_name)
+        port = self.get_input_port_by_name(port_name)
+        if port is None:
+            raise ValueError(f'{port_name} is not a valid input port.')
+        self.fill_input(port, value)
+    
+    def fill_input(self, port: Port, value: Any) -> None:
+        port.activate(value)
+
+    def activate_output_edges(self, port: str | Port, data: Any) -> None:
+        if isinstance(port, str):
+            port = self.get_output_port_by_name(port)
+        for output_edge in self.output_edges:
+            if output_edge.start_port == port:
+                output_edge.end_port.activate(data)
+
+    # for trigger nodes
+    def prepare_output_edges(self, port: Port, data: Any) -> None:
+        if isinstance(port, str):
+            port = self.get_output_port_by_name(port)
+        for output_edge in self.output_edges:
+            if output_edge.start_port == port:
+                output_edge.end_port.prepare(data)
+    
+    def trigger_output_edges(self, port: Port) -> None:
+        if isinstance(port, str):
+            port = self.get_output_port_by_name(port)
+        for output_edge in self.output_edges:
+            if output_edge.start_port == port:
+                output_edge.end_port.trigger()
+
+    # TODO: the result is outputed to the trigger now, maybe we should add a new function to output the result to the workflow
+    def output_to_workflow(self, data: Any) -> None:
+        self.workflow._handle_workflow_output(self.name, data)
+
+    def extended_output_name(self, name: str, index: int) -> str:
+        return name + '_' + str(index)
+
+    def _clone(self, context: Context) -> Node:
+        return node_clone(self, context)
+
+node_register = Register[Node]()

service_forge/workflow/nodes/__init__.py

@@ -0,0 +1,8 @@
+from .output.print_node import PrintNode
+from .output.kafka_output_node import KafkaOutputNode
+from .control.if_node import IfNode
+from .control.switch_node import SwitchNode
+from .llm.query_llm_node import QueryLLMNode
+from .test.if_console_input_node import IfConsoleInputNode
+from .nested.workflow_node import WorkflowNode
+from .test.time_consuming_node import TimeConsumingNode

service_forge/workflow/nodes/control/if_node.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+from ...node import Node
+from ...port import Port
+
+class IfNode(Node):
+    DEFAULT_INPUT_PORTS = [
+        Port("TRIGGER", bool),
+        Port("condition", str),
+    ]
+
+    DEFAULT_OUTPUT_PORTS = [
+        Port("true", bool),
+        Port("false", bool),
+    ]
+
+    def __init__(
+        self, 
+        name: str,
+    ) -> None:
+        super().__init__(
+            name,
+        )
+
+    async def _run(self, condition: str) -> None:
+        result = eval(condition)
+        if result:
+            self.activate_output_edges('true', True)
+        else:
+            self.activate_output_edges('false', False)

service_forge/workflow/nodes/control/switch_node.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+from typing import Any
+from ...node import Node
+from ...port import Port
+
+class SwitchNode(Node):
+    DEFAULT_INPUT_PORTS = [
+        Port("TRIGGER", bool),
+        Port("condition", str, is_extended=True),
+    ]
+
+    DEFAULT_OUTPUT_PORTS = [
+        Port("result", Any, is_extended=True),
+    ]
+
+    def __init__(
+        self, 
+        name: str,
+    ) -> None:
+        super().__init__(
+            name,
+        )
+
+    async def _run(self, condition: list[tuple[int, str]]) -> None:
+        for index, cond in condition:
+            if eval(cond):
+                self.activate_output_edges(self.extended_output_name('result', index), str(index))
+                break

service_forge/workflow/nodes/input/console_input_node.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+from service_forge.workflow.node import Node
+from service_forge.workflow.port import Port
+from service_forge.llm import chat_stream, Model
+
+class ConsoleInputNode(Node):
+    DEFAULT_INPUT_PORTS = [
+        Port("TRIGGER", bool),
+        Port("prompt", str),
+    ]
+
+    DEFAULT_OUTPUT_PORTS = [
+        Port("user_input", str),
+    ]
+
+    AUTO_FILL_INPUT_PORTS = []
+
+    def __init__(
+        self, 
+        name: str,
+    ) -> None:
+        super().__init__(name)
+
+    async def _run(self, prompt: str) -> None:
+        user_input = self._query_user(prompt)
+        self.activate_output_edges(self.get_output_port_by_name('user_input'), user_input)