pyflowx 0.1.1__py3-none-any.whl

pyflowx/__init__.py

@@ -0,0 +1,75 @@
+"""PyFlowX —— 轻量、类型安全的 DAG 任务调度器。
+
+公共 API
+--------
+* :class:`TaskSpec` —— 不可变任务描述符（唯一需要配置的东西）。
+* :class:`Graph` —— 由一组 spec 构建的 DAG；负责校验、分层、可视化。
+* :func:`run` —— 以 ``sequential`` / ``thread`` / ``async`` 策略执行图。
+* :class:`RunReport` —— 类型化、可查询的运行结果。
+* :class:`Context` —— 整体上下文注入的标注标记。
+* 状态后端：:class:`StateBackend`、:class:`MemoryBackend`、:class:`JSONBackend`。
+
+快速上手
+--------
+    import pyflowx as px
+
+    def extract() -> list[int]: return [1, 2, 3]
+    def double(extract: list[int]) -> list[int]: return [x * 2 for x in extract]
+
+    graph = px.Graph.from_specs([
+        px.TaskSpec("extract", extract),
+        px.TaskSpec("double", double, ("extract",)),
+    ])
+    report = px.run(graph, strategy="sequential")
+    print(report["double"])  # [2, 4, 6]
+"""
+
+from __future__ import annotations
+
+from .context import Context, build_call_args, describe_injection
+from .errors import (
+    CycleError,
+    DuplicateTaskError,
+    InjectionError,
+    MissingDependencyError,
+    PyFlowXError,
+    StorageError,
+    TaskFailedError,
+    TaskTimeoutError,
+)
+from .executors import run
+from .graph import Graph
+from .report import RunReport
+from .storage import JSONBackend, MemoryBackend, StateBackend
+from .task import TaskEvent, TaskResult, TaskSpec, TaskStatus
+
+__version__ = "0.1.1"
+
+__all__ = [
+    # 核心类型
+    "TaskSpec",
+    "TaskStatus",
+    "TaskResult",
+    "TaskEvent",
+    "Context",
+    "Graph",
+    "RunReport",
+    # 执行
+    "run",
+    # 状态后端
+    "StateBackend",
+    "MemoryBackend",
+    "JSONBackend",
+    # 错误
+    "PyFlowXError",
+    "DuplicateTaskError",
+    "MissingDependencyError",
+    "CycleError",
+    "TaskFailedError",
+    "TaskTimeoutError",
+    "InjectionError",
+    "StorageError",
+    # 辅助（高级）
+    "build_call_args",
+    "describe_injection",
+]

pyflowx/__main__.py

@@ -0,0 +1,9 @@
+from pyflowx.examples.async_aggregation import main as async_aggregation_main
+from pyflowx.examples.etl_pipeline import main as etl_pipeline_main
+from pyflowx.examples.parallel_run import main as parallel_run_main
+
+
+def main():
+    async_aggregation_main()
+    etl_pipeline_main()
+    parallel_run_main()

pyflowx/context.py

@@ -0,0 +1,194 @@
+"""上下文注入：把上游结果转换为函数参数。
+
+本机制让用户可以编写普通函数，其参数名*就是*依赖声明，从而消除其他
+DAG 库中泛滥的样板包装器（如 ``def wrapper(): return fn(workflow.get_task_result('x'))``）。
+
+注入规则（按顺序求值）
+----------------------
+1. **标注为** :class:`Context` 的参数接收完整结果映射。适用于需要遍历
+   所有输入的任务。
+2. **名称匹配某个依赖**的参数接收该依赖的结果。
+3. ``**kwargs`` 参数以 dict 形式接收*所有*依赖结果。
+4. ``TaskSpec.args`` / ``TaskSpec.kwargs`` 为*非依赖*参数提供静态值。
+
+若某参数无法解析且无默认值，则抛出 :class:`~pyflowx.errors.InjectionError`，
+并附带精确错误信息。
+"""
+
+from __future__ import annotations
+
+import inspect
+from typing import Any, Dict, List, Mapping, Set, Tuple
+
+from .errors import InjectionError
+from .task import Context, TaskSpec
+
+__all__ = ["Context", "build_call_args", "describe_injection", "_is_context_annotation"]
+
+
+def _is_context_annotation(annotation: Any) -> bool:
+    """判断参数标注是否为（或指向）``Context``。
+
+    处理三种形式：
+    * ``Context`` 别名对象本身；
+    * ``__name__``/``_name`` 为 ``Context`` 或 ``Mapping`` 的 typing 别名；
+    * *字符串*标注（``from __future__ import annotations`` 会在运行时
+      把所有标注变为字符串），如 ``"Context"`` 或 ``"px.Context"``。
+    """
+    if annotation is Context:
+        return True
+    # `from __future__ import annotations` 产生的字符串标注。
+    if isinstance(annotation, str):
+        # 匹配 "Context"、"px.Context"、"pyflowx.Context" 等。
+        return annotation == "Context" or annotation.endswith(".Context")
+    # 按限定名匹配，支持 ``from pyflowx import Context`` 再导出。
+    name = getattr(annotation, "__name__", None) or getattr(annotation, "_name", None)
+    if name in ("Context", "Mapping"):
+        return True
+    return False
+
+
+def build_call_args(
+    spec: TaskSpec[object],
+    context: Mapping[str, Any],
+) -> Tuple[Tuple[Any, ...], Dict[str, Any]]:
+    """解析用于调用 ``spec.fn`` 的 ``(args, kwargs)``。
+
+    参数
+    ----
+    spec:
+        任务 spec，提供 ``fn``、``depends_on``、``args``、``kwargs``。
+    context:
+        依赖名 -> 结果值的映射。仅保证本任务自身的 ``depends_on`` 条目
+        存在；其他任务的结果被排除，以保持注入的确定性。
+
+    返回
+    ----
+    (args, kwargs)
+        可直接展开为 ``spec.fn(*args, **kwargs)``。
+
+    抛出
+    ----
+    InjectionError
+        若必需参数无法满足，或静态 ``kwargs`` 与注入依赖名冲突。
+    """
+    sig = inspect.signature(spec.fn)
+    params = sig.parameters
+
+    # 检测特殊参数类型。
+    var_keyword = next(
+        (p for p in params.values() if p.kind == inspect.Parameter.VAR_KEYWORD),
+        None,
+    )
+
+    # 与本任务相关的上下文子集。
+    dep_context: Dict[str, Any] = {
+        name: context[name] for name in spec.depends_on if name in context
+    }
+
+    # 检测静态 kwargs 与依赖名的冲突。
+    collisions = set(spec.kwargs) & set(dep_context)
+    if collisions:
+        raise InjectionError(
+            spec.name,
+            f"static kwargs {sorted(collisions)} collide with dependency names; "
+            "rename the static kwarg or the dependency.",
+        )
+
+    injected_kwargs: Dict[str, Any] = {}
+    leftover_dep_results: Dict[str, Any] = dict(dep_context)
+
+    # 被 spec.args 消费的位置参数。记录哪些参数名已被位置填充，
+    # 以便在基于名称的注入（依赖 / Context / 静态 kwargs）时跳过。
+    positional_params: List[str] = []
+    positional_kinds = (
+        inspect.Parameter.POSITIONAL_ONLY,
+        inspect.Parameter.POSITIONAL_OR_KEYWORD,
+    )
+    for pname, param in params.items():
+        if param.kind in positional_kinds:
+            positional_params.append(pname)
+    # 前 len(spec.args) 个位置参数由 spec.args 填充。
+    args_filled: Set[str] = set(positional_params[: len(spec.args)])
+
+    for pname, param in params.items():
+        # 跳过已被位置 spec.args 填充的参数。
+        if pname in args_filled:
+            continue
+
+        # 规则 1：标注为 Context -> 完整映射。
+        if _is_context_annotation(param.annotation):
+            injected_kwargs[pname] = dep_context
+            continue
+
+        # 规则 2：名称匹配某个依赖。
+        if pname in dep_context:
+            injected_kwargs[pname] = dep_context[pname]
+            leftover_dep_results.pop(pname, None)
+            continue
+
+        # 规则 3：在循环后通过 **kwargs 处理。
+
+        # 规则 4：静态 kwargs 填充其余参数。
+        if pname in spec.kwargs:
+            injected_kwargs[pname] = spec.kwargs[pname]
+            continue
+
+        # 该参数无来源：必须有默认值，否则报错。
+        if param.default is inspect.Parameter.empty and param.kind not in (
+            inspect.Parameter.VAR_POSITIONAL,
+            inspect.Parameter.VAR_KEYWORD,
+        ):
+            raise InjectionError(
+                spec.name,
+                f"parameter {pname!r} has no dependency, static value, or default.",
+            )
+
+    # 规则 3：**kwargs 吞掉剩余依赖结果。
+    if var_keyword is not None and leftover_dep_results:
+        # 先合并静态 kwargs，再合并依赖结果（冲突已在上方拒绝）。
+        merged = dict(spec.kwargs)
+        merged.update(injected_kwargs)
+        merged.update(leftover_dep_results)
+        injected_kwargs = merged
+
+    return tuple(spec.args), injected_kwargs
+
+
+def describe_injection(spec: TaskSpec[object]) -> str:
+    """生成任务参数注入方式的人类可读描述。
+
+    供 ``dry_run`` 使用，在不执行的情况下展示执行计划。
+    """
+    sig = inspect.signature(spec.fn)
+    # 确定哪些位置参数由 spec.args 填充。
+    positional_params = [
+        p
+        for p, param in sig.parameters.items()
+        if param.kind
+        in (
+            inspect.Parameter.POSITIONAL_ONLY,
+            inspect.Parameter.POSITIONAL_OR_KEYWORD,
+        )
+    ]
+    args_filled = set(positional_params[: len(spec.args)])
+    parts = []
+    for pname, param in sig.parameters.items():
+        if pname in args_filled:
+            idx = positional_params.index(pname)
+            parts.append(f"{pname}={spec.args[idx]!r}")
+        elif _is_context_annotation(param.annotation):
+            parts.append(f"{pname}=<Context>")
+        elif pname in spec.depends_on:
+            parts.append(f"{pname}=<result:{pname}>")
+        elif pname in spec.kwargs:
+            parts.append(f"{pname}={spec.kwargs[pname]!r}")
+        elif param.default is not inspect.Parameter.empty:
+            parts.append(f"{pname}=<default>")
+        elif param.kind == inspect.Parameter.VAR_KEYWORD:
+            parts.append("**kwargs=<all-deps>")
+        elif param.kind == inspect.Parameter.VAR_POSITIONAL:
+            parts.append("*args")
+        else:
+            parts.append(f"{pname}=<UNRESOLVED>")
+    return f"{spec.name}({', '.join(parts)})"

pyflowx/errors.py

@@ -0,0 +1,92 @@
+"""PyFlowX 错误层级。
+
+所有错误都是 :class:`PyFlowXError` 的具体子类，调用者可以用单个
+``except`` 子句捕获整个错误家族，同时仍可按类型区分以做细粒度处理。
+"""
+
+from __future__ import annotations
+
+from typing import Any, Iterable, Optional
+
+
+class PyFlowXError(Exception):
+    """所有 PyFlowX 错误的基类。"""
+
+
+class DuplicateTaskError(PyFlowXError):
+    """任务名被重复注册时抛出。"""
+
+    def __init__(self, name: str) -> None:
+        super().__init__(f"Task '{name}' is already registered in the graph.")
+        self.name = name
+
+
+class MissingDependencyError(PyFlowXError):
+    """任务依赖了图中不存在的名称时抛出。"""
+
+    def __init__(self, task: str, dependency: str) -> None:
+        super().__init__(
+            f"Task '{task}' depends on unknown task '{dependency}'. "
+            "Add the dependency before (or together with) this task."
+        )
+        self.task = task
+        self.dependency = dependency
+
+
+class CycleError(PyFlowXError):
+    """依赖图存在环时抛出。"""
+
+    def __init__(self, cycle: Iterable[str]) -> None:
+        cycle_list = list(cycle)
+        chain = " -> ".join(cycle_list + cycle_list[:1])
+        super().__init__(f"The dependency graph contains a cycle: {chain}")
+        self.cycle = cycle_list
+
+
+class TaskFailedError(PyFlowXError):
+    """任务耗尽所有重试后仍失败时抛出。
+
+    原始异常保留在 :attr:`__cause__` 上，同时通过 :attr:`cause` 暴露，
+    便于用户代码访问。
+    """
+
+    def __init__(
+        self,
+        task: str,
+        cause: BaseException,
+        attempts: int,
+        layer: Optional[int] = None,
+    ) -> None:
+        location = f" (layer {layer})" if layer is not None else ""
+        super().__init__(
+            f"Task '{task}' failed after {attempts} attempt(s){location}: {cause}"
+        )
+        self.task = task
+        self.cause = cause
+        self.attempts = attempts
+        self.layer = layer
+
+
+class TaskTimeoutError(PyFlowXError):
+    """任务超出配置的超时时间时抛出。"""
+
+    def __init__(self, task: str, timeout: float) -> None:
+        super().__init__(f"Task '{task}' timed out after {timeout:.3f}s.")
+        self.task = task
+        self.timeout = timeout
+
+
+class InjectionError(PyFlowXError):
+    """上下文注入无法满足任务签名时抛出。"""
+
+    def __init__(self, task: str, detail: str) -> None:
+        super().__init__(f"Cannot inject context for task '{task}': {detail}")
+        self.task = task
+
+
+class StorageError(PyFlowXError):
+    """状态后端在持久化失败时抛出。"""
+
+    def __init__(self, detail: str, cause: Optional[BaseException] = None) -> None:
+        super().__init__(f"State storage error: {detail}")
+        self.cause: Any = cause

pyflowx/examples/async_aggregation.py

@@ -0,0 +1,58 @@
+"""Example 3: async aggregation with static args and Context injection.
+
+Shows:
+  * async task functions executed with strategy="async".
+  * static positional args (TaskSpec.args) for parameterised tasks.
+  * Context annotation to receive the full upstream result mapping.
+  * on_event callback for real-time progress.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Dict, List
+
+import pyflowx as px
+
+
+async def fetch_user(uid: int) -> dict:
+    await asyncio.sleep(0.2)
+    return {"id": uid, "name": f"User{uid}"}
+
+
+async def fetch_posts(uid: int) -> List[int]:
+    await asyncio.sleep(0.2)
+    return [uid, uid + 1]
+
+
+# Context annotation → receives the full mapping of upstream results.
+def aggregate(ctx: px.Context) -> Dict[str, Any]:
+    return dict(ctx)
+
+
+def main() -> None:
+    graph = px.Graph.from_specs(
+        [
+            # Static positional args parameterise the same function twice.
+            px.TaskSpec("fetch_user", fetch_user, args=(1,)),
+            px.TaskSpec("fetch_posts", fetch_posts, args=(1,)),
+            px.TaskSpec("aggregate", aggregate, ("fetch_user", "fetch_posts")),
+        ]
+    )
+
+    print("=== Dry run ===")
+    px.run(graph, strategy="async", dry_run=True)
+
+    events: List[px.TaskEvent] = []
+    print("\n=== Async execution ===")
+    report = px.run(graph, strategy="async", on_event=events.append)
+
+    for ev in events:
+        print(f"  event: {ev.task} -> {ev.status.value}")
+
+    print(f"\naggregate = {report['aggregate']}")
+    print(report.describe())
+
+
+if __name__ == "__main__":
+    main()

pyflowx/examples/etl_pipeline.py

@@ -0,0 +1,81 @@
+"""Example 1: ETL pipeline (sequential strategy).
+
+Demonstrates the core PyFlowX workflow:
+  * Define tasks as plain functions.
+  * Declare the DAG with a list of TaskSpec.
+  * Parameter names == dependency names → automatic context injection,
+    no wrappers needed (contrast with flowweaver's get_task_result boilerplate).
+  * dry_run to preview, then execute and read typed results from RunReport.
+"""
+
+from __future__ import annotations
+
+from typing import List
+
+import pyflowx as px
+
+# --- task functions: pure, testable, no framework coupling ------------- #
+
+
+def extract_customers() -> List[dict]:
+    return [
+        {"id": "C001", "name": "Alice"},
+        {"id": "C002", "name": "Bob"},
+    ]
+
+
+def extract_orders() -> List[dict]:
+    return [
+        {"id": "O001", "customer_id": "C001", "amount": 150.0},
+        {"id": "O002", "customer_id": "C002", "amount": 200.5},
+    ]
+
+
+# Parameter names match dependency names → automatic injection.
+def transform(
+    extract_customers: List[dict],
+    extract_orders: List[dict],
+) -> List[dict]:
+    cmap = {c["id"]: c for c in extract_customers}
+    return [
+        {**o, "customer_name": cmap[o["customer_id"]]["name"]}
+        for o in extract_orders
+        if o["customer_id"] in cmap
+    ]
+
+
+def load(transform: List[dict]) -> int:
+    print(f"  loaded {len(transform)} records")
+    return len(transform)
+
+
+def main() -> None:
+    graph = px.Graph.from_specs(
+        [
+            px.TaskSpec("extract_customers", extract_customers, tags=("extract",)),
+            px.TaskSpec("extract_orders", extract_orders, tags=("extract",)),
+            px.TaskSpec(
+                "transform",
+                transform,
+                ("extract_customers", "extract_orders"),
+                tags=("transform",),
+            ),
+            px.TaskSpec("load", load, ("transform",), retries=1, tags=("load",)),
+        ]
+    )
+
+    print("=== Execution plan ===")
+    print(graph.describe())
+
+    print("\n=== Dry run (no execution) ===")
+    px.run(graph, strategy="sequential", dry_run=True)
+
+    print("\n=== Sequential execution ===")
+    report = px.run(graph, strategy="sequential")
+    print(report.describe())
+    print(f"\nload result = {report['load']}")
+    print(f"summary = {report.summary()}")
+
+
+if __name__ == "__main__":
+    main()

pyflowx/examples/parallel_run.py

@@ -0,0 +1,59 @@
+"""Example 2: parallel execution (thread strategy).
+
+Same DAG run with sequential vs. thread strategy to show layer-internal
+parallelism. Tasks within a layer run concurrently; layers are barriers.
+
+Layer 1: [fetch_a, fetch_b]   (parallel)
+Layer 2: [merge]              (waits for both)
+"""
+
+from __future__ import annotations
+
+import time
+
+import pyflowx as px
+
+
+def fetch_a() -> str:
+    time.sleep(0.5)
+    return "a"
+
+
+def fetch_b() -> str:
+    time.sleep(0.5)
+    return "b"
+
+
+def merge(fetch_a: str, fetch_b: str) -> str:
+    return fetch_a + fetch_b
+
+
+def main() -> None:
+    graph = px.Graph.from_specs(
+        [
+            px.TaskSpec("fetch_a", fetch_a),
+            px.TaskSpec("fetch_b", fetch_b),
+            px.TaskSpec("merge", merge, ("fetch_a", "fetch_b")),
+        ]
+    )
+
+    print("=== Mermaid diagram ===")
+    print(graph.to_mermaid("LR"))
+
+    print("\n=== Sequential (expect ~1.0s) ===")
+    start = time.time()
+    report_seq = px.run(graph, strategy="sequential")
+    t_seq = time.time() - start
+    print(f"  result={report_seq['merge']}  time={t_seq:.2f}s")
+
+    print("\n=== Threaded (expect ~0.5s) ===")
+    start = time.time()
+    report_thr = px.run(graph, strategy="thread", max_workers=2)
+    t_thr = time.time() - start
+    print(f"  result={report_thr['merge']}  time={t_thr:.2f}s")
+
+    print(f"\nspeedup = {t_seq / t_thr:.2f}x")
+
+
+if __name__ == "__main__":
+    main()