flowlet 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowlet-0.1.0/PKG-INFO +127 -0
- flowlet-0.1.0/README.md +120 -0
- flowlet-0.1.0/flowlet/__init__.py +21 -0
- flowlet-0.1.0/flowlet/core.py +484 -0
- flowlet-0.1.0/flowlet.egg-info/PKG-INFO +127 -0
- flowlet-0.1.0/flowlet.egg-info/SOURCES.txt +8 -0
- flowlet-0.1.0/flowlet.egg-info/dependency_links.txt +1 -0
- flowlet-0.1.0/flowlet.egg-info/top_level.txt +1 -0
- flowlet-0.1.0/pyproject.toml +7 -0
- flowlet-0.1.0/setup.cfg +4 -0
flowlet-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: flowlet
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A lightweight Python workflow engine with DAG nodes, async execution, and conditional branches.
|
|
5
|
+
Requires-Python: >=3.8
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
|
|
8
|
+
# flowlet
|
|
9
|
+
|
|
10
|
+
A lightweight Python workflow engine with DAG nodes, async execution, conditional branches, and optional dependencies.
|
|
11
|
+
|
|
12
|
+
## Features
|
|
13
|
+
|
|
14
|
+
- Function-based node definition with dependency resolution
|
|
15
|
+
- Concurrent execution within each DAG level (asyncio)
|
|
16
|
+
- Conditional branches via `when`
|
|
17
|
+
- Optional dependencies via `optional(...)`
|
|
18
|
+
- Runtime inputs injected from `WORKFLOW_PARAM` (JSON)
|
|
19
|
+
- Execution context: trace_id, run_id, timings, logs, outputs
|
|
20
|
+
- Exportable workflow graph via `workflow_compile_graph`
|
|
21
|
+
|
|
22
|
+
## Installation
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install flowlet
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Quickstart
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
import asyncio
|
|
32
|
+
from flowlet import Input, node, optional, workflow_compile, workflow_run
|
|
33
|
+
|
|
34
|
+
class Inputs:
|
|
35
|
+
a = Input(int, desc="param a")
|
|
36
|
+
b = Input(int, desc="param b")
|
|
37
|
+
|
|
38
|
+
@node(inputs={"x": Inputs.a}, outputs={"result": "x"})
|
|
39
|
+
async def step1(x):
|
|
40
|
+
await asyncio.sleep(0.1)
|
|
41
|
+
return x + 1
|
|
42
|
+
|
|
43
|
+
@node(inputs={"y": Inputs.b}, outputs={"result": "y"})
|
|
44
|
+
def step2(y):
|
|
45
|
+
return y * 2
|
|
46
|
+
|
|
47
|
+
@node(inputs={"x": step1.result, "y": step2.result}, outputs={"route": "branch"})
|
|
48
|
+
def route(x, y):
|
|
49
|
+
return "A" if x + y >= 0 else "B"
|
|
50
|
+
|
|
51
|
+
@node(
|
|
52
|
+
inputs={"route": route.route, "x": step1.result},
|
|
53
|
+
outputs={"result": "A"},
|
|
54
|
+
when=lambda route, **_: route == "A",
|
|
55
|
+
)
|
|
56
|
+
def step_a(route, x):
|
|
57
|
+
return x * 10
|
|
58
|
+
|
|
59
|
+
@node(
|
|
60
|
+
inputs={"route": route.route, "y": step2.result},
|
|
61
|
+
outputs={"result": "B"},
|
|
62
|
+
when=lambda route, **_: route == "B",
|
|
63
|
+
)
|
|
64
|
+
def step_b(route, y):
|
|
65
|
+
return y * -10
|
|
66
|
+
|
|
67
|
+
@node(
|
|
68
|
+
inputs={"a": optional(step_a.result), "b": optional(step_b.result)},
|
|
69
|
+
outputs={"result": "merge"},
|
|
70
|
+
)
|
|
71
|
+
def merge(a=None, b=None):
|
|
72
|
+
return a if a is not None else b
|
|
73
|
+
|
|
74
|
+
compiled = workflow_compile(Inputs)
|
|
75
|
+
ctx, output = workflow_run(compiled)
|
|
76
|
+
print(output)
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Provide runtime inputs via environment variable:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
export WORKFLOW_PARAM='{"a": 1, "b": 2}'
|
|
83
|
+
python your_app.py
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## Concepts
|
|
87
|
+
|
|
88
|
+
### Inputs
|
|
89
|
+
|
|
90
|
+
Declare inputs with `Input(type, desc)` and provide values through `WORKFLOW_PARAM`. Types are cast at runtime.
|
|
91
|
+
|
|
92
|
+
### Node
|
|
93
|
+
|
|
94
|
+
Use `@node(inputs=..., outputs=..., when=...)` to wrap a function:
|
|
95
|
+
|
|
96
|
+
- `inputs`: mapping of parameter name to `Input` or upstream output
|
|
97
|
+
- `outputs`: output names and descriptions
|
|
98
|
+
- `when`: callable that returns True/False to control execution
|
|
99
|
+
|
|
100
|
+
### Optional dependencies
|
|
101
|
+
|
|
102
|
+
Use `optional(step.result)` for dependencies that may be missing; they are injected as `None`.
|
|
103
|
+
|
|
104
|
+
### Execution
|
|
105
|
+
|
|
106
|
+
`workflow_compile(Inputs)` builds a compiled graph. `workflow_run(compiled)` runs the workflow and returns:
|
|
107
|
+
|
|
108
|
+
- `ctx` with trace_id, run_id, timings, logs, outputs, skipped
|
|
109
|
+
- `output` is the last node's output
|
|
110
|
+
|
|
111
|
+
### Graph export
|
|
112
|
+
|
|
113
|
+
`workflow_compile_graph(Inputs)` returns a serializable DAG:
|
|
114
|
+
|
|
115
|
+
- `inputs`: input definitions
|
|
116
|
+
- `nodes`: node metadata including docstring and source
|
|
117
|
+
- `edges`: dependency edges
|
|
118
|
+
- `levels`: parallel execution levels
|
|
119
|
+
|
|
120
|
+
## Notes
|
|
121
|
+
|
|
122
|
+
- `WORKFLOW_PARAM` must be valid JSON.
|
|
123
|
+
- For a single output, return a scalar. For multiple outputs, return a dict or tuple/list.
|
|
124
|
+
|
|
125
|
+
## License
|
|
126
|
+
|
|
127
|
+
MIT
|
flowlet-0.1.0/README.md
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# flowlet
|
|
2
|
+
|
|
3
|
+
A lightweight Python workflow engine with DAG nodes, async execution, conditional branches, and optional dependencies.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- Function-based node definition with dependency resolution
|
|
8
|
+
- Concurrent execution within each DAG level (asyncio)
|
|
9
|
+
- Conditional branches via `when`
|
|
10
|
+
- Optional dependencies via `optional(...)`
|
|
11
|
+
- Runtime inputs injected from `WORKFLOW_PARAM` (JSON)
|
|
12
|
+
- Execution context: trace_id, run_id, timings, logs, outputs
|
|
13
|
+
- Exportable workflow graph via `workflow_compile_graph`
|
|
14
|
+
|
|
15
|
+
## Installation
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
pip install flowlet
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
## Quickstart
|
|
22
|
+
|
|
23
|
+
```python
|
|
24
|
+
import asyncio
|
|
25
|
+
from flowlet import Input, node, optional, workflow_compile, workflow_run
|
|
26
|
+
|
|
27
|
+
class Inputs:
|
|
28
|
+
a = Input(int, desc="param a")
|
|
29
|
+
b = Input(int, desc="param b")
|
|
30
|
+
|
|
31
|
+
@node(inputs={"x": Inputs.a}, outputs={"result": "x"})
|
|
32
|
+
async def step1(x):
|
|
33
|
+
await asyncio.sleep(0.1)
|
|
34
|
+
return x + 1
|
|
35
|
+
|
|
36
|
+
@node(inputs={"y": Inputs.b}, outputs={"result": "y"})
|
|
37
|
+
def step2(y):
|
|
38
|
+
return y * 2
|
|
39
|
+
|
|
40
|
+
@node(inputs={"x": step1.result, "y": step2.result}, outputs={"route": "branch"})
|
|
41
|
+
def route(x, y):
|
|
42
|
+
return "A" if x + y >= 0 else "B"
|
|
43
|
+
|
|
44
|
+
@node(
|
|
45
|
+
inputs={"route": route.route, "x": step1.result},
|
|
46
|
+
outputs={"result": "A"},
|
|
47
|
+
when=lambda route, **_: route == "A",
|
|
48
|
+
)
|
|
49
|
+
def step_a(route, x):
|
|
50
|
+
return x * 10
|
|
51
|
+
|
|
52
|
+
@node(
|
|
53
|
+
inputs={"route": route.route, "y": step2.result},
|
|
54
|
+
outputs={"result": "B"},
|
|
55
|
+
when=lambda route, **_: route == "B",
|
|
56
|
+
)
|
|
57
|
+
def step_b(route, y):
|
|
58
|
+
return y * -10
|
|
59
|
+
|
|
60
|
+
@node(
|
|
61
|
+
inputs={"a": optional(step_a.result), "b": optional(step_b.result)},
|
|
62
|
+
outputs={"result": "merge"},
|
|
63
|
+
)
|
|
64
|
+
def merge(a=None, b=None):
|
|
65
|
+
return a if a is not None else b
|
|
66
|
+
|
|
67
|
+
compiled = workflow_compile(Inputs)
|
|
68
|
+
ctx, output = workflow_run(compiled)
|
|
69
|
+
print(output)
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Provide runtime inputs via environment variable:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
export WORKFLOW_PARAM='{"a": 1, "b": 2}'
|
|
76
|
+
python your_app.py
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
## Concepts
|
|
80
|
+
|
|
81
|
+
### Inputs
|
|
82
|
+
|
|
83
|
+
Declare inputs with `Input(type, desc)` and provide values through `WORKFLOW_PARAM`. Types are cast at runtime.
|
|
84
|
+
|
|
85
|
+
### Node
|
|
86
|
+
|
|
87
|
+
Use `@node(inputs=..., outputs=..., when=...)` to wrap a function:
|
|
88
|
+
|
|
89
|
+
- `inputs`: mapping of parameter name to `Input` or upstream output
|
|
90
|
+
- `outputs`: output names and descriptions
|
|
91
|
+
- `when`: callable that returns True/False to control execution
|
|
92
|
+
|
|
93
|
+
### Optional dependencies
|
|
94
|
+
|
|
95
|
+
Use `optional(step.result)` for dependencies that may be missing; they are injected as `None`.
|
|
96
|
+
|
|
97
|
+
### Execution
|
|
98
|
+
|
|
99
|
+
`workflow_compile(Inputs)` builds a compiled graph. `workflow_run(compiled)` runs the workflow and returns:
|
|
100
|
+
|
|
101
|
+
- `ctx` with trace_id, run_id, timings, logs, outputs, skipped
|
|
102
|
+
- `output` is the last node's output
|
|
103
|
+
|
|
104
|
+
### Graph export
|
|
105
|
+
|
|
106
|
+
`workflow_compile_graph(Inputs)` returns a serializable DAG:
|
|
107
|
+
|
|
108
|
+
- `inputs`: input definitions
|
|
109
|
+
- `nodes`: node metadata including docstring and source
|
|
110
|
+
- `edges`: dependency edges
|
|
111
|
+
- `levels`: parallel execution levels
|
|
112
|
+
|
|
113
|
+
## Notes
|
|
114
|
+
|
|
115
|
+
- `WORKFLOW_PARAM` must be valid JSON.
|
|
116
|
+
- For a single output, return a scalar. For multiple outputs, return a dict or tuple/list.
|
|
117
|
+
|
|
118
|
+
## License
|
|
119
|
+
|
|
120
|
+
MIT
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# 只从核心文件中导入你想暴露的接口
|
|
2
|
+
from .core import (
|
|
3
|
+
workflow_run,
|
|
4
|
+
node,
|
|
5
|
+
Input,
|
|
6
|
+
workflow_compile,
|
|
7
|
+
optional,
|
|
8
|
+
SKIP,
|
|
9
|
+
workflow_compile_graph
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
# 显式定义出口
|
|
13
|
+
__all__ = [
|
|
14
|
+
'workflow_run',
|
|
15
|
+
'node',
|
|
16
|
+
'Input',
|
|
17
|
+
'workflow_compile',
|
|
18
|
+
'optional',
|
|
19
|
+
'SKIP',
|
|
20
|
+
'workflow_compile_graph'
|
|
21
|
+
]
|
|
@@ -0,0 +1,484 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import contextvars
|
|
3
|
+
import inspect
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
import textwrap
|
|
9
|
+
import time
|
|
10
|
+
import uuid
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Input:
|
|
14
|
+
def __init__(self, typ, desc=""):
|
|
15
|
+
self.type = typ
|
|
16
|
+
self.desc = desc
|
|
17
|
+
self.name = None
|
|
18
|
+
|
|
19
|
+
def __repr__(self):
|
|
20
|
+
return f"Input(name={self.name!r}, type={self.type}, desc={self.desc!r})"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class _OutputRef:
|
|
24
|
+
def __init__(self, node, key):
|
|
25
|
+
self.node = node
|
|
26
|
+
self.key = key
|
|
27
|
+
|
|
28
|
+
def __repr__(self):
|
|
29
|
+
return f"OutputRef(node={self.node.name!r}, key={self.key!r})"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class _OptionalRef:
|
|
33
|
+
def __init__(self, src):
|
|
34
|
+
self.src = src
|
|
35
|
+
|
|
36
|
+
def __repr__(self):
|
|
37
|
+
return f"OptionalRef(src={self.src!r})"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def optional(src):
|
|
41
|
+
return _OptionalRef(src)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class _SkipValue:
|
|
45
|
+
def __repr__(self):
|
|
46
|
+
return "SKIP"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
SKIP = _SkipValue()
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class _Node:
|
|
53
|
+
def __init__(self, func, inputs, outputs, when=None):
|
|
54
|
+
self.func = func
|
|
55
|
+
self.inputs = inputs or {}
|
|
56
|
+
self.outputs = outputs or {}
|
|
57
|
+
self.when = when
|
|
58
|
+
self.name = func.__name__
|
|
59
|
+
self.output_keys = list(self.outputs.keys())
|
|
60
|
+
self.__name__ = func.__name__
|
|
61
|
+
self.__doc__ = func.__doc__
|
|
62
|
+
|
|
63
|
+
def __call__(self, *args, **kwargs):
|
|
64
|
+
return self.func(*args, **kwargs)
|
|
65
|
+
|
|
66
|
+
def __getattr__(self, item):
|
|
67
|
+
if item in self.outputs:
|
|
68
|
+
return _OutputRef(self, item)
|
|
69
|
+
raise AttributeError(f"{self.name} has no output {item!r}")
|
|
70
|
+
|
|
71
|
+
def __repr__(self):
|
|
72
|
+
return f"Node(name={self.name!r})"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def node(inputs=None, outputs=None, when=None):
|
|
76
|
+
def decorator(func):
|
|
77
|
+
return _Node(func, inputs, outputs, when=when)
|
|
78
|
+
return decorator
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _collect_inputs(inputs_cls):
|
|
82
|
+
inputs = {}
|
|
83
|
+
for name, value in inputs_cls.__dict__.items():
|
|
84
|
+
if isinstance(value, Input):
|
|
85
|
+
value.name = name
|
|
86
|
+
inputs[name] = value
|
|
87
|
+
return inputs
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _collect_nodes(namespace):
|
|
91
|
+
return [value for value in namespace.values() if isinstance(value, _Node)]
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class _WorkflowContext:
|
|
95
|
+
def __init__(self, logger=None, trace_id=None, run_id=None):
|
|
96
|
+
self.logger = logger or logging.getLogger("workflow")
|
|
97
|
+
self.trace_id = trace_id or uuid.uuid4().hex
|
|
98
|
+
self.run_id = run_id or uuid.uuid4().hex
|
|
99
|
+
self.timings = {}
|
|
100
|
+
self.skipped = {}
|
|
101
|
+
self.results = {}
|
|
102
|
+
self.outputs = {}
|
|
103
|
+
self.logs = {}
|
|
104
|
+
self.start_time = None
|
|
105
|
+
self.end_time = None
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
_current_node = contextvars.ContextVar("workflow_current_node", default=None)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class _ContextLogHandler(logging.Handler):
|
|
112
|
+
def __init__(self, ctx):
|
|
113
|
+
super().__init__()
|
|
114
|
+
self.ctx = ctx
|
|
115
|
+
|
|
116
|
+
def emit(self, record):
|
|
117
|
+
node_name = _current_node.get()
|
|
118
|
+
if not node_name:
|
|
119
|
+
return
|
|
120
|
+
entry = {
|
|
121
|
+
"time": record.created,
|
|
122
|
+
"level": record.levelname,
|
|
123
|
+
"logger": record.name,
|
|
124
|
+
"message": record.getMessage(),
|
|
125
|
+
}
|
|
126
|
+
if record.pathname:
|
|
127
|
+
entry["pathname"] = record.pathname
|
|
128
|
+
if record.lineno:
|
|
129
|
+
entry["lineno"] = record.lineno
|
|
130
|
+
if record.exc_info:
|
|
131
|
+
entry["exc_info"] = self.formatException(record.exc_info)
|
|
132
|
+
self.ctx.logs.setdefault(node_name, []).append(entry)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _attach_log_handler(ctx, level=logging.INFO):
|
|
136
|
+
handler = _ContextLogHandler(ctx)
|
|
137
|
+
handler.setLevel(logging.NOTSET)
|
|
138
|
+
root = logging.getLogger()
|
|
139
|
+
old_level = root.level
|
|
140
|
+
if old_level > level:
|
|
141
|
+
root.setLevel(level)
|
|
142
|
+
root.addHandler(handler)
|
|
143
|
+
return handler, root, old_level
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _detach_log_handler(handler, root, old_level=None):
|
|
147
|
+
if handler and root:
|
|
148
|
+
root.removeHandler(handler)
|
|
149
|
+
if old_level is not None:
|
|
150
|
+
root.setLevel(old_level)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _build_graph(nodes):
|
|
154
|
+
deps = {node: set() for node in nodes}
|
|
155
|
+
adj = {node: set() for node in nodes}
|
|
156
|
+
for node in nodes:
|
|
157
|
+
for src in node.inputs.values():
|
|
158
|
+
if isinstance(src, _OptionalRef):
|
|
159
|
+
src = src.src
|
|
160
|
+
if isinstance(src, _OutputRef):
|
|
161
|
+
if src.node not in deps:
|
|
162
|
+
raise ValueError(f"Unknown node dependency: {src.node}")
|
|
163
|
+
deps[node].add(src.node)
|
|
164
|
+
adj[src.node].add(node)
|
|
165
|
+
return deps, adj
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _toposort_levels(nodes, deps, adj):
|
|
169
|
+
indeg = {node: len(deps[node]) for node in nodes}
|
|
170
|
+
queue = [node for node in nodes if indeg[node] == 0]
|
|
171
|
+
levels = []
|
|
172
|
+
order = []
|
|
173
|
+
while queue:
|
|
174
|
+
level = list(queue)
|
|
175
|
+
levels.append(level)
|
|
176
|
+
queue = []
|
|
177
|
+
for node in level:
|
|
178
|
+
order.append(node)
|
|
179
|
+
for nxt in adj[node]:
|
|
180
|
+
indeg[nxt] -= 1
|
|
181
|
+
if indeg[nxt] == 0:
|
|
182
|
+
queue.append(nxt)
|
|
183
|
+
if len(order) != len(nodes):
|
|
184
|
+
raise ValueError("Circular dependency detected.")
|
|
185
|
+
return levels, order
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _validate_signatures(nodes):
|
|
189
|
+
for node in nodes:
|
|
190
|
+
sig = inspect.signature(node.func)
|
|
191
|
+
params = sig.parameters
|
|
192
|
+
param_names = set(params.keys())
|
|
193
|
+
has_var_kw = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
|
|
194
|
+
|
|
195
|
+
invalid_inputs = [name for name in node.inputs.keys() if name not in param_names]
|
|
196
|
+
if invalid_inputs and not has_var_kw:
|
|
197
|
+
raise ValueError(
|
|
198
|
+
f"{node.name} has inputs {invalid_inputs} not present in "
|
|
199
|
+
f"function signature {list(param_names)}"
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
missing_required = []
|
|
203
|
+
for name, p in params.items():
|
|
204
|
+
if p.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD):
|
|
205
|
+
continue
|
|
206
|
+
if p.kind == inspect.Parameter.POSITIONAL_ONLY:
|
|
207
|
+
if p.default is inspect._empty:
|
|
208
|
+
missing_required.append(name)
|
|
209
|
+
continue
|
|
210
|
+
if p.default is inspect._empty and name not in node.inputs:
|
|
211
|
+
missing_required.append(name)
|
|
212
|
+
|
|
213
|
+
if missing_required:
|
|
214
|
+
raise ValueError(
|
|
215
|
+
f"{node.name} is missing required inputs: {missing_required} "
|
|
216
|
+
f"(signature: {sig})"
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
for name, p in params.items():
|
|
220
|
+
if p.kind == inspect.Parameter.POSITIONAL_ONLY and name in node.inputs:
|
|
221
|
+
raise ValueError(
|
|
222
|
+
f"{node.name} uses positional-only parameter {name}; "
|
|
223
|
+
"workflow injection only supports keyword arguments"
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def workflow_compile(inputs_cls, namespace=None):
|
|
228
|
+
if namespace is None:
|
|
229
|
+
module = sys.modules.get(inputs_cls.__module__)
|
|
230
|
+
namespace = module.__dict__ if module else globals()
|
|
231
|
+
inputs = _collect_inputs(inputs_cls)
|
|
232
|
+
nodes = _collect_nodes(namespace)
|
|
233
|
+
_validate_signatures(nodes)
|
|
234
|
+
deps, adj = _build_graph(nodes)
|
|
235
|
+
levels, order = _toposort_levels(nodes, deps, adj)
|
|
236
|
+
return {"inputs": inputs, "nodes": nodes, "order": levels, "flat_order": order}
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def workflow_compile_graph(inputs_cls, namespace=None):
|
|
240
|
+
compiled = workflow_compile(inputs_cls, namespace)
|
|
241
|
+
ordered_nodes = compiled["flat_order"]
|
|
242
|
+
inputs_payload = []
|
|
243
|
+
for name, inp in compiled["inputs"].items():
|
|
244
|
+
inputs_payload.append({
|
|
245
|
+
"name": name,
|
|
246
|
+
"type": inp.type.__name__ if inp.type else None,
|
|
247
|
+
"description": inp.desc,
|
|
248
|
+
})
|
|
249
|
+
|
|
250
|
+
def _strip_decorators(source_text):
|
|
251
|
+
if not source_text:
|
|
252
|
+
return source_text
|
|
253
|
+
lines = source_text.splitlines()
|
|
254
|
+
start = 0
|
|
255
|
+
for idx, line in enumerate(lines):
|
|
256
|
+
stripped = line.lstrip()
|
|
257
|
+
if stripped.startswith("def ") or stripped.startswith("async def "):
|
|
258
|
+
start = idx
|
|
259
|
+
break
|
|
260
|
+
return "\n".join(lines[start:]).strip()
|
|
261
|
+
|
|
262
|
+
nodes_payload = []
|
|
263
|
+
for node in ordered_nodes:
|
|
264
|
+
doc = node.__doc__ or ""
|
|
265
|
+
description = textwrap.dedent(doc).strip()
|
|
266
|
+
try:
|
|
267
|
+
source = textwrap.dedent(inspect.getsource(node.func)).strip()
|
|
268
|
+
source = _strip_decorators(source)
|
|
269
|
+
except (OSError, TypeError):
|
|
270
|
+
source = ""
|
|
271
|
+
nodes_payload.append({
|
|
272
|
+
"name": node.name,
|
|
273
|
+
"inputs": list(node.inputs.keys()),
|
|
274
|
+
"outputs": dict(node.outputs),
|
|
275
|
+
"description": description,
|
|
276
|
+
"source": source,
|
|
277
|
+
})
|
|
278
|
+
|
|
279
|
+
edges_payload = []
|
|
280
|
+
for node in compiled["nodes"]:
|
|
281
|
+
for input_name, src in node.inputs.items():
|
|
282
|
+
optional_input = False
|
|
283
|
+
if isinstance(src, _OptionalRef):
|
|
284
|
+
optional_input = True
|
|
285
|
+
src = src.src
|
|
286
|
+
if isinstance(src, _OutputRef):
|
|
287
|
+
edges_payload.append({
|
|
288
|
+
"source": src.node.name,
|
|
289
|
+
"target": node.name,
|
|
290
|
+
"source_output": src.key,
|
|
291
|
+
"target_input": input_name,
|
|
292
|
+
"optional": optional_input,
|
|
293
|
+
})
|
|
294
|
+
|
|
295
|
+
levels_payload = [
|
|
296
|
+
[node.name for node in level]
|
|
297
|
+
for level in compiled["order"]
|
|
298
|
+
]
|
|
299
|
+
return {
|
|
300
|
+
"inputs": inputs_payload,
|
|
301
|
+
"nodes": nodes_payload,
|
|
302
|
+
"edges": edges_payload,
|
|
303
|
+
"levels": levels_payload,
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _load_params(inputs):
|
|
308
|
+
raw = os.environ.get("WORKFLOW_PARAM", "{}")
|
|
309
|
+
try:
|
|
310
|
+
data = json.loads(raw) if raw else {}
|
|
311
|
+
except json.JSONDecodeError as exc:
|
|
312
|
+
raise ValueError(f"WORKFLOW_PARAM is not valid JSON: {raw}") from exc
|
|
313
|
+
params = {}
|
|
314
|
+
for name, inp in inputs.items():
|
|
315
|
+
if name not in data:
|
|
316
|
+
raise ValueError(f"Missing input parameter: {name}")
|
|
317
|
+
value = data[name]
|
|
318
|
+
if inp.type is None:
|
|
319
|
+
params[name] = value
|
|
320
|
+
continue
|
|
321
|
+
try:
|
|
322
|
+
params[name] = inp.type(value)
|
|
323
|
+
except Exception as exc:
|
|
324
|
+
raise ValueError(
|
|
325
|
+
f"Parameter {name} cannot be converted to {inp.type}: {value}"
|
|
326
|
+
) from exc
|
|
327
|
+
return params
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _normalize_outputs(node, result):
|
|
331
|
+
if not node.output_keys:
|
|
332
|
+
return {}
|
|
333
|
+
if len(node.output_keys) == 1:
|
|
334
|
+
return {node.output_keys[0]: result}
|
|
335
|
+
if isinstance(result, dict):
|
|
336
|
+
missing = [key for key in node.output_keys if key not in result]
|
|
337
|
+
if missing:
|
|
338
|
+
raise ValueError(f"{node.name} is missing outputs: {missing}")
|
|
339
|
+
return {key: result[key] for key in node.output_keys}
|
|
340
|
+
if isinstance(result, (list, tuple)) and len(result) == len(node.output_keys):
|
|
341
|
+
return dict(zip(node.output_keys, result))
|
|
342
|
+
raise ValueError(f"{node.name} output does not match declared outputs")
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _resolve_kwargs(node, params, results):
|
|
346
|
+
kwargs = {}
|
|
347
|
+
missing_required = []
|
|
348
|
+
for name, src in node.inputs.items():
|
|
349
|
+
optional_input = False
|
|
350
|
+
if isinstance(src, _OptionalRef):
|
|
351
|
+
optional_input = True
|
|
352
|
+
src = src.src
|
|
353
|
+
if isinstance(src, Input):
|
|
354
|
+
kwargs[name] = params[src.name]
|
|
355
|
+
elif isinstance(src, _OutputRef):
|
|
356
|
+
if src.node not in results:
|
|
357
|
+
if optional_input:
|
|
358
|
+
kwargs[name] = None
|
|
359
|
+
else:
|
|
360
|
+
missing_required.append(name)
|
|
361
|
+
continue
|
|
362
|
+
value = results[src.node].get(src.key, SKIP)
|
|
363
|
+
if value is SKIP:
|
|
364
|
+
if optional_input:
|
|
365
|
+
kwargs[name] = None
|
|
366
|
+
else:
|
|
367
|
+
missing_required.append(name)
|
|
368
|
+
continue
|
|
369
|
+
kwargs[name] = value
|
|
370
|
+
else:
|
|
371
|
+
kwargs[name] = src
|
|
372
|
+
return kwargs, missing_required
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _skip_outputs(node):
|
|
376
|
+
if not node.output_keys:
|
|
377
|
+
return {}
|
|
378
|
+
return {key: SKIP for key in node.output_keys}
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _call_when(node, kwargs, ctx):
|
|
382
|
+
if node.when is None:
|
|
383
|
+
return True
|
|
384
|
+
try:
|
|
385
|
+
sig = inspect.signature(node.when)
|
|
386
|
+
except (TypeError, ValueError):
|
|
387
|
+
return bool(node.when(**kwargs))
|
|
388
|
+
params = sig.parameters
|
|
389
|
+
has_var_kw = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
|
|
390
|
+
if has_var_kw:
|
|
391
|
+
call_kwargs = dict(kwargs)
|
|
392
|
+
else:
|
|
393
|
+
call_kwargs = {k: v for k, v in kwargs.items() if k in params}
|
|
394
|
+
if "ctx" in params:
|
|
395
|
+
call_kwargs["ctx"] = ctx
|
|
396
|
+
return bool(node.when(**call_kwargs))
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def _final_output(compiled, results):
|
|
400
|
+
last_node = None
|
|
401
|
+
flat_order = compiled.get("flat_order") or []
|
|
402
|
+
if flat_order:
|
|
403
|
+
last_node = flat_order[-1]
|
|
404
|
+
return results.get(last_node) if last_node else None
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
async def _workflow_run_async(compiled):
|
|
408
|
+
ctx = _WorkflowContext()
|
|
409
|
+
params = _load_params(compiled["inputs"])
|
|
410
|
+
results = {}
|
|
411
|
+
levels = compiled.get("order") or []
|
|
412
|
+
max_concurrency = max((len(level) for level in levels), default=0)
|
|
413
|
+
sem = asyncio.Semaphore(max_concurrency) if max_concurrency else None
|
|
414
|
+
ctx.start_time = time.perf_counter()
|
|
415
|
+
log_handler, log_root, log_old_level = _attach_log_handler(ctx)
|
|
416
|
+
ctx.logger.info(
|
|
417
|
+
"workflow start trace_id=%s run_id=%s", ctx.trace_id, ctx.run_id
|
|
418
|
+
)
|
|
419
|
+
|
|
420
|
+
async def _run_node(node):
|
|
421
|
+
token = _current_node.set(node.name)
|
|
422
|
+
try:
|
|
423
|
+
kwargs, missing_required = _resolve_kwargs(node, params, results)
|
|
424
|
+
if missing_required:
|
|
425
|
+
ctx.skipped[node.name] = f"missing inputs: {missing_required}"
|
|
426
|
+
ctx.timings[node.name] = 0.0
|
|
427
|
+
output = _skip_outputs(node)
|
|
428
|
+
ctx.outputs[node.name] = output
|
|
429
|
+
return node, output
|
|
430
|
+
if not _call_when(node, kwargs, ctx):
|
|
431
|
+
ctx.skipped[node.name] = "condition false"
|
|
432
|
+
ctx.timings[node.name] = 0.0
|
|
433
|
+
output = _skip_outputs(node)
|
|
434
|
+
ctx.outputs[node.name] = output
|
|
435
|
+
return node, output
|
|
436
|
+
node_start = time.perf_counter()
|
|
437
|
+
async def _call():
|
|
438
|
+
result = await _execute_node(node, kwargs)
|
|
439
|
+
return node, _normalize_outputs(node, result)
|
|
440
|
+
if sem:
|
|
441
|
+
async with sem:
|
|
442
|
+
node, output = await _call()
|
|
443
|
+
else:
|
|
444
|
+
node, output = await _call()
|
|
445
|
+
ctx.timings[node.name] = time.perf_counter() - node_start
|
|
446
|
+
ctx.outputs[node.name] = output
|
|
447
|
+
ctx.logger.debug("node done name=%s elapsed=%.6fs", node.name, ctx.timings[node.name])
|
|
448
|
+
return node, output
|
|
449
|
+
finally:
|
|
450
|
+
_current_node.reset(token)
|
|
451
|
+
|
|
452
|
+
async def _execute_node(node, kwargs):
|
|
453
|
+
if inspect.iscoroutinefunction(node.func):
|
|
454
|
+
result = await node.func(**kwargs)
|
|
455
|
+
else:
|
|
456
|
+
result = node.func(**kwargs)
|
|
457
|
+
if inspect.isawaitable(result):
|
|
458
|
+
result = await result
|
|
459
|
+
return result
|
|
460
|
+
|
|
461
|
+
try:
|
|
462
|
+
for level in levels:
|
|
463
|
+
tasks = [asyncio.create_task(_run_node(node)) for node in level]
|
|
464
|
+
for node, output in await asyncio.gather(*tasks):
|
|
465
|
+
results[node] = output
|
|
466
|
+
finally:
|
|
467
|
+
ctx.end_time = time.perf_counter()
|
|
468
|
+
ctx.logger.info(
|
|
469
|
+
"workflow end trace_id=%s run_id=%s elapsed=%.6fs",
|
|
470
|
+
ctx.trace_id,
|
|
471
|
+
ctx.run_id,
|
|
472
|
+
ctx.end_time - ctx.start_time,
|
|
473
|
+
)
|
|
474
|
+
_detach_log_handler(log_handler, log_root, log_old_level)
|
|
475
|
+
ctx.results = results
|
|
476
|
+
return ctx, _final_output(compiled, results)
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def workflow_run(compiled):
|
|
480
|
+
return asyncio.run(
|
|
481
|
+
_workflow_run_async(compiled)
|
|
482
|
+
)
|
|
483
|
+
|
|
484
|
+
__all__ = ['workflow_run', 'node', 'Input', 'workflow_compile', 'workflow', 'optional', 'SKIP', 'workflow_compile_graph']
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: flowlet
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A lightweight Python workflow engine with DAG nodes, async execution, and conditional branches.
|
|
5
|
+
Requires-Python: >=3.8
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
|
|
8
|
+
# flowlet
|
|
9
|
+
|
|
10
|
+
A lightweight Python workflow engine with DAG nodes, async execution, conditional branches, and optional dependencies.
|
|
11
|
+
|
|
12
|
+
## Features
|
|
13
|
+
|
|
14
|
+
- Function-based node definition with dependency resolution
|
|
15
|
+
- Concurrent execution within each DAG level (asyncio)
|
|
16
|
+
- Conditional branches via `when`
|
|
17
|
+
- Optional dependencies via `optional(...)`
|
|
18
|
+
- Runtime inputs injected from `WORKFLOW_PARAM` (JSON)
|
|
19
|
+
- Execution context: trace_id, run_id, timings, logs, outputs
|
|
20
|
+
- Exportable workflow graph via `workflow_compile_graph`
|
|
21
|
+
|
|
22
|
+
## Installation
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install flowlet
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Quickstart
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
import asyncio
|
|
32
|
+
from flowlet import Input, node, optional, workflow_compile, workflow_run
|
|
33
|
+
|
|
34
|
+
class Inputs:
|
|
35
|
+
a = Input(int, desc="param a")
|
|
36
|
+
b = Input(int, desc="param b")
|
|
37
|
+
|
|
38
|
+
@node(inputs={"x": Inputs.a}, outputs={"result": "x"})
|
|
39
|
+
async def step1(x):
|
|
40
|
+
await asyncio.sleep(0.1)
|
|
41
|
+
return x + 1
|
|
42
|
+
|
|
43
|
+
@node(inputs={"y": Inputs.b}, outputs={"result": "y"})
|
|
44
|
+
def step2(y):
|
|
45
|
+
return y * 2
|
|
46
|
+
|
|
47
|
+
@node(inputs={"x": step1.result, "y": step2.result}, outputs={"route": "branch"})
|
|
48
|
+
def route(x, y):
|
|
49
|
+
return "A" if x + y >= 0 else "B"
|
|
50
|
+
|
|
51
|
+
@node(
|
|
52
|
+
inputs={"route": route.route, "x": step1.result},
|
|
53
|
+
outputs={"result": "A"},
|
|
54
|
+
when=lambda route, **_: route == "A",
|
|
55
|
+
)
|
|
56
|
+
def step_a(route, x):
|
|
57
|
+
return x * 10
|
|
58
|
+
|
|
59
|
+
@node(
|
|
60
|
+
inputs={"route": route.route, "y": step2.result},
|
|
61
|
+
outputs={"result": "B"},
|
|
62
|
+
when=lambda route, **_: route == "B",
|
|
63
|
+
)
|
|
64
|
+
def step_b(route, y):
|
|
65
|
+
return y * -10
|
|
66
|
+
|
|
67
|
+
@node(
|
|
68
|
+
inputs={"a": optional(step_a.result), "b": optional(step_b.result)},
|
|
69
|
+
outputs={"result": "merge"},
|
|
70
|
+
)
|
|
71
|
+
def merge(a=None, b=None):
|
|
72
|
+
return a if a is not None else b
|
|
73
|
+
|
|
74
|
+
compiled = workflow_compile(Inputs)
|
|
75
|
+
ctx, output = workflow_run(compiled)
|
|
76
|
+
print(output)
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Provide runtime inputs via environment variable:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
export WORKFLOW_PARAM='{"a": 1, "b": 2}'
|
|
83
|
+
python your_app.py
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## Concepts
|
|
87
|
+
|
|
88
|
+
### Inputs
|
|
89
|
+
|
|
90
|
+
Declare inputs with `Input(type, desc)` and provide values through `WORKFLOW_PARAM`. Types are cast at runtime.
|
|
91
|
+
|
|
92
|
+
### Node
|
|
93
|
+
|
|
94
|
+
Use `@node(inputs=..., outputs=..., when=...)` to wrap a function:
|
|
95
|
+
|
|
96
|
+
- `inputs`: mapping of parameter name to `Input` or upstream output
|
|
97
|
+
- `outputs`: output names and descriptions
|
|
98
|
+
- `when`: callable that returns True/False to control execution
|
|
99
|
+
|
|
100
|
+
### Optional dependencies
|
|
101
|
+
|
|
102
|
+
Use `optional(step.result)` for dependencies that may be missing; they are injected as `None`.
|
|
103
|
+
|
|
104
|
+
### Execution
|
|
105
|
+
|
|
106
|
+
`workflow_compile(Inputs)` builds a compiled graph. `workflow_run(compiled)` runs the workflow and returns:
|
|
107
|
+
|
|
108
|
+
- `ctx` with trace_id, run_id, timings, logs, outputs, skipped
|
|
109
|
+
- `output` is the last node's output
|
|
110
|
+
|
|
111
|
+
### Graph export
|
|
112
|
+
|
|
113
|
+
`workflow_compile_graph(Inputs)` returns a serializable DAG:
|
|
114
|
+
|
|
115
|
+
- `inputs`: input definitions
|
|
116
|
+
- `nodes`: node metadata including docstring and source
|
|
117
|
+
- `edges`: dependency edges
|
|
118
|
+
- `levels`: parallel execution levels
|
|
119
|
+
|
|
120
|
+
## Notes
|
|
121
|
+
|
|
122
|
+
- `WORKFLOW_PARAM` must be valid JSON.
|
|
123
|
+
- For a single output, return a scalar. For multiple outputs, return a dict or tuple/list.
|
|
124
|
+
|
|
125
|
+
## License
|
|
126
|
+
|
|
127
|
+
MIT
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
flowlet
|
flowlet-0.1.0/setup.cfg
ADDED