stageflow-framework 0.9.0__tar.gz → 0.10.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stageflow_framework-0.10.0/PKG-INFO +186 -0
- stageflow_framework-0.10.0/README.md +153 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/pyproject.toml +5 -1
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/__init__.py +2 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/builtins/_args.py +4 -4
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/builtins/lists.py +5 -5
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/builtins/strings.py +4 -4
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/builtins/vars.py +1 -1
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/__init__.py +2 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/cel.py +4 -4
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/context.py +2 -2
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/debug.py +1 -1
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/nodes/__init__.py +2 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/nodes/base.py +4 -4
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/nodes/bindings.py +3 -3
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/nodes/branching.py +5 -5
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/nodes/entry.py +6 -6
- stageflow_framework-0.10.0/stageflow/core/nodes/map_block.py +269 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/nodes/parallel.py +7 -7
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/nodes/stage.py +14 -14
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/nodes/subpipeline.py +6 -6
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/nodes/try_block.py +44 -11
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/pipeline.py +7 -7
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/session.py +4 -1
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/typesys.py +16 -16
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/docs/schemas/pipeline.json +111 -12
- stageflow_framework-0.10.0/stageflow_framework.egg-info/PKG-INFO +186 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow_framework.egg-info/SOURCES.txt +2 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow_framework.egg-info/requires.txt +4 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/tests/test_core_flow.py +6 -6
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/tests/test_debug.py +10 -10
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/tests/test_entry_node.py +28 -28
- stageflow_framework-0.10.0/tests/test_map.py +405 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/tests/test_removed_local_scope.py +2 -2
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/tests/test_std_stages.py +9 -9
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/tests/test_try_block.py +60 -2
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/tests/test_typesystem.py +3 -3
- stageflow_framework-0.9.0/PKG-INFO +0 -478
- stageflow_framework-0.9.0/README.md +0 -448
- stageflow_framework-0.9.0/stageflow_framework.egg-info/PKG-INFO +0 -478
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/LICENSE +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/setup.cfg +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/builtins/__init__.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/builtins/dicts.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/builtins/logic.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/event.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/inputs.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/nodes/recovery.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/nodes/terminal.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/payload_schema.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/registry.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/spec.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/core/stage.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/docs/__init__.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/docs/schema.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/exceptions.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/py.typed +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow/testing.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow_framework.egg-info/dependency_links.txt +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/stageflow_framework.egg-info/top_level.txt +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/tests/test_docs_schema.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/tests/test_full_pipeline.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/tests/test_packaging.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/tests/test_parallel.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/tests/test_payload_validation.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/tests/test_pipeline_tester.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/tests/test_session_control.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/tests/test_session_wait_input.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/tests/test_snapshot.py +0 -0
- {stageflow_framework-0.9.0 → stageflow_framework-0.10.0}/tests/test_subpipeline.py +0 -0
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: stageflow-framework
|
|
3
|
+
Version: 0.10.0
|
|
4
|
+
Summary: StageFlow: pipeline framework for stages
|
|
5
|
+
Author-email: лень <pzrnqt1vrss@protonmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Repository, https://github.com/leo-need-more-coffee/stageflow
|
|
8
|
+
Project-URL: Issues, https://github.com/leo-need-more-coffee/stageflow/issues
|
|
9
|
+
Keywords: pipeline,workflow,orchestration,state-machine,cel
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
17
|
+
Classifier: Typing :: Typed
|
|
18
|
+
Requires-Python: >=3.11
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: pyyaml
|
|
22
|
+
Requires-Dist: jsonschema>=4.0.0
|
|
23
|
+
Requires-Dist: immutables>=0.20
|
|
24
|
+
Requires-Dist: common-expression-language>=0.9
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: flake8; extra == "dev"
|
|
27
|
+
Requires-Dist: build; extra == "dev"
|
|
28
|
+
Requires-Dist: twine; extra == "dev"
|
|
29
|
+
Provides-Extra: docs
|
|
30
|
+
Requires-Dist: mkdocs-material; extra == "docs"
|
|
31
|
+
Requires-Dist: mkdocs-static-i18n; extra == "docs"
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
<div align="center">
|
|
35
|
+
|
|
36
|
+
# StageFlow
|
|
37
|
+
|
|
38
|
+
**Pipelines described in JSON, executed in Python — and debuggable node by node.**
|
|
39
|
+
|
|
40
|
+
[](https://github.com/leo-need-more-coffee/stageflow/actions/workflows/tests.yml)
|
|
41
|
+
[](https://pypi.org/project/stageflow-framework/)
|
|
42
|
+
[](https://pypi.org/project/stageflow-framework/)
|
|
43
|
+
[](https://leo-need-more-coffee.github.io/stageflow/)
|
|
44
|
+
[](LICENSE)
|
|
45
|
+
|
|
46
|
+
[Documentation](https://leo-need-more-coffee.github.io/stageflow/) ·
|
|
47
|
+
[Tutorial](https://leo-need-more-coffee.github.io/stageflow/tutorial/) ·
|
|
48
|
+
[Документация на русском](https://leo-need-more-coffee.github.io/stageflow/ru/)
|
|
49
|
+
|
|
50
|
+
</div>
|
|
51
|
+
|
|
52
|
+
A pipeline is a graph of nodes in JSON. The work happens in stages — Python
|
|
53
|
+
classes you write. Between them travels one immutable frame of variables, and
|
|
54
|
+
everything the graph does with that frame is visible in the JSON: branching,
|
|
55
|
+
retries, error handling, concurrency, nested graphs.
|
|
56
|
+
|
|
57
|
+
Because the pipeline is data, it can be stored, diffed, generated, validated
|
|
58
|
+
before it runs — and drawn:
|
|
59
|
+
|
|
60
|
+

|
|
61
|
+
|
|
62
|
+
That is the [editor](https://github.com/leo-need-more-coffee/stageflow-ui): a
|
|
63
|
+
separate static page that draws and debugs a graph while this core executes it.
|
|
64
|
+
|
|
65
|
+
## Install
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
pip install stageflow-framework
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Python 3.11+.
|
|
72
|
+
|
|
73
|
+
## A pipeline in 30 seconds
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
import asyncio
|
|
77
|
+
|
|
78
|
+
from stageflow import BaseStage, Pipeline, Session, register_stage
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@register_stage("HelloStage")
|
|
82
|
+
class HelloStage(BaseStage):
|
|
83
|
+
"""
|
|
84
|
+
description: "Greets whoever the pipeline points at"
|
|
85
|
+
arguments:
|
|
86
|
+
name: string
|
|
87
|
+
outputs:
|
|
88
|
+
greeting: string
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
async def run(self):
|
|
92
|
+
name = self.get_arguments().get("name", "world")
|
|
93
|
+
self.set_outputs({"greeting": f"Hello, {name}!"})
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
pipeline = Pipeline.from_dict({
|
|
97
|
+
"nodes": [
|
|
98
|
+
{"id": "start", "type": "entry",
|
|
99
|
+
"variables": {"user_name": "Alice"}, "next": "hello"},
|
|
100
|
+
{"id": "hello", "type": "stage", "stage": "HelloStage",
|
|
101
|
+
"arguments": {"vars": {"name": "user_name"}},
|
|
102
|
+
"outputs": {"greeting": "greeting"}, "next": "finish"},
|
|
103
|
+
{"id": "finish", "type": "terminal",
|
|
104
|
+
"result": {"status": "ok"}, "artifacts": ["greeting"]},
|
|
105
|
+
],
|
|
106
|
+
})
|
|
107
|
+
pipeline.validate()
|
|
108
|
+
|
|
109
|
+
result = asyncio.run(Session(id="demo", pipeline=pipeline).run())
|
|
110
|
+
print(result.result) # {'status': 'ok'}
|
|
111
|
+
print(result.artifacts) # {'greeting': 'Hello, Alice!'}
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
The docstring is the stage's specification. `validate()` checks the graph
|
|
115
|
+
against it, so a wrong argument name is an error before anything runs.
|
|
116
|
+
|
|
117
|
+
## What the graph can do
|
|
118
|
+
|
|
119
|
+
| | |
|
|
120
|
+
|---|---|
|
|
121
|
+
| **Nine node types** | `entry`, `stage`, `condition`, `switch`, `parallel`, `try`, `map`, `subpipeline`, `terminal` |
|
|
122
|
+
| **One immutable frame** | variables travel along the path; a write produces a new frame, so branches never collide |
|
|
123
|
+
| **CEL expressions** | in conditions, in switch cases, and in any argument or output through the `.$` suffix |
|
|
124
|
+
| **Errors as roads** | `retry` on a node, `try`/`except` over a region of the graph derived from its shape |
|
|
125
|
+
| **Real concurrency** | `parallel` branches with their own frames and explicit merge rules |
|
|
126
|
+
| **Loops over data** | `map` runs a region of the graph once per element, sequentially or at once |
|
|
127
|
+
| **Nested graphs** | a `subpipeline` starts with a fresh frame and returns artifacts |
|
|
128
|
+
| **Gradual typing** | declare the variables that matter; checked at validation and on every write |
|
|
129
|
+
| **Step debugging** | stop between nodes, read and edit the frame, replay the event stream |
|
|
130
|
+
|
|
131
|
+
## Documentation
|
|
132
|
+
|
|
133
|
+
The [tutorial](https://leo-need-more-coffee.github.io/stageflow/tutorial/)
|
|
134
|
+
builds one working pipeline step by step, with screenshots from the editor.
|
|
135
|
+
The reference covers the rest:
|
|
136
|
+
|
|
137
|
+
[Quick start](https://leo-need-more-coffee.github.io/stageflow/quick-start/) ·
|
|
138
|
+
[Node types](https://leo-need-more-coffee.github.io/stageflow/node-types/) ·
|
|
139
|
+
[Data model](https://leo-need-more-coffee.github.io/stageflow/data-model/) ·
|
|
140
|
+
[Expressions](https://leo-need-more-coffee.github.io/stageflow/expressions/) ·
|
|
141
|
+
[Errors](https://leo-need-more-coffee.github.io/stageflow/errors/) ·
|
|
142
|
+
[Variable typing](https://leo-need-more-coffee.github.io/stageflow/variable-typing/) ·
|
|
143
|
+
[Session control](https://leo-need-more-coffee.github.io/stageflow/session-control/) ·
|
|
144
|
+
[Step debugging](https://leo-need-more-coffee.github.io/stageflow/step-debugging/)
|
|
145
|
+
|
|
146
|
+
One page per node type:
|
|
147
|
+
[entry](https://leo-need-more-coffee.github.io/stageflow/entry-node/) ·
|
|
148
|
+
[stage](https://leo-need-more-coffee.github.io/stageflow/stage-node/) ·
|
|
149
|
+
[condition](https://leo-need-more-coffee.github.io/stageflow/condition-node/) ·
|
|
150
|
+
[switch](https://leo-need-more-coffee.github.io/stageflow/switch-node/) ·
|
|
151
|
+
[parallel](https://leo-need-more-coffee.github.io/stageflow/parallel-branches/) ·
|
|
152
|
+
[try](https://leo-need-more-coffee.github.io/stageflow/try-node/) ·
|
|
153
|
+
[map](https://leo-need-more-coffee.github.io/stageflow/map-node/) ·
|
|
154
|
+
[subpipeline](https://leo-need-more-coffee.github.io/stageflow/subpipeline-node/) ·
|
|
155
|
+
[terminal](https://leo-need-more-coffee.github.io/stageflow/terminal-node/)
|
|
156
|
+
|
|
157
|
+
Sources are in [`docs/`](docs) (`*.md` English, `*.ru.md` Russian) and publish
|
|
158
|
+
themselves on every push to `main`.
|
|
159
|
+
|
|
160
|
+
## The rest of the project
|
|
161
|
+
|
|
162
|
+
| Repository | What it is |
|
|
163
|
+
|---|---|
|
|
164
|
+
| **stageflow** | this one: the core that runs the pipelines |
|
|
165
|
+
| [stageflow-ui](https://github.com/leo-need-more-coffee/stageflow-ui) | the editor: a static page that draws and debugs a graph |
|
|
166
|
+
| [stageflow-example](https://github.com/leo-need-more-coffee/stageflow-example) | a working backend for the editor: a support bot in four pipelines |
|
|
167
|
+
|
|
168
|
+
## Development
|
|
169
|
+
|
|
170
|
+
```bash
|
|
171
|
+
pip install -e ".[dev]"
|
|
172
|
+
python -m unittest discover -s tests
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Pipelines can be tested declaratively:
|
|
176
|
+
|
|
177
|
+
```python
|
|
178
|
+
from stageflow.testing import PipelineTestSpec, run_pipeline_test
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
Releases are tag-driven: the tag has to match `project.version` in
|
|
182
|
+
`pyproject.toml`, and the workflow publishes to PyPI.
|
|
183
|
+
|
|
184
|
+
## License
|
|
185
|
+
|
|
186
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
# StageFlow
|
|
4
|
+
|
|
5
|
+
**Pipelines described in JSON, executed in Python — and debuggable node by node.**
|
|
6
|
+
|
|
7
|
+
[](https://github.com/leo-need-more-coffee/stageflow/actions/workflows/tests.yml)
|
|
8
|
+
[](https://pypi.org/project/stageflow-framework/)
|
|
9
|
+
[](https://pypi.org/project/stageflow-framework/)
|
|
10
|
+
[](https://leo-need-more-coffee.github.io/stageflow/)
|
|
11
|
+
[](LICENSE)
|
|
12
|
+
|
|
13
|
+
[Documentation](https://leo-need-more-coffee.github.io/stageflow/) ·
|
|
14
|
+
[Tutorial](https://leo-need-more-coffee.github.io/stageflow/tutorial/) ·
|
|
15
|
+
[Документация на русском](https://leo-need-more-coffee.github.io/stageflow/ru/)
|
|
16
|
+
|
|
17
|
+
</div>
|
|
18
|
+
|
|
19
|
+
A pipeline is a graph of nodes in JSON. The work happens in stages — Python
|
|
20
|
+
classes you write. Between them travels one immutable frame of variables, and
|
|
21
|
+
everything the graph does with that frame is visible in the JSON: branching,
|
|
22
|
+
retries, error handling, concurrency, nested graphs.
|
|
23
|
+
|
|
24
|
+
Because the pipeline is data, it can be stored, diffed, generated, validated
|
|
25
|
+
before it runs — and drawn:
|
|
26
|
+
|
|
27
|
+

|
|
28
|
+
|
|
29
|
+
That is the [editor](https://github.com/leo-need-more-coffee/stageflow-ui): a
|
|
30
|
+
separate static page that draws and debugs a graph while this core executes it.
|
|
31
|
+
|
|
32
|
+
## Install
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install stageflow-framework
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Python 3.11+.
|
|
39
|
+
|
|
40
|
+
## A pipeline in 30 seconds
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
import asyncio
|
|
44
|
+
|
|
45
|
+
from stageflow import BaseStage, Pipeline, Session, register_stage
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@register_stage("HelloStage")
|
|
49
|
+
class HelloStage(BaseStage):
|
|
50
|
+
"""
|
|
51
|
+
description: "Greets whoever the pipeline points at"
|
|
52
|
+
arguments:
|
|
53
|
+
name: string
|
|
54
|
+
outputs:
|
|
55
|
+
greeting: string
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
async def run(self):
|
|
59
|
+
name = self.get_arguments().get("name", "world")
|
|
60
|
+
self.set_outputs({"greeting": f"Hello, {name}!"})
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
pipeline = Pipeline.from_dict({
|
|
64
|
+
"nodes": [
|
|
65
|
+
{"id": "start", "type": "entry",
|
|
66
|
+
"variables": {"user_name": "Alice"}, "next": "hello"},
|
|
67
|
+
{"id": "hello", "type": "stage", "stage": "HelloStage",
|
|
68
|
+
"arguments": {"vars": {"name": "user_name"}},
|
|
69
|
+
"outputs": {"greeting": "greeting"}, "next": "finish"},
|
|
70
|
+
{"id": "finish", "type": "terminal",
|
|
71
|
+
"result": {"status": "ok"}, "artifacts": ["greeting"]},
|
|
72
|
+
],
|
|
73
|
+
})
|
|
74
|
+
pipeline.validate()
|
|
75
|
+
|
|
76
|
+
result = asyncio.run(Session(id="demo", pipeline=pipeline).run())
|
|
77
|
+
print(result.result) # {'status': 'ok'}
|
|
78
|
+
print(result.artifacts) # {'greeting': 'Hello, Alice!'}
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
The docstring is the stage's specification. `validate()` checks the graph
|
|
82
|
+
against it, so a wrong argument name is an error before anything runs.
|
|
83
|
+
|
|
84
|
+
## What the graph can do
|
|
85
|
+
|
|
86
|
+
| | |
|
|
87
|
+
|---|---|
|
|
88
|
+
| **Nine node types** | `entry`, `stage`, `condition`, `switch`, `parallel`, `try`, `map`, `subpipeline`, `terminal` |
|
|
89
|
+
| **One immutable frame** | variables travel along the path; a write produces a new frame, so branches never collide |
|
|
90
|
+
| **CEL expressions** | in conditions, in switch cases, and in any argument or output through the `.$` suffix |
|
|
91
|
+
| **Errors as roads** | `retry` on a node, `try`/`except` over a region of the graph derived from its shape |
|
|
92
|
+
| **Real concurrency** | `parallel` branches with their own frames and explicit merge rules |
|
|
93
|
+
| **Loops over data** | `map` runs a region of the graph once per element, sequentially or at once |
|
|
94
|
+
| **Nested graphs** | a `subpipeline` starts with a fresh frame and returns artifacts |
|
|
95
|
+
| **Gradual typing** | declare the variables that matter; checked at validation and on every write |
|
|
96
|
+
| **Step debugging** | stop between nodes, read and edit the frame, replay the event stream |
|
|
97
|
+
|
|
98
|
+
## Documentation
|
|
99
|
+
|
|
100
|
+
The [tutorial](https://leo-need-more-coffee.github.io/stageflow/tutorial/)
|
|
101
|
+
builds one working pipeline step by step, with screenshots from the editor.
|
|
102
|
+
The reference covers the rest:
|
|
103
|
+
|
|
104
|
+
[Quick start](https://leo-need-more-coffee.github.io/stageflow/quick-start/) ·
|
|
105
|
+
[Node types](https://leo-need-more-coffee.github.io/stageflow/node-types/) ·
|
|
106
|
+
[Data model](https://leo-need-more-coffee.github.io/stageflow/data-model/) ·
|
|
107
|
+
[Expressions](https://leo-need-more-coffee.github.io/stageflow/expressions/) ·
|
|
108
|
+
[Errors](https://leo-need-more-coffee.github.io/stageflow/errors/) ·
|
|
109
|
+
[Variable typing](https://leo-need-more-coffee.github.io/stageflow/variable-typing/) ·
|
|
110
|
+
[Session control](https://leo-need-more-coffee.github.io/stageflow/session-control/) ·
|
|
111
|
+
[Step debugging](https://leo-need-more-coffee.github.io/stageflow/step-debugging/)
|
|
112
|
+
|
|
113
|
+
One page per node type:
|
|
114
|
+
[entry](https://leo-need-more-coffee.github.io/stageflow/entry-node/) ·
|
|
115
|
+
[stage](https://leo-need-more-coffee.github.io/stageflow/stage-node/) ·
|
|
116
|
+
[condition](https://leo-need-more-coffee.github.io/stageflow/condition-node/) ·
|
|
117
|
+
[switch](https://leo-need-more-coffee.github.io/stageflow/switch-node/) ·
|
|
118
|
+
[parallel](https://leo-need-more-coffee.github.io/stageflow/parallel-branches/) ·
|
|
119
|
+
[try](https://leo-need-more-coffee.github.io/stageflow/try-node/) ·
|
|
120
|
+
[map](https://leo-need-more-coffee.github.io/stageflow/map-node/) ·
|
|
121
|
+
[subpipeline](https://leo-need-more-coffee.github.io/stageflow/subpipeline-node/) ·
|
|
122
|
+
[terminal](https://leo-need-more-coffee.github.io/stageflow/terminal-node/)
|
|
123
|
+
|
|
124
|
+
Sources are in [`docs/`](docs) (`*.md` English, `*.ru.md` Russian) and publish
|
|
125
|
+
themselves on every push to `main`.
|
|
126
|
+
|
|
127
|
+
## The rest of the project
|
|
128
|
+
|
|
129
|
+
| Repository | What it is |
|
|
130
|
+
|---|---|
|
|
131
|
+
| **stageflow** | this one: the core that runs the pipelines |
|
|
132
|
+
| [stageflow-ui](https://github.com/leo-need-more-coffee/stageflow-ui) | the editor: a static page that draws and debugs a graph |
|
|
133
|
+
| [stageflow-example](https://github.com/leo-need-more-coffee/stageflow-example) | a working backend for the editor: a support bot in four pipelines |
|
|
134
|
+
|
|
135
|
+
## Development
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
pip install -e ".[dev]"
|
|
139
|
+
python -m unittest discover -s tests
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Pipelines can be tested declaratively:
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
from stageflow.testing import PipelineTestSpec, run_pipeline_test
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Releases are tag-driven: the tag has to match `project.version` in
|
|
149
|
+
`pyproject.toml`, and the workflow publishes to PyPI.
|
|
150
|
+
|
|
151
|
+
## License
|
|
152
|
+
|
|
153
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "stageflow-framework"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.10.0"
|
|
8
8
|
description = "StageFlow: pipeline framework for stages"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -41,6 +41,10 @@ dev = [
|
|
|
41
41
|
"build",
|
|
42
42
|
"twine",
|
|
43
43
|
]
|
|
44
|
+
docs = [
|
|
45
|
+
"mkdocs-material",
|
|
46
|
+
"mkdocs-static-i18n",
|
|
47
|
+
]
|
|
44
48
|
|
|
45
49
|
[tool.setuptools.packages.find]
|
|
46
50
|
where = ["."]
|
|
@@ -9,6 +9,7 @@ from .core import (
|
|
|
9
9
|
Event,
|
|
10
10
|
EventSpec,
|
|
11
11
|
InputSpec,
|
|
12
|
+
MapNode,
|
|
12
13
|
Node,
|
|
13
14
|
ParallelNode,
|
|
14
15
|
Pipeline,
|
|
@@ -67,6 +68,7 @@ __all__ = [
|
|
|
67
68
|
"ConditionNode",
|
|
68
69
|
"SwitchNode",
|
|
69
70
|
"ParallelNode",
|
|
71
|
+
"MapNode",
|
|
70
72
|
"SubPipelineNode",
|
|
71
73
|
"TerminalNode",
|
|
72
74
|
"Retrier",
|
|
@@ -8,7 +8,7 @@ from ..exceptions import StageContractError
|
|
|
8
8
|
def require_list(stage_name: str, field: str, value: Any) -> list:
|
|
9
9
|
if not isinstance(value, list):
|
|
10
10
|
raise StageContractError(
|
|
11
|
-
f"{stage_name}: '{field}'
|
|
11
|
+
f"{stage_name}: '{field}' must be a list, got {type(value).__name__}"
|
|
12
12
|
)
|
|
13
13
|
return value
|
|
14
14
|
|
|
@@ -16,7 +16,7 @@ def require_list(stage_name: str, field: str, value: Any) -> list:
|
|
|
16
16
|
def require_dict(stage_name: str, field: str, value: Any) -> dict:
|
|
17
17
|
if not isinstance(value, dict):
|
|
18
18
|
raise StageContractError(
|
|
19
|
-
f"{stage_name}: '{field}'
|
|
19
|
+
f"{stage_name}: '{field}' must be an object, got {type(value).__name__}"
|
|
20
20
|
)
|
|
21
21
|
return value
|
|
22
22
|
|
|
@@ -24,12 +24,12 @@ def require_dict(stage_name: str, field: str, value: Any) -> dict:
|
|
|
24
24
|
def require_number(stage_name: str, field: str, value: Any) -> int | float:
|
|
25
25
|
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
26
26
|
raise StageContractError(
|
|
27
|
-
f"{stage_name}: '{field}'
|
|
27
|
+
f"{stage_name}: '{field}' must be a number, got {type(value).__name__}"
|
|
28
28
|
)
|
|
29
29
|
return value
|
|
30
30
|
|
|
31
31
|
|
|
32
32
|
def require_present(stage_name: str, field: str, value: Any) -> Any:
|
|
33
33
|
if value is None:
|
|
34
|
-
raise StageContractError(f"{stage_name}:
|
|
34
|
+
raise StageContractError(f"{stage_name}: argument '{field}' is required")
|
|
35
35
|
return value
|
|
@@ -29,7 +29,7 @@ class AppendListStage(BaseStage):
|
|
|
29
29
|
async def run(self):
|
|
30
30
|
args = self.get_arguments()
|
|
31
31
|
if "value" not in args:
|
|
32
|
-
raise StageContractError("AppendListStage:
|
|
32
|
+
raise StageContractError("AppendListStage: argument 'value' is required")
|
|
33
33
|
value = args["value"]
|
|
34
34
|
base = require_list("AppendListStage", "list", args.get("list") or [])
|
|
35
35
|
self.set_outputs({"list": [*base, value]})
|
|
@@ -63,7 +63,7 @@ class ExtendListStage(BaseStage):
|
|
|
63
63
|
items = args.get("items", [])
|
|
64
64
|
if not isinstance(items, (list, tuple)):
|
|
65
65
|
raise StageContractError(
|
|
66
|
-
f"ExtendListStage: 'items'
|
|
66
|
+
f"ExtendListStage: 'items' must be a list, got {type(items).__name__}"
|
|
67
67
|
)
|
|
68
68
|
self.set_outputs({"list": [*base, *items]})
|
|
69
69
|
|
|
@@ -95,7 +95,7 @@ class FilterListStage(BaseStage):
|
|
|
95
95
|
args = self.get_arguments()
|
|
96
96
|
condition = args.pop("condition", None)
|
|
97
97
|
if condition is None:
|
|
98
|
-
raise StageContractError("FilterListStage:
|
|
98
|
+
raise StageContractError("FilterListStage: argument 'condition' is required")
|
|
99
99
|
items = require_list("FilterListStage", "items", args.pop("items", []))
|
|
100
100
|
scope = Context(vars=args)
|
|
101
101
|
result = [item for item in items if self.session.cel.eval(condition, scope, item=item)]
|
|
@@ -159,13 +159,13 @@ class PopListStage(BaseStage):
|
|
|
159
159
|
args = self.get_arguments()
|
|
160
160
|
items = require_list("PopListStage", "items", args.get("items", []))
|
|
161
161
|
if not items:
|
|
162
|
-
raise StageContractError("PopListStage:
|
|
162
|
+
raise StageContractError("PopListStage: cannot pop from an empty list")
|
|
163
163
|
index = args.get("index", -1)
|
|
164
164
|
remaining = list(items)
|
|
165
165
|
try:
|
|
166
166
|
popped = remaining.pop(index)
|
|
167
167
|
except IndexError:
|
|
168
168
|
raise StageContractError(
|
|
169
|
-
f"PopListStage:
|
|
169
|
+
f"PopListStage: index {index} is out of range for a list of length {len(items)}"
|
|
170
170
|
) from None
|
|
171
171
|
self.set_outputs({"list": remaining, "popped": popped})
|
|
@@ -9,13 +9,13 @@ class _NameOnlyFormatter(string.Formatter):
|
|
|
9
9
|
def get_field(self, field_name, args, kwargs):
|
|
10
10
|
if not field_name.isidentifier():
|
|
11
11
|
raise StageContractError(
|
|
12
|
-
f"TemplateStage:
|
|
13
|
-
"
|
|
12
|
+
f"TemplateStage: placeholder '{{{field_name}}}' is not an argument name; "
|
|
13
|
+
"attributes, indexes and positional numbers are not allowed"
|
|
14
14
|
)
|
|
15
15
|
if field_name not in kwargs:
|
|
16
16
|
raise StageContractError(
|
|
17
|
-
f"TemplateStage:
|
|
18
|
-
f"
|
|
17
|
+
f"TemplateStage: template references '{field_name}', "
|
|
18
|
+
f"but there is no such argument (available: {sorted(kwargs)})"
|
|
19
19
|
)
|
|
20
20
|
return kwargs[field_name], field_name
|
|
21
21
|
|
|
@@ -23,7 +23,7 @@ class SetValueStage(BaseStage):
|
|
|
23
23
|
async def run(self):
|
|
24
24
|
args = self.get_arguments()
|
|
25
25
|
if "value" not in args:
|
|
26
|
-
raise StageContractError("SetValueStage:
|
|
26
|
+
raise StageContractError("SetValueStage: argument 'value' is required")
|
|
27
27
|
self.set_outputs({"value": args["value"]})
|
|
28
28
|
|
|
29
29
|
|
|
@@ -6,6 +6,7 @@ from .inputs import InputHub
|
|
|
6
6
|
from .nodes import (
|
|
7
7
|
ConditionNode,
|
|
8
8
|
EntryNode,
|
|
9
|
+
MapNode,
|
|
9
10
|
Node,
|
|
10
11
|
ParallelNode,
|
|
11
12
|
Retrier,
|
|
@@ -48,6 +49,7 @@ __all__ = [
|
|
|
48
49
|
"ExceptHandler",
|
|
49
50
|
"EntryNode",
|
|
50
51
|
"StageNode",
|
|
52
|
+
"MapNode",
|
|
51
53
|
"ConditionNode",
|
|
52
54
|
"SwitchNode",
|
|
53
55
|
"ParallelNode",
|
|
@@ -28,8 +28,8 @@ class CelEngine:
|
|
|
28
28
|
self._compiled: dict[str, Any] = {}
|
|
29
29
|
if _BACKEND is None: # pragma: no cover
|
|
30
30
|
raise ExpressionError(
|
|
31
|
-
"
|
|
32
|
-
"'common-expression-language' (
|
|
31
|
+
"No CEL backend found. Install "
|
|
32
|
+
"'common-expression-language' (recommended) or 'cel-python'."
|
|
33
33
|
)
|
|
34
34
|
|
|
35
35
|
@property
|
|
@@ -46,7 +46,7 @@ class CelEngine:
|
|
|
46
46
|
env = _celpy.Environment()
|
|
47
47
|
program = env.program(env.compile(expr))
|
|
48
48
|
except Exception as exc: # noqa: BLE001
|
|
49
|
-
raise ExpressionError(f"
|
|
49
|
+
raise ExpressionError(f"Cannot compile CEL {expr!r}: {exc}") from exc
|
|
50
50
|
self._compiled[expr] = program
|
|
51
51
|
return program
|
|
52
52
|
|
|
@@ -72,4 +72,4 @@ class CelEngine:
|
|
|
72
72
|
return program.execute(activation)
|
|
73
73
|
return program.evaluate(activation) # pragma: no cover
|
|
74
74
|
except Exception as exc: # noqa: BLE001
|
|
75
|
-
raise ExpressionError(f"
|
|
75
|
+
raise ExpressionError(f"Cannot evaluate CEL {expr!r}: {exc}") from exc
|
|
@@ -39,8 +39,8 @@ class Context:
|
|
|
39
39
|
def from_dict(cls, data: dict[str, Any]) -> "Context":
|
|
40
40
|
if "local" in data and "vars" not in data:
|
|
41
41
|
raise PipelineDefinitionError(
|
|
42
|
-
"
|
|
43
|
-
"
|
|
42
|
+
"Snapshot contains the 'local' scope (pre-0.7.0); "
|
|
43
|
+
"the frame is now called 'vars'"
|
|
44
44
|
)
|
|
45
45
|
return cls(vars=data.get("vars", {}))
|
|
46
46
|
|
|
@@ -124,7 +124,7 @@ class StepDebugger:
|
|
|
124
124
|
for name, value in values.items():
|
|
125
125
|
try:
|
|
126
126
|
if types is not None:
|
|
127
|
-
types.check_write(name, value, f"
|
|
127
|
+
types.check_write(name, value, f"debugger before node '{self.node}'")
|
|
128
128
|
except Exception as exc: # noqa: BLE001
|
|
129
129
|
self._emit("var_rejected", name=name, error=str(exc))
|
|
130
130
|
continue
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from .base import Node, get_node_types, register_node
|
|
2
2
|
from .branching import ConditionNode, SwitchNode
|
|
3
3
|
from .entry import EntryNode
|
|
4
|
+
from .map_block import MapNode
|
|
4
5
|
from .parallel import ParallelNode
|
|
5
6
|
from .recovery import Retrier, run_with_retry
|
|
6
7
|
from .stage import StageNode
|
|
@@ -18,6 +19,7 @@ __all__ = [
|
|
|
18
19
|
"ExceptHandler",
|
|
19
20
|
"EntryNode",
|
|
20
21
|
"StageNode",
|
|
22
|
+
"MapNode",
|
|
21
23
|
"ConditionNode",
|
|
22
24
|
"SwitchNode",
|
|
23
25
|
"ParallelNode",
|
|
@@ -77,10 +77,10 @@ class Node:
|
|
|
77
77
|
errors: list[str] = []
|
|
78
78
|
for src, dst in self.expose.items():
|
|
79
79
|
malformed = False
|
|
80
|
-
for path, side in ((src, "
|
|
80
|
+
for path, side in ((src, "source"), (dst, "destination")):
|
|
81
81
|
if not isinstance(path, str) or not path.isidentifier():
|
|
82
82
|
errors.append(
|
|
83
|
-
f"{self.id}: expose {side} '{path}'
|
|
83
|
+
f"{self.id}: expose {side} '{path}' must be a variable name"
|
|
84
84
|
)
|
|
85
85
|
malformed = True
|
|
86
86
|
if malformed:
|
|
@@ -90,8 +90,8 @@ class Node:
|
|
|
90
90
|
dst_type = ts.declared(dst)
|
|
91
91
|
if src_type and dst_type and not ts.expose_compatible(src_type, dst_type):
|
|
92
92
|
errors.append(
|
|
93
|
-
f"{self.id}: expose {src} -> {dst}:
|
|
94
|
-
f"'{src_type}'
|
|
93
|
+
f"{self.id}: expose {src} -> {dst}: incompatible types "
|
|
94
|
+
f"'{src_type}' and '{dst_type}'"
|
|
95
95
|
)
|
|
96
96
|
return errors
|
|
97
97
|
|
|
@@ -12,7 +12,7 @@ if TYPE_CHECKING: # pragma: no cover
|
|
|
12
12
|
CEL_SUFFIX = ".$"
|
|
13
13
|
|
|
14
14
|
|
|
15
|
-
def
|
|
15
|
+
def normalize_bucket(bucket: Any) -> dict[str, str]:
|
|
16
16
|
if bucket is None:
|
|
17
17
|
return {}
|
|
18
18
|
if isinstance(bucket, list):
|
|
@@ -35,7 +35,7 @@ def resolve_arguments(
|
|
|
35
35
|
else:
|
|
36
36
|
kwargs[key] = value
|
|
37
37
|
|
|
38
|
-
for arg_name, ref in
|
|
38
|
+
for arg_name, ref in normalize_bucket(arguments.get("vars")).items():
|
|
39
39
|
if arg_name.endswith(CEL_SUFFIX):
|
|
40
40
|
kwargs[arg_name[: -len(CEL_SUFFIX)]] = cel.eval(ref, ctx, **extra)
|
|
41
41
|
else:
|
|
@@ -75,7 +75,7 @@ def apply_outputs(
|
|
|
75
75
|
dest = spec
|
|
76
76
|
if key not in output_ns:
|
|
77
77
|
raise StageOutputError(
|
|
78
|
-
f"
|
|
78
|
+
f"Stage did not return field '{key}' (available: {sorted(output_ns)})"
|
|
79
79
|
)
|
|
80
80
|
value = output_ns[key]
|
|
81
81
|
writes.append((dest, value))
|
|
@@ -33,9 +33,9 @@ class ConditionNode(Node):
|
|
|
33
33
|
def validate(self, pipeline: "Pipeline") -> list[str]:
|
|
34
34
|
errors = self._validate_common(pipeline)
|
|
35
35
|
if not pipeline.has_node(self.then):
|
|
36
|
-
errors.append(f"{self.id}: then '{self.then}'
|
|
36
|
+
errors.append(f"{self.id}: then '{self.then}' not found in the graph")
|
|
37
37
|
if self.else_ and not pipeline.has_node(self.else_):
|
|
38
|
-
errors.append(f"{self.id}: else '{self.else_}'
|
|
38
|
+
errors.append(f"{self.id}: else '{self.else_}' not found in the graph")
|
|
39
39
|
return errors
|
|
40
40
|
|
|
41
41
|
async def execute(self, session: "Session", ctx: Context) -> tuple[Node | None, Context]:
|
|
@@ -69,12 +69,12 @@ class SwitchNode(Node):
|
|
|
69
69
|
errors = self._validate_common(pipeline)
|
|
70
70
|
for case in self.cases:
|
|
71
71
|
if "when" not in case or "next" not in case:
|
|
72
|
-
errors.append(f"{self.id}:
|
|
72
|
+
errors.append(f"{self.id}: every case needs 'when' and 'next'")
|
|
73
73
|
continue
|
|
74
74
|
if not pipeline.has_node(case["next"]):
|
|
75
|
-
errors.append(f"{self.id}: case next '{case['next']}'
|
|
75
|
+
errors.append(f"{self.id}: case next '{case['next']}' not found in the graph")
|
|
76
76
|
if self.default and not pipeline.has_node(self.default):
|
|
77
|
-
errors.append(f"{self.id}: default '{self.default}'
|
|
77
|
+
errors.append(f"{self.id}: default '{self.default}' not found in the graph")
|
|
78
78
|
return errors
|
|
79
79
|
|
|
80
80
|
async def execute(self, session: "Session", ctx: Context) -> tuple[Node | None, Context]:
|