Flowpipe 1.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowpipe-1.0.2/LICENSE +21 -0
- flowpipe-1.0.2/PKG-INFO +290 -0
- flowpipe-1.0.2/README.md +266 -0
- flowpipe-1.0.2/flowpipe/__init__.py +10 -0
- flowpipe-1.0.2/flowpipe/errors.py +9 -0
- flowpipe-1.0.2/flowpipe/evaluator.py +269 -0
- flowpipe-1.0.2/flowpipe/event.py +51 -0
- flowpipe-1.0.2/flowpipe/graph.py +500 -0
- flowpipe-1.0.2/flowpipe/node.py +700 -0
- flowpipe-1.0.2/flowpipe/plug.py +469 -0
- flowpipe-1.0.2/flowpipe/utilities.py +149 -0
- flowpipe-1.0.2/pyproject.toml +54 -0
flowpipe-1.0.2/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2017 Paul Schweizer
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
flowpipe-1.0.2/PKG-INFO
ADDED
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: Flowpipe
|
|
3
|
+
Version: 1.0.2
|
|
4
|
+
Summary: A lightweight framework for flow-based programming in python.
|
|
5
|
+
License: MIT
|
|
6
|
+
Author: Paul Schweizer
|
|
7
|
+
Author-email: paulschweizer@gmx.net
|
|
8
|
+
Requires-Python: >=3.9
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Programming Language :: Python
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
19
|
+
Requires-Dist: ascii-canvas (>=2.0.0)
|
|
20
|
+
Project-URL: Documentation, https://flowpipe.readthedocs.io/en/latest/
|
|
21
|
+
Project-URL: Repository, https://github.com/PaulSchweizer/flowpipe
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
[](https://pypi.org/project/flowpipe/)
|
|
25
|
+
|
|
26
|
+
<!-- Pytest Coverage Comment:Begin -->
|
|
27
|
+
|
|
28
|
+
<a href="https://github.com/PaulSchweizer/flowpipe/blob/main/README.md"><img alt="Coverage" src="https://img.shields.io/badge/Coverage-100%25-brightgreen.svg" /></a><br/><details><summary>Coverage Report </summary><table><tr><th>File</th><th>Stmts</th><th>Miss</th><th>Cover</th></tr><tbody><tr><td><b>TOTAL</b></td><td><b>940</b></td><td><b>0</b></td><td><b>100%</b></td></tr></tbody></table></details>
|
|
29
|
+
|
|
30
|
+
<!-- Pytest Coverage Comment:End -->
|
|
31
|
+
|
|
32
|
+
[](LICENSE)  [](https://flowpipe.readthedocs.io/en/latest) [](https://github.com/psf/black)
|
|
33
|
+
|
|
34
|
+

|
|
35
|
+
|
|
36
|
+
# Flow-based Programming
|
|
37
|
+
|
|
38
|
+
A lightweight framework for flow-based programming in python.
|
|
39
|
+
|
|
40
|
+
```c
|
|
41
|
+
+-------------------+ +---------------------+
|
|
42
|
+
| Invite People | | Birthday Party |
|
|
43
|
+
|-------------------| |---------------------|
|
|
44
|
+
o amount<4> | +----->o attendees<> |
|
|
45
|
+
| people o---+ +--->o cake<> |
|
|
46
|
+
+-------------------+ | +---------------------+
|
|
47
|
+
|
|
|
48
|
+
+-------------------+ |
|
|
49
|
+
| Bake a cake | |
|
|
50
|
+
+-------------------+ |
|
|
51
|
+
o type<"Chocolate"> | |
|
|
52
|
+
| cake o-----+
|
|
53
|
+
+-------------------+
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Benefits:
|
|
57
|
+
|
|
58
|
+
- Visualize code
|
|
59
|
+
- Re-usability
|
|
60
|
+
- Streamlined code design
|
|
61
|
+
- Built-in concurrency
|
|
62
|
+
- Represent workflows one to one in the code
|
|
63
|
+
|
|
64
|
+
# Quick Example
|
|
65
|
+
|
|
66
|
+
Consider this simple example on how to represent the construction of a house with Flowpipe:
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
from flowpipe import Graph, INode, Node, InputPlug, OutputPlug
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class HireWorkers(INode):
|
|
73
|
+
"""A node can be derived from the INode interface.
|
|
74
|
+
|
|
75
|
+
The plugs are defined in the init method.
|
|
76
|
+
The compute method received the inputs from any connected upstream nodes.
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
def __init__(self, amount=None, **kwargs):
|
|
80
|
+
super(HireWorkers, self).__init__(**kwargs)
|
|
81
|
+
InputPlug('amount', self, amount)
|
|
82
|
+
OutputPlug('workers', self)
|
|
83
|
+
|
|
84
|
+
def compute(self, amount):
|
|
85
|
+
workers = ['John', 'Jane', 'Mike', 'Michelle']
|
|
86
|
+
print('{0} workers are hired to build the house.'.format(amount))
|
|
87
|
+
return {'workers.{0}'.format(i): workers[i] for i in range(amount)}
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@Node(outputs=['workers'])
|
|
91
|
+
def Build(workers, section):
|
|
92
|
+
"""A node can also be created by the Node decorator.outputs
|
|
93
|
+
|
|
94
|
+
The inputs to the function are turned into InputsPlugs, otuputs are defined
|
|
95
|
+
in the decorator itself. The wrapped function is used as the compute method.
|
|
96
|
+
"""
|
|
97
|
+
print('{0} are building the {1}'.format(', '.join(workers.values()), section))
|
|
98
|
+
return {'workers.{0}'.format(i): worker for i, worker in workers.items()}
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@Node()
|
|
102
|
+
def Party(attendees):
|
|
103
|
+
print('{0} and {1} are having a great party!'.format(
|
|
104
|
+
', '.join(list(attendees.values())[:-1]), list(attendees.values())[-1]))
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
# Create a graph with the necessary nodes
|
|
108
|
+
graph = Graph(name='How to build a house')
|
|
109
|
+
workers = HireWorkers(graph=graph, amount=4)
|
|
110
|
+
build_walls = Build(graph=graph, name='Build Walls', section='walls')
|
|
111
|
+
build_roof = Build(graph=graph, name='Build Roof', section='roof')
|
|
112
|
+
party = Party(graph=graph, name='Housewarming Party')
|
|
113
|
+
|
|
114
|
+
# Wire up the connections between the nodes
|
|
115
|
+
workers.outputs['workers']['0'].connect(build_walls.inputs['workers']['0'])
|
|
116
|
+
workers.outputs['workers']['1'].connect(build_walls.inputs['workers']['1'])
|
|
117
|
+
workers.outputs['workers']['2'].connect(build_roof.inputs['workers']['0'])
|
|
118
|
+
workers.outputs['workers']['3'].connect(build_roof.inputs['workers']['1'])
|
|
119
|
+
build_walls.outputs['workers']['0'] >> party.inputs['attendees']['0']
|
|
120
|
+
build_walls.outputs['workers']['1'] >> party.inputs['attendees']['2']
|
|
121
|
+
build_roof.outputs['workers']['0'] >> party.inputs['attendees']['1']
|
|
122
|
+
build_roof.outputs['workers']['1'] >> party.inputs['attendees']['3']
|
|
123
|
+
party.inputs['attendees']['4'].value = 'Homeowner'
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
Visualize the code as a graph or as a listing:
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
print(graph.name)
|
|
130
|
+
print(graph)
|
|
131
|
+
print(graph.list_repr())
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Output:
|
|
135
|
+
|
|
136
|
+
```c
|
|
137
|
+
How to build a house
|
|
138
|
+
+------------------------+ +------------------------+ +---------------------------+
|
|
139
|
+
| HireWorkers | | Build Roof | | Housewarming Party |
|
|
140
|
+
|------------------------| |------------------------| |---------------------------|
|
|
141
|
+
o amount<4> | o section<"roof"> | % attendees |
|
|
142
|
+
| workers % % workers | +--->o attendees.0<> |
|
|
143
|
+
| workers.0 o-----+--->o workers.0<> | |--->o attendees.1<> |
|
|
144
|
+
| workers.1 o-----|--->o workers.1<> | |--->o attendees.2<> |
|
|
145
|
+
| workers.2 o-----| | workers % |--->o attendees.3<> |
|
|
146
|
+
| workers.3 o-----| | workers.0 o-----| o attendees.4<"Homeowner> |
|
|
147
|
+
+------------------------+ | | workers.1 o-----| +---------------------------+
|
|
148
|
+
| +------------------------+ |
|
|
149
|
+
| +------------------------+ |
|
|
150
|
+
| | Build Walls | |
|
|
151
|
+
| |------------------------| |
|
|
152
|
+
| o section<"walls"> | |
|
|
153
|
+
| % workers | |
|
|
154
|
+
+--->o workers.0<> | |
|
|
155
|
+
+--->o workers.1<> | |
|
|
156
|
+
| workers % |
|
|
157
|
+
| workers.0 o-----+
|
|
158
|
+
| workers.1 o-----+
|
|
159
|
+
+------------------------+
|
|
160
|
+
|
|
161
|
+
Build a House
|
|
162
|
+
HireWorkers
|
|
163
|
+
[i] amount: 4
|
|
164
|
+
[o] workers
|
|
165
|
+
[o] workers.0 >> Build Walls.workers.0
|
|
166
|
+
[o] workers.1 >> Build Walls.workers.1
|
|
167
|
+
[o] workers.2 >> Build Roof.workers.0
|
|
168
|
+
[o] workers.3 >> Build Roof.workers.1
|
|
169
|
+
Build Roof
|
|
170
|
+
[i] section: "roof"
|
|
171
|
+
[i] workers
|
|
172
|
+
[i] workers.0 << HireWorkers.workers.2
|
|
173
|
+
[i] workers.1 << HireWorkers.workers.3
|
|
174
|
+
[o] workers
|
|
175
|
+
[o] workers.0 >> Housewarming Party.attendees.1
|
|
176
|
+
[o] workers.1 >> Housewarming Party.attendees.3
|
|
177
|
+
Build Walls
|
|
178
|
+
[i] section: "walls"
|
|
179
|
+
[i] workers
|
|
180
|
+
[i] workers.0 << HireWorkers.workers.0
|
|
181
|
+
[i] workers.1 << HireWorkers.workers.1
|
|
182
|
+
[o] workers
|
|
183
|
+
[o] workers.0 >> Housewarming Party.attendees.0
|
|
184
|
+
[o] workers.1 >> Housewarming Party.attendees.2
|
|
185
|
+
Housewarming Party
|
|
186
|
+
[i] attendees
|
|
187
|
+
[i] attendees.0 << Build Walls.workers.0
|
|
188
|
+
[i] attendees.1 << Build Roof.workers.0
|
|
189
|
+
[i] attendees.2 << Build Walls.workers.1
|
|
190
|
+
[i] attendees.3 << Build Roof.workers.1
|
|
191
|
+
[i] attendees.4: "Homeowner"
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
Now build the house:
|
|
195
|
+
|
|
196
|
+
```python
|
|
197
|
+
graph.evaluate(mode='threading') # Options are linear, threading and multiprocessing
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
Output:
|
|
201
|
+
|
|
202
|
+
```c
|
|
203
|
+
4 workers are hired to build the house.
|
|
204
|
+
Michelle, Mike are building the roof
|
|
205
|
+
Jane, John are building the walls
|
|
206
|
+
Mike, John, Michelle, Jane and Homeowner are having a great party!
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
(Note: for more elaborate evaluation schemes, see [Evaluators](#evaluators))
|
|
210
|
+
|
|
211
|
+
We now know how to throw a party, so let's invite some people and re-use these skills for a birthday:
|
|
212
|
+
|
|
213
|
+
```python
|
|
214
|
+
graph = Graph(name='How to throw a birthday party')
|
|
215
|
+
|
|
216
|
+
@Node(outputs=['people'])
|
|
217
|
+
def InvitePeople(amount):
|
|
218
|
+
people = ['John', 'Jane', 'Mike', 'Michelle']
|
|
219
|
+
d = {'people.{0}'.format(i): people[i] for i in range(amount)}
|
|
220
|
+
d['people'] = {people[i]: people[i] for i in range(amount)}
|
|
221
|
+
return d
|
|
222
|
+
|
|
223
|
+
invite = InvitePeople(graph=graph, amount=4)
|
|
224
|
+
birthday_party = Party(graph=graph, name='Birthday Party')
|
|
225
|
+
invite.outputs['people'] >> birthday_party.inputs['attendees']
|
|
226
|
+
|
|
227
|
+
print(graph.name)
|
|
228
|
+
print(graph)
|
|
229
|
+
graph.evaluate()
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
Output:
|
|
233
|
+
|
|
234
|
+
```c
|
|
235
|
+
How to throw a birthday party
|
|
236
|
+
+-------------------+ +---------------------+
|
|
237
|
+
| InvitePeople | | Birthday Party |
|
|
238
|
+
|-------------------| |---------------------|
|
|
239
|
+
o amount<4> | +--->o attendees<> |
|
|
240
|
+
| people o-----+ +---------------------+
|
|
241
|
+
+-------------------+
|
|
242
|
+
|
|
243
|
+
Jane, Michelle, Mike and John are having a great party!
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
## More Examples
|
|
247
|
+
|
|
248
|
+
There are more examples for common use cases of flowpipe:
|
|
249
|
+
|
|
250
|
+
The code for these examples:
|
|
251
|
+
[house_and_birthday.py](examples/house_and_birthday.py)!
|
|
252
|
+
|
|
253
|
+
Another simple example:
|
|
254
|
+
[world_clock.py](examples/world_clock.py)!
|
|
255
|
+
|
|
256
|
+
How to make use of nested subgraphs:
|
|
257
|
+
[nested_graphs.py](examples/nested_graphs.py)!
|
|
258
|
+
|
|
259
|
+
Using the command pattern with flowpipe successfully:
|
|
260
|
+
[workflow_design_pattern.py](examples/workflow_design_pattern.py)!
|
|
261
|
+
|
|
262
|
+
Use flowpipe on a remote cluster of machines, commonly refered to as a "render farm" in the VFX/Animation industry:
|
|
263
|
+
[vfx_render_farm_conversion.py](examples/vfx_render_farm_conversion.py)!
|
|
264
|
+
|
|
265
|
+
An example graph showcasing a common workflow encountered in the VFX/Animation industry:
|
|
266
|
+
[vfx_rendering.py](examples/vfx_rendering.py)!
|
|
267
|
+
|
|
268
|
+
## VFX Pipeline
|
|
269
|
+
|
|
270
|
+
If you are working in the VFX/Animation industry, please check out this extensive guide on how to use [flowpipe in a vfx pipeline](flowpipe-for-vfx-pipelines.md)!
|
|
271
|
+
|
|
272
|
+
# Evaluators
|
|
273
|
+
|
|
274
|
+
If your nodes just need sequential, threaded or multiprocessing evaluation, the `Graph.evaluate()` method will serve you just fine. If you want to take more control over the way your Graph is being evaluated, `Evaluators` are for you. This can also be used to add, e.g. logging or tracing to node evaluation.
|
|
275
|
+
|
|
276
|
+
Evaluators allow you to take control of node evaluation order, or their scheduling.
|
|
277
|
+
See `flowpipe/evaluator.py` to see the `Graph.evaluate()` method's evaluation schemes.
|
|
278
|
+
|
|
279
|
+
To use a custom evaluator, subclass `flowpipe.evaluator.Evaluator`, and provide at least an `_evaluate_nodes(self, nodes)` method.
|
|
280
|
+
This method should take a list of nodes and call their respective `node.evalaute()` methods (along with any other task you want to do for each node being evaluated).
|
|
281
|
+
To use a cusom evaluator, create it and call its `Evalator.evaluate()` method with the Graph to evaluate as an argument:
|
|
282
|
+
|
|
283
|
+
```py
|
|
284
|
+
from flowpipe.evaluators import LinearEvaluator
|
|
285
|
+
|
|
286
|
+
# assuming you created a graph to evaluate above, called `graph`
|
|
287
|
+
lin_eval = LinearEvaluator()
|
|
288
|
+
lin_eval.evaluate(graph)
|
|
289
|
+
```
|
|
290
|
+
|
flowpipe-1.0.2/README.md
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
[](https://pypi.org/project/flowpipe/)
|
|
2
|
+
|
|
3
|
+
<!-- Pytest Coverage Comment:Begin -->
|
|
4
|
+
|
|
5
|
+
<a href="https://github.com/PaulSchweizer/flowpipe/blob/main/README.md"><img alt="Coverage" src="https://img.shields.io/badge/Coverage-100%25-brightgreen.svg" /></a><br/><details><summary>Coverage Report </summary><table><tr><th>File</th><th>Stmts</th><th>Miss</th><th>Cover</th></tr><tbody><tr><td><b>TOTAL</b></td><td><b>940</b></td><td><b>0</b></td><td><b>100%</b></td></tr></tbody></table></details>
|
|
6
|
+
|
|
7
|
+
<!-- Pytest Coverage Comment:End -->
|
|
8
|
+
|
|
9
|
+
[](LICENSE)  [](https://flowpipe.readthedocs.io/en/latest) [](https://github.com/psf/black)
|
|
10
|
+
|
|
11
|
+

|
|
12
|
+
|
|
13
|
+
# Flow-based Programming
|
|
14
|
+
|
|
15
|
+
A lightweight framework for flow-based programming in python.
|
|
16
|
+
|
|
17
|
+
```c
|
|
18
|
+
+-------------------+ +---------------------+
|
|
19
|
+
| Invite People | | Birthday Party |
|
|
20
|
+
|-------------------| |---------------------|
|
|
21
|
+
o amount<4> | +----->o attendees<> |
|
|
22
|
+
| people o---+ +--->o cake<> |
|
|
23
|
+
+-------------------+ | +---------------------+
|
|
24
|
+
|
|
|
25
|
+
+-------------------+ |
|
|
26
|
+
| Bake a cake | |
|
|
27
|
+
+-------------------+ |
|
|
28
|
+
o type<"Chocolate"> | |
|
|
29
|
+
| cake o-----+
|
|
30
|
+
+-------------------+
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Benefits:
|
|
34
|
+
|
|
35
|
+
- Visualize code
|
|
36
|
+
- Re-usability
|
|
37
|
+
- Streamlined code design
|
|
38
|
+
- Built-in concurrency
|
|
39
|
+
- Represent workflows one to one in the code
|
|
40
|
+
|
|
41
|
+
# Quick Example
|
|
42
|
+
|
|
43
|
+
Consider this simple example on how to represent the construction of a house with Flowpipe:
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
from flowpipe import Graph, INode, Node, InputPlug, OutputPlug
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class HireWorkers(INode):
|
|
50
|
+
"""A node can be derived from the INode interface.
|
|
51
|
+
|
|
52
|
+
The plugs are defined in the init method.
|
|
53
|
+
The compute method received the inputs from any connected upstream nodes.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
def __init__(self, amount=None, **kwargs):
|
|
57
|
+
super(HireWorkers, self).__init__(**kwargs)
|
|
58
|
+
InputPlug('amount', self, amount)
|
|
59
|
+
OutputPlug('workers', self)
|
|
60
|
+
|
|
61
|
+
def compute(self, amount):
|
|
62
|
+
workers = ['John', 'Jane', 'Mike', 'Michelle']
|
|
63
|
+
print('{0} workers are hired to build the house.'.format(amount))
|
|
64
|
+
return {'workers.{0}'.format(i): workers[i] for i in range(amount)}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@Node(outputs=['workers'])
|
|
68
|
+
def Build(workers, section):
|
|
69
|
+
"""A node can also be created by the Node decorator.outputs
|
|
70
|
+
|
|
71
|
+
The inputs to the function are turned into InputsPlugs, otuputs are defined
|
|
72
|
+
in the decorator itself. The wrapped function is used as the compute method.
|
|
73
|
+
"""
|
|
74
|
+
print('{0} are building the {1}'.format(', '.join(workers.values()), section))
|
|
75
|
+
return {'workers.{0}'.format(i): worker for i, worker in workers.items()}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@Node()
|
|
79
|
+
def Party(attendees):
|
|
80
|
+
print('{0} and {1} are having a great party!'.format(
|
|
81
|
+
', '.join(list(attendees.values())[:-1]), list(attendees.values())[-1]))
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
# Create a graph with the necessary nodes
|
|
85
|
+
graph = Graph(name='How to build a house')
|
|
86
|
+
workers = HireWorkers(graph=graph, amount=4)
|
|
87
|
+
build_walls = Build(graph=graph, name='Build Walls', section='walls')
|
|
88
|
+
build_roof = Build(graph=graph, name='Build Roof', section='roof')
|
|
89
|
+
party = Party(graph=graph, name='Housewarming Party')
|
|
90
|
+
|
|
91
|
+
# Wire up the connections between the nodes
|
|
92
|
+
workers.outputs['workers']['0'].connect(build_walls.inputs['workers']['0'])
|
|
93
|
+
workers.outputs['workers']['1'].connect(build_walls.inputs['workers']['1'])
|
|
94
|
+
workers.outputs['workers']['2'].connect(build_roof.inputs['workers']['0'])
|
|
95
|
+
workers.outputs['workers']['3'].connect(build_roof.inputs['workers']['1'])
|
|
96
|
+
build_walls.outputs['workers']['0'] >> party.inputs['attendees']['0']
|
|
97
|
+
build_walls.outputs['workers']['1'] >> party.inputs['attendees']['2']
|
|
98
|
+
build_roof.outputs['workers']['0'] >> party.inputs['attendees']['1']
|
|
99
|
+
build_roof.outputs['workers']['1'] >> party.inputs['attendees']['3']
|
|
100
|
+
party.inputs['attendees']['4'].value = 'Homeowner'
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Visualize the code as a graph or as a listing:
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
print(graph.name)
|
|
107
|
+
print(graph)
|
|
108
|
+
print(graph.list_repr())
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Output:
|
|
112
|
+
|
|
113
|
+
```c
|
|
114
|
+
How to build a house
|
|
115
|
+
+------------------------+ +------------------------+ +---------------------------+
|
|
116
|
+
| HireWorkers | | Build Roof | | Housewarming Party |
|
|
117
|
+
|------------------------| |------------------------| |---------------------------|
|
|
118
|
+
o amount<4> | o section<"roof"> | % attendees |
|
|
119
|
+
| workers % % workers | +--->o attendees.0<> |
|
|
120
|
+
| workers.0 o-----+--->o workers.0<> | |--->o attendees.1<> |
|
|
121
|
+
| workers.1 o-----|--->o workers.1<> | |--->o attendees.2<> |
|
|
122
|
+
| workers.2 o-----| | workers % |--->o attendees.3<> |
|
|
123
|
+
| workers.3 o-----| | workers.0 o-----| o attendees.4<"Homeowner> |
|
|
124
|
+
+------------------------+ | | workers.1 o-----| +---------------------------+
|
|
125
|
+
| +------------------------+ |
|
|
126
|
+
| +------------------------+ |
|
|
127
|
+
| | Build Walls | |
|
|
128
|
+
| |------------------------| |
|
|
129
|
+
| o section<"walls"> | |
|
|
130
|
+
| % workers | |
|
|
131
|
+
+--->o workers.0<> | |
|
|
132
|
+
+--->o workers.1<> | |
|
|
133
|
+
| workers % |
|
|
134
|
+
| workers.0 o-----+
|
|
135
|
+
| workers.1 o-----+
|
|
136
|
+
+------------------------+
|
|
137
|
+
|
|
138
|
+
Build a House
|
|
139
|
+
HireWorkers
|
|
140
|
+
[i] amount: 4
|
|
141
|
+
[o] workers
|
|
142
|
+
[o] workers.0 >> Build Walls.workers.0
|
|
143
|
+
[o] workers.1 >> Build Walls.workers.1
|
|
144
|
+
[o] workers.2 >> Build Roof.workers.0
|
|
145
|
+
[o] workers.3 >> Build Roof.workers.1
|
|
146
|
+
Build Roof
|
|
147
|
+
[i] section: "roof"
|
|
148
|
+
[i] workers
|
|
149
|
+
[i] workers.0 << HireWorkers.workers.2
|
|
150
|
+
[i] workers.1 << HireWorkers.workers.3
|
|
151
|
+
[o] workers
|
|
152
|
+
[o] workers.0 >> Housewarming Party.attendees.1
|
|
153
|
+
[o] workers.1 >> Housewarming Party.attendees.3
|
|
154
|
+
Build Walls
|
|
155
|
+
[i] section: "walls"
|
|
156
|
+
[i] workers
|
|
157
|
+
[i] workers.0 << HireWorkers.workers.0
|
|
158
|
+
[i] workers.1 << HireWorkers.workers.1
|
|
159
|
+
[o] workers
|
|
160
|
+
[o] workers.0 >> Housewarming Party.attendees.0
|
|
161
|
+
[o] workers.1 >> Housewarming Party.attendees.2
|
|
162
|
+
Housewarming Party
|
|
163
|
+
[i] attendees
|
|
164
|
+
[i] attendees.0 << Build Walls.workers.0
|
|
165
|
+
[i] attendees.1 << Build Roof.workers.0
|
|
166
|
+
[i] attendees.2 << Build Walls.workers.1
|
|
167
|
+
[i] attendees.3 << Build Roof.workers.1
|
|
168
|
+
[i] attendees.4: "Homeowner"
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
Now build the house:
|
|
172
|
+
|
|
173
|
+
```python
|
|
174
|
+
graph.evaluate(mode='threading') # Options are linear, threading and multiprocessing
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
Output:
|
|
178
|
+
|
|
179
|
+
```c
|
|
180
|
+
4 workers are hired to build the house.
|
|
181
|
+
Michelle, Mike are building the roof
|
|
182
|
+
Jane, John are building the walls
|
|
183
|
+
Mike, John, Michelle, Jane and Homeowner are having a great party!
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
(Note: for more elaborate evaluation schemes, see [Evaluators](#evaluators))
|
|
187
|
+
|
|
188
|
+
We now know how to throw a party, so let's invite some people and re-use these skills for a birthday:
|
|
189
|
+
|
|
190
|
+
```python
|
|
191
|
+
graph = Graph(name='How to throw a birthday party')
|
|
192
|
+
|
|
193
|
+
@Node(outputs=['people'])
|
|
194
|
+
def InvitePeople(amount):
|
|
195
|
+
people = ['John', 'Jane', 'Mike', 'Michelle']
|
|
196
|
+
d = {'people.{0}'.format(i): people[i] for i in range(amount)}
|
|
197
|
+
d['people'] = {people[i]: people[i] for i in range(amount)}
|
|
198
|
+
return d
|
|
199
|
+
|
|
200
|
+
invite = InvitePeople(graph=graph, amount=4)
|
|
201
|
+
birthday_party = Party(graph=graph, name='Birthday Party')
|
|
202
|
+
invite.outputs['people'] >> birthday_party.inputs['attendees']
|
|
203
|
+
|
|
204
|
+
print(graph.name)
|
|
205
|
+
print(graph)
|
|
206
|
+
graph.evaluate()
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
Output:
|
|
210
|
+
|
|
211
|
+
```c
|
|
212
|
+
How to throw a birthday party
|
|
213
|
+
+-------------------+ +---------------------+
|
|
214
|
+
| InvitePeople | | Birthday Party |
|
|
215
|
+
|-------------------| |---------------------|
|
|
216
|
+
o amount<4> | +--->o attendees<> |
|
|
217
|
+
| people o-----+ +---------------------+
|
|
218
|
+
+-------------------+
|
|
219
|
+
|
|
220
|
+
Jane, Michelle, Mike and John are having a great party!
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
## More Examples
|
|
224
|
+
|
|
225
|
+
There are more examples for common use cases of flowpipe:
|
|
226
|
+
|
|
227
|
+
The code for these examples:
|
|
228
|
+
[house_and_birthday.py](examples/house_and_birthday.py)!
|
|
229
|
+
|
|
230
|
+
Another simple example:
|
|
231
|
+
[world_clock.py](examples/world_clock.py)!
|
|
232
|
+
|
|
233
|
+
How to make use of nested subgraphs:
|
|
234
|
+
[nested_graphs.py](examples/nested_graphs.py)!
|
|
235
|
+
|
|
236
|
+
Using the command pattern with flowpipe successfully:
|
|
237
|
+
[workflow_design_pattern.py](examples/workflow_design_pattern.py)!
|
|
238
|
+
|
|
239
|
+
Use flowpipe on a remote cluster of machines, commonly refered to as a "render farm" in the VFX/Animation industry:
|
|
240
|
+
[vfx_render_farm_conversion.py](examples/vfx_render_farm_conversion.py)!
|
|
241
|
+
|
|
242
|
+
An example graph showcasing a common workflow encountered in the VFX/Animation industry:
|
|
243
|
+
[vfx_rendering.py](examples/vfx_rendering.py)!
|
|
244
|
+
|
|
245
|
+
## VFX Pipeline
|
|
246
|
+
|
|
247
|
+
If you are working in the VFX/Animation industry, please check out this extensive guide on how to use [flowpipe in a vfx pipeline](flowpipe-for-vfx-pipelines.md)!
|
|
248
|
+
|
|
249
|
+
# Evaluators
|
|
250
|
+
|
|
251
|
+
If your nodes just need sequential, threaded or multiprocessing evaluation, the `Graph.evaluate()` method will serve you just fine. If you want to take more control over the way your Graph is being evaluated, `Evaluators` are for you. This can also be used to add, e.g. logging or tracing to node evaluation.
|
|
252
|
+
|
|
253
|
+
Evaluators allow you to take control of node evaluation order, or their scheduling.
|
|
254
|
+
See `flowpipe/evaluator.py` to see the `Graph.evaluate()` method's evaluation schemes.
|
|
255
|
+
|
|
256
|
+
To use a custom evaluator, subclass `flowpipe.evaluator.Evaluator`, and provide at least an `_evaluate_nodes(self, nodes)` method.
|
|
257
|
+
This method should take a list of nodes and call their respective `node.evalaute()` methods (along with any other task you want to do for each node being evaluated).
|
|
258
|
+
To use a cusom evaluator, create it and call its `Evalator.evaluate()` method with the Graph to evaluate as an argument:
|
|
259
|
+
|
|
260
|
+
```py
|
|
261
|
+
from flowpipe.evaluators import LinearEvaluator
|
|
262
|
+
|
|
263
|
+
# assuming you created a graph to evaluate above, called `graph`
|
|
264
|
+
lin_eval = LinearEvaluator()
|
|
265
|
+
lin_eval.evaluate(graph)
|
|
266
|
+
```
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""Exceptions raised by flowpipe."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class CycleError(Exception):
|
|
5
|
+
"""Raised when an action would result in a cycle in a graph."""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class FlowpipeMultiprocessingError(Exception):
|
|
9
|
+
"""Raised when a Node can not be pickled, most likely due to inputs not being picklable."""
|