ewoksppf 2.0.2__tar.gz → 3.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ewoksppf-2.0.2/src/ewoksppf.egg-info → ewoksppf-3.0.0}/PKG-INFO +10 -3
- ewoksppf-3.0.0/README.md +26 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/pyproject.toml +3 -3
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/bindings.py +163 -44
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow21.py +8 -8
- ewoksppf-3.0.0/src/ewoksppf/tests/test_ppf_workflow25.py +172 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0/src/ewoksppf.egg-info}/PKG-INFO +10 -3
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf.egg-info/SOURCES.txt +1 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf.egg-info/requires.txt +2 -2
- ewoksppf-2.0.2/README.md +0 -19
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/LICENSE.md +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/setup.cfg +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/__init__.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/engine.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/ppfrunscript.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/__init__.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/conftest.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_examples.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/__init__.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAdd.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAdd2.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAddA.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAddA2B.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAddABC2D.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAddB.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAddB2C.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAddC2D.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAddWithoutSleep.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorCheck.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorDiamondTest.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorTest.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonErrorHandlerTest.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_end.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow1.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow10.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow11.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow12.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow13.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow14.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow15.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow16.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow17.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow18.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow19.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow2.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow20.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow22.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow23.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow24.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow3.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow6.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow7.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow8.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow9.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_workflow_events.py +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf.egg-info/dependency_links.txt +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf.egg-info/entry_points.txt +0 -0
- {ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ewoksppf
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.0.0
|
|
4
4
|
Summary: Pypushflow binding for Ewoks
|
|
5
5
|
Author-email: ESRF <dau-pydev@esrf.fr>
|
|
6
6
|
License: # MIT License
|
|
@@ -35,8 +35,8 @@ Classifier: Programming Language :: Python :: 3
|
|
|
35
35
|
Requires-Python: >=3.8
|
|
36
36
|
Description-Content-Type: text/markdown
|
|
37
37
|
License-File: LICENSE.md
|
|
38
|
-
Requires-Dist: ewokscore>=
|
|
39
|
-
Requires-Dist: pypushflow>=
|
|
38
|
+
Requires-Dist: ewokscore>=5.0.0
|
|
39
|
+
Requires-Dist: pypushflow>=2.0.0
|
|
40
40
|
Provides-Extra: test
|
|
41
41
|
Requires-Dist: pytest>=7; extra == "test"
|
|
42
42
|
Provides-Extra: dev
|
|
@@ -54,6 +54,13 @@ Dynamic: license-file
|
|
|
54
54
|
|
|
55
55
|
# ewoksppf
|
|
56
56
|
|
|
57
|
+
[](https://github.com/ewoks-kit/ewoksppf/actions/workflows/test.yml)
|
|
58
|
+
[](https://github.com/psf/black)
|
|
59
|
+
[](https://github.com/ewoks-kit/ewoksppf/blob/main/LICENSE.md)
|
|
60
|
+
[](https://codecov.io/gh/ewoks-kit/ewoksppf)
|
|
61
|
+
[](https://ewoksppf.readthedocs.io/en/latest/?badge=latest)
|
|
62
|
+
[](https://pypi.org/project/ewoksppf/)
|
|
63
|
+
|
|
57
64
|
ewoksppf provides task scheduling for cyclic [ewoks](https://ewoks.readthedocs.io/) workflows.
|
|
58
65
|
|
|
59
66
|
## Install
|
ewoksppf-3.0.0/README.md
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# ewoksppf
|
|
2
|
+
|
|
3
|
+
[](https://github.com/ewoks-kit/ewoksppf/actions/workflows/test.yml)
|
|
4
|
+
[](https://github.com/psf/black)
|
|
5
|
+
[](https://github.com/ewoks-kit/ewoksppf/blob/main/LICENSE.md)
|
|
6
|
+
[](https://codecov.io/gh/ewoks-kit/ewoksppf)
|
|
7
|
+
[](https://ewoksppf.readthedocs.io/en/latest/?badge=latest)
|
|
8
|
+
[](https://pypi.org/project/ewoksppf/)
|
|
9
|
+
|
|
10
|
+
ewoksppf provides task scheduling for cyclic [ewoks](https://ewoks.readthedocs.io/) workflows.
|
|
11
|
+
|
|
12
|
+
## Install
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
pip install ewoksppf[test]
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Test
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
pytest --pyargs ewoksppf.tests
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Documentation
|
|
25
|
+
|
|
26
|
+
https://ewoksppf.readthedocs.io/
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "ewoksppf"
|
|
7
|
-
version = "
|
|
7
|
+
version = "3.0.0"
|
|
8
8
|
authors = [{ name = "ESRF", email = "dau-pydev@esrf.fr" }]
|
|
9
9
|
description = "Pypushflow binding for Ewoks"
|
|
10
10
|
readme = { file = "README.md", content-type = "text/markdown" }
|
|
@@ -16,8 +16,8 @@ classifiers = [
|
|
|
16
16
|
]
|
|
17
17
|
requires-python = ">=3.8"
|
|
18
18
|
dependencies = [
|
|
19
|
-
"ewokscore >=
|
|
20
|
-
"pypushflow >=
|
|
19
|
+
"ewokscore >=5.0.0",
|
|
20
|
+
"pypushflow >=2.0.0",
|
|
21
21
|
]
|
|
22
22
|
|
|
23
23
|
[project.urls]
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import os
|
|
2
|
-
import
|
|
2
|
+
import threading
|
|
3
3
|
import warnings
|
|
4
4
|
from contextlib import contextmanager
|
|
5
|
+
from typing import Dict
|
|
5
6
|
from typing import Generator
|
|
6
7
|
from typing import List
|
|
7
8
|
from typing import Optional
|
|
@@ -81,13 +82,13 @@ class EwoksPythonActor(PythonActor):
|
|
|
81
82
|
kw["name"] = ppfname(node_id)
|
|
82
83
|
super().__init__(**kw)
|
|
83
84
|
|
|
84
|
-
def
|
|
85
|
+
def _execute(self, inData: dict, _scope_id: Optional[str] = None) -> None:
|
|
85
86
|
# Update the INFOKEY with node information
|
|
86
87
|
infokey = ppfrunscript.INFOKEY
|
|
87
88
|
inData[infokey] = dict(inData[infokey])
|
|
88
89
|
inData[infokey]["node_id"] = self.node_id
|
|
89
90
|
inData[infokey]["node_attrs"] = self.node_attrs
|
|
90
|
-
|
|
91
|
+
super()._execute(inData, _scope_id=_scope_id)
|
|
91
92
|
|
|
92
93
|
def compileDownstreamData(self, result: dict) -> dict:
|
|
93
94
|
# Merging inputs and outputs to trigger the next task
|
|
@@ -152,8 +153,7 @@ class ConditionalActor(AbstractActor):
|
|
|
152
153
|
return False
|
|
153
154
|
return True
|
|
154
155
|
|
|
155
|
-
def
|
|
156
|
-
self.logger.info("triggered with inData =\n %s", pprint.pformat(inData))
|
|
156
|
+
def _execute(self, inData: dict, _scope_id: Optional[str] = None) -> None:
|
|
157
157
|
self.setStarted()
|
|
158
158
|
trigger = self._conditions_fulfilled(inData)
|
|
159
159
|
self.setFinished()
|
|
@@ -174,6 +174,7 @@ class NameMapperActor(AbstractActor):
|
|
|
174
174
|
name="Name mapper",
|
|
175
175
|
trigger_on_error=False,
|
|
176
176
|
required=False,
|
|
177
|
+
cache_if_optional=False,
|
|
177
178
|
**kw,
|
|
178
179
|
):
|
|
179
180
|
super().__init__(name=name, **kw)
|
|
@@ -181,14 +182,14 @@ class NameMapperActor(AbstractActor):
|
|
|
181
182
|
self.map_all_data = map_all_data
|
|
182
183
|
self.trigger_on_error = trigger_on_error
|
|
183
184
|
self.required = required
|
|
185
|
+
self.cache_if_optional = cache_if_optional
|
|
184
186
|
|
|
185
187
|
def connect(self, actor):
|
|
186
188
|
super().connect(actor)
|
|
187
189
|
if isinstance(actor, InputMergeActor):
|
|
188
|
-
actor.
|
|
190
|
+
actor.register_input_actor(self)
|
|
189
191
|
|
|
190
|
-
def
|
|
191
|
-
self.logger.info("triggered with inData =\n %s", pprint.pformat(inData))
|
|
192
|
+
def _execute(self, inData: dict, _scope_id: Optional[str] = None) -> None:
|
|
192
193
|
is_error = "WorkflowExceptionInstance" in inData and inData.get(
|
|
193
194
|
"_NewWorkflowException"
|
|
194
195
|
)
|
|
@@ -218,55 +219,163 @@ class NameMapperActor(AbstractActor):
|
|
|
218
219
|
|
|
219
220
|
|
|
220
221
|
class InputMergeActor(AbstractActor):
|
|
221
|
-
"""Requires triggers from some input actors before triggering
|
|
222
|
-
the
|
|
223
|
-
|
|
224
|
-
It remembers the last input from the required uptstream actors.
|
|
225
|
-
Only the last non-required input is remembered.
|
|
222
|
+
"""Requires triggers from some input actors before triggering the downstream actors.
|
|
223
|
+
Optional triggers are cached or buffered (before first execution) and the last one is retained.
|
|
226
224
|
"""
|
|
227
225
|
|
|
228
226
|
def __init__(self, parent=None, name="Input merger", **kw):
|
|
229
227
|
super().__init__(parent=parent, name=name, **kw)
|
|
230
|
-
self.startInData = list()
|
|
231
|
-
self.requiredInData = dict()
|
|
232
|
-
self.nonrequiredInData = dict()
|
|
233
228
|
|
|
234
|
-
|
|
229
|
+
# List of input dicts provided by the graph startargs (not part of the Ewoks SPEC)
|
|
230
|
+
self._cached_start_triggers: List[dict] = list()
|
|
231
|
+
|
|
232
|
+
# Map actor to input dict provided by that actor
|
|
233
|
+
self._cached_required_triggers: Dict[AbstractActor, dict] = dict()
|
|
234
|
+
self._cached_optional_triggers: Dict[AbstractActor, dict] = dict()
|
|
235
|
+
|
|
236
|
+
# List of input dicts provided by optional links without caching
|
|
237
|
+
# that arrived before all required triggers arrived
|
|
238
|
+
self._buffer_optional_triggers: List[dict] = list()
|
|
239
|
+
self._buffering = True
|
|
240
|
+
|
|
241
|
+
# Retain only one input dict provided by optional links without caching
|
|
242
|
+
# after all required triggers arrived
|
|
243
|
+
self._retained_optional_trigger: Optional[dict] = None
|
|
244
|
+
|
|
245
|
+
self._lock = threading.Lock()
|
|
246
|
+
|
|
247
|
+
def register_input_actor(self, actor: Optional[AbstractActor]):
|
|
235
248
|
if actor.required:
|
|
236
|
-
|
|
249
|
+
info = "(required): cache inputs"
|
|
250
|
+
self._cached_required_triggers[actor] = None
|
|
251
|
+
elif actor.cache_if_optional:
|
|
252
|
+
info = "(optional): cache inputs"
|
|
253
|
+
self._cached_optional_triggers[actor] = None
|
|
254
|
+
else:
|
|
255
|
+
info = "(optional): buffer inputs before first execution and then retain the last one"
|
|
256
|
+
# see self._buffer_optional_triggers
|
|
257
|
+
self.logger.info("%s %s", actor.name, info)
|
|
237
258
|
|
|
238
|
-
def
|
|
239
|
-
self
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
259
|
+
def _execute(
|
|
260
|
+
self,
|
|
261
|
+
inData: dict,
|
|
262
|
+
_scope_id: Optional[str] = None,
|
|
263
|
+
source: Optional[AbstractActor] = None,
|
|
264
|
+
) -> None:
|
|
265
|
+
with self._lock:
|
|
266
|
+
self.setStarted()
|
|
267
|
+
try:
|
|
268
|
+
self._cache_inputs(source, inData)
|
|
269
|
+
finally:
|
|
270
|
+
self.setFinished()
|
|
271
|
+
|
|
272
|
+
if not self._has_all_required_triggers():
|
|
273
|
+
return
|
|
274
|
+
|
|
275
|
+
self._propagate_cached_inputs()
|
|
276
|
+
|
|
277
|
+
def _propagate_cached_inputs(self) -> None:
|
|
278
|
+
if not self._buffering:
|
|
279
|
+
# Execute with the retained inputs from the last trigger
|
|
280
|
+
# of an optional link without caching. Might be `None`
|
|
281
|
+
# when there is none.
|
|
282
|
+
buffer = [self._retained_optional_trigger]
|
|
244
283
|
else:
|
|
245
|
-
if
|
|
246
|
-
|
|
284
|
+
if self._buffer_optional_triggers:
|
|
285
|
+
# Execute for each retained inputs from optional links without caching.
|
|
286
|
+
buffer = list(self._buffer_optional_triggers)
|
|
287
|
+
else:
|
|
288
|
+
# Execute once without any retained inputs.
|
|
289
|
+
buffer = [None]
|
|
290
|
+
|
|
291
|
+
for i, retained_inputs in enumerate(buffer):
|
|
292
|
+
try:
|
|
293
|
+
self._trigger_downstream(retained_inputs)
|
|
294
|
+
except Exception:
|
|
295
|
+
if self._buffering:
|
|
296
|
+
# Keep the inputs not successfully propagated.
|
|
297
|
+
self._buffer_optional_triggers = buffer[i:]
|
|
298
|
+
raise
|
|
299
|
+
|
|
300
|
+
if self._buffering:
|
|
301
|
+
if buffer:
|
|
302
|
+
# Retain the last one for the next trigger.
|
|
303
|
+
# Might be `None` when there is none.
|
|
304
|
+
self._retained_optional_trigger = buffer[-1]
|
|
247
305
|
else:
|
|
248
|
-
self.
|
|
249
|
-
|
|
250
|
-
|
|
306
|
+
self._retained_optional_trigger = None
|
|
307
|
+
|
|
308
|
+
# No more buffering, only retain one.
|
|
309
|
+
self._buffering = False
|
|
310
|
+
|
|
311
|
+
# No longer needed so do not keep references.
|
|
312
|
+
self._buffer_optional_triggers.clear()
|
|
313
|
+
|
|
314
|
+
def _cache_inputs(self, source: Optional[AbstractActor], inData: dict) -> None:
|
|
315
|
+
if source is None:
|
|
316
|
+
self._cached_start_triggers.append(inData)
|
|
317
|
+
return
|
|
318
|
+
|
|
319
|
+
if source in self._cached_required_triggers:
|
|
320
|
+
# Cache inputs from required link
|
|
321
|
+
self._cached_required_triggers[source] = inData
|
|
322
|
+
elif source in self._cached_optional_triggers:
|
|
323
|
+
# Cache inputs from optional link
|
|
324
|
+
self._cached_optional_triggers[source] = inData
|
|
325
|
+
elif self._buffering:
|
|
326
|
+
# Did not execute yet
|
|
327
|
+
self._buffer_optional_triggers.append(inData)
|
|
328
|
+
else:
|
|
329
|
+
# Executed at least once
|
|
330
|
+
self._retained_optional_trigger = inData
|
|
331
|
+
|
|
332
|
+
def _has_all_required_triggers(self) -> bool:
|
|
333
|
+
missing_required = {
|
|
334
|
+
k: v for k, v in self._cached_required_triggers.items() if v is None
|
|
335
|
+
}
|
|
336
|
+
if missing_required:
|
|
251
337
|
self.logger.info(
|
|
252
338
|
"not triggering downstream actors because missing inputs from actors %s",
|
|
253
|
-
[actor.name for actor in
|
|
339
|
+
[actor.name for actor in missing_required],
|
|
254
340
|
)
|
|
255
|
-
return
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
int(bool(self.nonrequiredInData)),
|
|
261
|
-
)
|
|
262
|
-
newInData = dict()
|
|
263
|
-
for data in self.startInData:
|
|
264
|
-
newInData.update(data)
|
|
265
|
-
for data in self.requiredInData.values():
|
|
266
|
-
newInData.update(data)
|
|
267
|
-
newInData.update(self.nonrequiredInData)
|
|
341
|
+
return False
|
|
342
|
+
return True
|
|
343
|
+
|
|
344
|
+
def _trigger_downstream(self, retained_inputs: Optional[dict]):
|
|
345
|
+
merged_inputs = self._downstream_inputs(retained_inputs)
|
|
268
346
|
for actor in self.listDownStreamActor:
|
|
269
|
-
actor.trigger(
|
|
347
|
+
actor.trigger(merged_inputs)
|
|
348
|
+
|
|
349
|
+
def _downstream_inputs(self, retained_inputs: Optional[dict]) -> dict:
|
|
350
|
+
self.logger.debug(
|
|
351
|
+
"Trigger downstream actor with merged inputs from\n "
|
|
352
|
+
"%d graph start triggers\n "
|
|
353
|
+
"%d cached required links\n "
|
|
354
|
+
"%d cached optional links\n "
|
|
355
|
+
"%d retained optional links",
|
|
356
|
+
len(self._cached_start_triggers),
|
|
357
|
+
len(self._cached_required_triggers),
|
|
358
|
+
len(self._cached_optional_triggers),
|
|
359
|
+
int(retained_inputs is not None),
|
|
360
|
+
)
|
|
361
|
+
|
|
362
|
+
merged_inputs = dict()
|
|
363
|
+
for data in self._cached_start_triggers:
|
|
364
|
+
merged_inputs.update(data)
|
|
365
|
+
|
|
366
|
+
for data in self._cached_required_triggers.values():
|
|
367
|
+
merged_inputs.update(data)
|
|
368
|
+
|
|
369
|
+
for data in self._cached_optional_triggers.values():
|
|
370
|
+
if data is None:
|
|
371
|
+
# Optional link not triggered yet
|
|
372
|
+
continue
|
|
373
|
+
merged_inputs.update(data)
|
|
374
|
+
|
|
375
|
+
if retained_inputs:
|
|
376
|
+
merged_inputs.update(retained_inputs)
|
|
377
|
+
|
|
378
|
+
return merged_inputs
|
|
270
379
|
|
|
271
380
|
|
|
272
381
|
class EwoksWorkflow(Workflow):
|
|
@@ -446,25 +555,35 @@ class EwoksWorkflow(Workflow):
|
|
|
446
555
|
self, taskgraph: TaskGraph, source_id: NodeIdType, target_id: NodeIdType
|
|
447
556
|
) -> NameMapperActor:
|
|
448
557
|
link_attrs = taskgraph.graph[source_id][target_id]
|
|
558
|
+
|
|
559
|
+
# Data mapping
|
|
449
560
|
map_all_data = link_attrs.get("map_all_data", False)
|
|
450
561
|
data_mapping = link_attrs.get("data_mapping", list())
|
|
451
562
|
data_mapping = {
|
|
452
563
|
item["target_input"]: item["source_output"] for item in data_mapping
|
|
453
564
|
}
|
|
565
|
+
|
|
566
|
+
# Conditional link
|
|
454
567
|
on_error = link_attrs.get("on_error", False)
|
|
568
|
+
cache_if_optional = link_attrs.get("cache_if_optional", False)
|
|
569
|
+
|
|
570
|
+
# Required link
|
|
455
571
|
required = analysis.link_is_required(taskgraph.graph, source_id, target_id)
|
|
572
|
+
|
|
456
573
|
source_label = ppfname(source_id)
|
|
457
574
|
target_label = ppfname(target_id)
|
|
458
575
|
if on_error:
|
|
459
576
|
name = f"Name mapper <{source_label} -only on error- {target_label}>"
|
|
460
577
|
else:
|
|
461
578
|
name = f"Name mapper <{source_label} - {target_label}>"
|
|
579
|
+
|
|
462
580
|
return NameMapperActor(
|
|
463
581
|
name=name,
|
|
464
582
|
namemap=data_mapping,
|
|
465
583
|
map_all_data=map_all_data,
|
|
466
584
|
trigger_on_error=on_error,
|
|
467
585
|
required=required,
|
|
586
|
+
cache_if_optional=cache_if_optional,
|
|
468
587
|
**self._actor_arguments,
|
|
469
588
|
)
|
|
470
589
|
|
|
@@ -99,8 +99,10 @@ def submodel21_on_error():
|
|
|
99
99
|
def workflow21(on_error):
|
|
100
100
|
if on_error:
|
|
101
101
|
submodel21 = submodel21_on_error
|
|
102
|
+
out1_required = False
|
|
102
103
|
else:
|
|
103
104
|
submodel21 = submodel21_conditions
|
|
105
|
+
out1_required = None
|
|
104
106
|
|
|
105
107
|
nodes = [
|
|
106
108
|
{"id": "in", "task_type": "method", "task_identifier": qualname(passthrough)},
|
|
@@ -132,6 +134,7 @@ def workflow21(on_error):
|
|
|
132
134
|
{
|
|
133
135
|
"source": "out1",
|
|
134
136
|
"target": "out",
|
|
137
|
+
"required": out1_required,
|
|
135
138
|
"data_mapping": [{"source_output": "return_value", "target_input": "a"}],
|
|
136
139
|
},
|
|
137
140
|
{
|
|
@@ -155,16 +158,13 @@ ARG_SUCCESS = {"inputs": {"a": 20}, "return_value": 1}
|
|
|
155
158
|
ARG_FAILURE = {"inputs": {"a": 0}, "return_value": 2}
|
|
156
159
|
|
|
157
160
|
|
|
158
|
-
@pytest.mark.parametrize(
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
)
|
|
162
|
-
@pytest.mark.parametrize("on_error", [True, False])
|
|
163
|
-
@pytest.mark.parametrize("persist", [True, False])
|
|
164
|
-
def test_workflow21(args, on_error, persist, ppf_log_config, tmpdir):
|
|
161
|
+
@pytest.mark.parametrize("args", [ARG_SUCCESS, ARG_FAILURE], ids=["success", "failure"])
|
|
162
|
+
@pytest.mark.parametrize("on_error", [True, False], ids=["on_error", "-"])
|
|
163
|
+
@pytest.mark.parametrize("persist", [True, False], ids=["persist", "-"])
|
|
164
|
+
def test_workflow21(args, on_error, persist, ppf_log_config, tmp_path):
|
|
165
165
|
"""Test conditions in output nodes"""
|
|
166
166
|
if persist:
|
|
167
|
-
varinfo = {"root_uri": str(
|
|
167
|
+
varinfo = {"root_uri": str(tmp_path)}
|
|
168
168
|
else:
|
|
169
169
|
varinfo = None
|
|
170
170
|
graph = workflow21(on_error=on_error)
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import itertools
|
|
2
|
+
import time
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
from ewokscore.task import Task
|
|
6
|
+
from ewoksutils.import_utils import qualname
|
|
7
|
+
|
|
8
|
+
from ..bindings import execute_graph
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Required(Task, input_names=["compute_time"], output_names=["required"]):
|
|
12
|
+
def run(self):
|
|
13
|
+
time.sleep(self.inputs.compute_time)
|
|
14
|
+
self.outputs.required = True
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Optional(Task, input_names=["compute_time"], output_names=["optional"]):
|
|
18
|
+
def run(self):
|
|
19
|
+
time.sleep(self.inputs.compute_time)
|
|
20
|
+
self.outputs.optional = True
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Gather(
|
|
24
|
+
Task,
|
|
25
|
+
input_names=["required1", "required2"],
|
|
26
|
+
optional_input_names=["optional1", "optional2", "retained1", "retained2"],
|
|
27
|
+
output_names=["cached"],
|
|
28
|
+
):
|
|
29
|
+
def run(self):
|
|
30
|
+
global _GATHER_CACHE
|
|
31
|
+
cached = self.get_input_values()
|
|
32
|
+
_GATHER_CACHE = cached
|
|
33
|
+
print(f"\nDecider executed with inputs: {cached}")
|
|
34
|
+
self.outputs.cached = cached
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def workflow():
|
|
38
|
+
nodes = [
|
|
39
|
+
{
|
|
40
|
+
"id": "required1",
|
|
41
|
+
"task_type": "class",
|
|
42
|
+
"task_identifier": qualname(Required),
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"id": "required2",
|
|
46
|
+
"task_type": "class",
|
|
47
|
+
"task_identifier": qualname(Required),
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"id": "optional1",
|
|
51
|
+
"task_type": "class",
|
|
52
|
+
"task_identifier": qualname(Optional),
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"id": "optional2",
|
|
56
|
+
"task_type": "class",
|
|
57
|
+
"task_identifier": qualname(Optional),
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"id": "retained1",
|
|
61
|
+
"task_type": "class",
|
|
62
|
+
"task_identifier": qualname(Optional),
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"id": "retained2",
|
|
66
|
+
"task_type": "class",
|
|
67
|
+
"task_identifier": qualname(Optional),
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"id": "gather",
|
|
71
|
+
"task_type": "class",
|
|
72
|
+
"task_identifier": qualname(Gather),
|
|
73
|
+
},
|
|
74
|
+
]
|
|
75
|
+
links = [
|
|
76
|
+
{
|
|
77
|
+
"source": "required1",
|
|
78
|
+
"target": "gather",
|
|
79
|
+
"data_mapping": [
|
|
80
|
+
{"source_output": "required", "target_input": "required1"}
|
|
81
|
+
],
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
"source": "required2",
|
|
85
|
+
"target": "gather",
|
|
86
|
+
"data_mapping": [
|
|
87
|
+
{"source_output": "required", "target_input": "required2"}
|
|
88
|
+
],
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
"source": "optional1",
|
|
92
|
+
"target": "gather",
|
|
93
|
+
"required": False,
|
|
94
|
+
"cache_if_optional": True,
|
|
95
|
+
"data_mapping": [
|
|
96
|
+
{"source_output": "optional", "target_input": "optional1"}
|
|
97
|
+
],
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
"source": "optional2",
|
|
101
|
+
"target": "gather",
|
|
102
|
+
"required": False,
|
|
103
|
+
"cache_if_optional": True,
|
|
104
|
+
"data_mapping": [
|
|
105
|
+
{"source_output": "optional", "target_input": "optional2"}
|
|
106
|
+
],
|
|
107
|
+
},
|
|
108
|
+
{
|
|
109
|
+
"source": "retained1",
|
|
110
|
+
"target": "gather",
|
|
111
|
+
"required": False,
|
|
112
|
+
"cache_if_optional": False,
|
|
113
|
+
"data_mapping": [
|
|
114
|
+
{"source_output": "optional", "target_input": "retained1"}
|
|
115
|
+
],
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
"source": "retained2",
|
|
119
|
+
"target": "gather",
|
|
120
|
+
"required": False,
|
|
121
|
+
"cache_if_optional": False,
|
|
122
|
+
"data_mapping": [
|
|
123
|
+
{"source_output": "optional", "target_input": "retained2"}
|
|
124
|
+
],
|
|
125
|
+
},
|
|
126
|
+
]
|
|
127
|
+
return {"graph": {"id": "workflow"}, "nodes": nodes, "links": links}
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def get_inputs(required, optional, retained):
|
|
131
|
+
return [
|
|
132
|
+
{"id": "required1", "name": "compute_time", "value": required},
|
|
133
|
+
{"id": "required2", "name": "compute_time", "value": required},
|
|
134
|
+
{"id": "optional1", "name": "compute_time", "value": optional},
|
|
135
|
+
{"id": "optional2", "name": "compute_time", "value": optional},
|
|
136
|
+
{"id": "retained1", "name": "compute_time", "value": retained},
|
|
137
|
+
{"id": "retained2", "name": "compute_time", "value": retained},
|
|
138
|
+
]
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
_ORDER = list(itertools.permutations(["required", "optional", "retained"]))
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@pytest.mark.parametrize("order", _ORDER, ids=["-".join(keys) for keys in _ORDER])
|
|
145
|
+
def test_ppf_workflow25(ppf_log_config, order):
|
|
146
|
+
"""Test input caching for different types of links executed in different orders."""
|
|
147
|
+
global _GATHER_CACHE
|
|
148
|
+
_GATHER_CACHE = None
|
|
149
|
+
compute_times = [0, 0.5, 1]
|
|
150
|
+
inputs = get_inputs(**dict(zip(order, compute_times)))
|
|
151
|
+
|
|
152
|
+
# result = execute_graph(workflow(), inputs=inputs)
|
|
153
|
+
# cached = set(result["cached"])
|
|
154
|
+
#
|
|
155
|
+
# When
|
|
156
|
+
#
|
|
157
|
+
# order = ('retained', 'required', 'optional')
|
|
158
|
+
#
|
|
159
|
+
# the last two calls to "Gather" could be for example
|
|
160
|
+
#
|
|
161
|
+
# {'required1': True, 'required2': True, 'optional1': True, 'retained2': True}
|
|
162
|
+
# {'required1': True, 'required2': True, 'optional1': True, 'optional2': True, 'retained2': True}
|
|
163
|
+
#
|
|
164
|
+
# Since these calls happen in parallel and there is nothing in the workflow
|
|
165
|
+
# that guarantees we get one or the other as the final workflow result we
|
|
166
|
+
# cannot use the result to test the caching.
|
|
167
|
+
|
|
168
|
+
_ = execute_graph(workflow(), pool_type="thread", inputs=inputs)
|
|
169
|
+
cached = set(_GATHER_CACHE)
|
|
170
|
+
cached1 = {"required1", "required2", "optional1", "optional2", "retained1"}
|
|
171
|
+
cached2 = {"required1", "required2", "optional1", "optional2", "retained2"}
|
|
172
|
+
assert cached == cached1 or cached == cached2, cached
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ewoksppf
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.0.0
|
|
4
4
|
Summary: Pypushflow binding for Ewoks
|
|
5
5
|
Author-email: ESRF <dau-pydev@esrf.fr>
|
|
6
6
|
License: # MIT License
|
|
@@ -35,8 +35,8 @@ Classifier: Programming Language :: Python :: 3
|
|
|
35
35
|
Requires-Python: >=3.8
|
|
36
36
|
Description-Content-Type: text/markdown
|
|
37
37
|
License-File: LICENSE.md
|
|
38
|
-
Requires-Dist: ewokscore>=
|
|
39
|
-
Requires-Dist: pypushflow>=
|
|
38
|
+
Requires-Dist: ewokscore>=5.0.0
|
|
39
|
+
Requires-Dist: pypushflow>=2.0.0
|
|
40
40
|
Provides-Extra: test
|
|
41
41
|
Requires-Dist: pytest>=7; extra == "test"
|
|
42
42
|
Provides-Extra: dev
|
|
@@ -54,6 +54,13 @@ Dynamic: license-file
|
|
|
54
54
|
|
|
55
55
|
# ewoksppf
|
|
56
56
|
|
|
57
|
+
[](https://github.com/ewoks-kit/ewoksppf/actions/workflows/test.yml)
|
|
58
|
+
[](https://github.com/psf/black)
|
|
59
|
+
[](https://github.com/ewoks-kit/ewoksppf/blob/main/LICENSE.md)
|
|
60
|
+
[](https://codecov.io/gh/ewoks-kit/ewoksppf)
|
|
61
|
+
[](https://ewoksppf.readthedocs.io/en/latest/?badge=latest)
|
|
62
|
+
[](https://pypi.org/project/ewoksppf/)
|
|
63
|
+
|
|
57
64
|
ewoksppf provides task scheduling for cyclic [ewoks](https://ewoks.readthedocs.io/) workflows.
|
|
58
65
|
|
|
59
66
|
## Install
|
|
@@ -32,6 +32,7 @@ src/ewoksppf/tests/test_ppf_workflow21.py
|
|
|
32
32
|
src/ewoksppf/tests/test_ppf_workflow22.py
|
|
33
33
|
src/ewoksppf/tests/test_ppf_workflow23.py
|
|
34
34
|
src/ewoksppf/tests/test_ppf_workflow24.py
|
|
35
|
+
src/ewoksppf/tests/test_ppf_workflow25.py
|
|
35
36
|
src/ewoksppf/tests/test_ppf_workflow3.py
|
|
36
37
|
src/ewoksppf/tests/test_ppf_workflow6.py
|
|
37
38
|
src/ewoksppf/tests/test_ppf_workflow7.py
|
ewoksppf-2.0.2/README.md
DELETED
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
# ewoksppf
|
|
2
|
-
|
|
3
|
-
ewoksppf provides task scheduling for cyclic [ewoks](https://ewoks.readthedocs.io/) workflows.
|
|
4
|
-
|
|
5
|
-
## Install
|
|
6
|
-
|
|
7
|
-
```bash
|
|
8
|
-
pip install ewoksppf[test]
|
|
9
|
-
```
|
|
10
|
-
|
|
11
|
-
## Test
|
|
12
|
-
|
|
13
|
-
```bash
|
|
14
|
-
pytest --pyargs ewoksppf.tests
|
|
15
|
-
```
|
|
16
|
-
|
|
17
|
-
## Documentation
|
|
18
|
-
|
|
19
|
-
https://ewoksppf.readthedocs.io/
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAddWithoutSleep.py
RENAMED
|
File without changes
|
|
File without changes
|
{ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorDiamondTest.py
RENAMED
|
File without changes
|
|
File without changes
|
{ewoksppf-2.0.2 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonErrorHandlerTest.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|