ewoksppf 2.0.1__tar.gz → 3.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. {ewoksppf-2.0.1/src/ewoksppf.egg-info → ewoksppf-3.0.0}/PKG-INFO +14 -7
  2. ewoksppf-3.0.0/README.md +26 -0
  3. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/pyproject.toml +7 -7
  4. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/bindings.py +163 -44
  5. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_examples.py +1 -1
  6. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow21.py +8 -8
  7. ewoksppf-3.0.0/src/ewoksppf/tests/test_ppf_workflow25.py +172 -0
  8. {ewoksppf-2.0.1 → ewoksppf-3.0.0/src/ewoksppf.egg-info}/PKG-INFO +14 -7
  9. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf.egg-info/SOURCES.txt +1 -0
  10. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf.egg-info/requires.txt +2 -2
  11. ewoksppf-2.0.1/README.md +0 -19
  12. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/LICENSE.md +0 -0
  13. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/setup.cfg +0 -0
  14. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/__init__.py +0 -0
  15. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/engine.py +0 -0
  16. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/ppfrunscript.py +0 -0
  17. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/__init__.py +0 -0
  18. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/conftest.py +0 -0
  19. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/__init__.py +0 -0
  20. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAdd.py +0 -0
  21. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAdd2.py +0 -0
  22. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAddA.py +0 -0
  23. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAddA2B.py +0 -0
  24. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAddABC2D.py +0 -0
  25. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAddB.py +0 -0
  26. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAddB2C.py +0 -0
  27. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAddC2D.py +0 -0
  28. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorAddWithoutSleep.py +0 -0
  29. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorCheck.py +0 -0
  30. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorDiamondTest.py +0 -0
  31. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonActorTest.py +0 -0
  32. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_actors/pythonErrorHandlerTest.py +0 -0
  33. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_end.py +0 -0
  34. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow1.py +0 -0
  35. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow10.py +0 -0
  36. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow11.py +0 -0
  37. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow12.py +0 -0
  38. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow13.py +0 -0
  39. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow14.py +0 -0
  40. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow15.py +0 -0
  41. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow16.py +0 -0
  42. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow17.py +0 -0
  43. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow18.py +0 -0
  44. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow19.py +0 -0
  45. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow2.py +0 -0
  46. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow20.py +0 -0
  47. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow22.py +0 -0
  48. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow23.py +0 -0
  49. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow24.py +0 -0
  50. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow3.py +0 -0
  51. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow6.py +0 -0
  52. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow7.py +0 -0
  53. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow8.py +0 -0
  54. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_ppf_workflow9.py +0 -0
  55. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf/tests/test_workflow_events.py +0 -0
  56. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf.egg-info/dependency_links.txt +0 -0
  57. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf.egg-info/entry_points.txt +0 -0
  58. {ewoksppf-2.0.1 → ewoksppf-3.0.0}/src/ewoksppf.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ewoksppf
3
- Version: 2.0.1
3
+ Version: 3.0.0
4
4
  Summary: Pypushflow binding for Ewoks
5
5
  Author-email: ESRF <dau-pydev@esrf.fr>
6
6
  License: # MIT License
@@ -24,19 +24,19 @@ License: # MIT License
24
24
  IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
25
25
  CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26
26
 
27
- Project-URL: Homepage, https://gitlab.esrf.fr/workflow/ewoks/ewoksppf/
27
+ Project-URL: Homepage, https://github.com/ewoks-kit/ewoksppf/
28
28
  Project-URL: Documentation, https://ewoksppf.readthedocs.io/
29
- Project-URL: Repository, https://gitlab.esrf.fr/workflow/ewoks/ewoksppf/
30
- Project-URL: Issues, https://gitlab.esrf.fr/workflow/ewoks/ewoksppf/issues
31
- Project-URL: Changelog, https://gitlab.esrf.fr/workflow/ewoks/ewoksppf/-/blob/main/CHANGELOG.md
29
+ Project-URL: Repository, https://github.com/ewoks-kit/ewoksppf/
30
+ Project-URL: Issues, https://github.com/ewoks-kit/ewoksppf/issues
31
+ Project-URL: Changelog, https://github.com/ewoks-kit/ewoksppf/blob/main/CHANGELOG.md
32
32
  Classifier: Intended Audience :: Science/Research
33
33
  Classifier: License :: OSI Approved :: MIT License
34
34
  Classifier: Programming Language :: Python :: 3
35
35
  Requires-Python: >=3.8
36
36
  Description-Content-Type: text/markdown
37
37
  License-File: LICENSE.md
38
- Requires-Dist: ewokscore>=4.0.1
39
- Requires-Dist: pypushflow>=1.1.0
38
+ Requires-Dist: ewokscore>=5.0.0
39
+ Requires-Dist: pypushflow>=2.0.0
40
40
  Provides-Extra: test
41
41
  Requires-Dist: pytest>=7; extra == "test"
42
42
  Provides-Extra: dev
@@ -54,6 +54,13 @@ Dynamic: license-file
54
54
 
55
55
  # ewoksppf
56
56
 
57
+ [![Pipeline](https://github.com/ewoks-kit/ewoksppf/actions/workflows/test.yml/badge.svg?branch=main)](https://github.com/ewoks-kit/ewoksppf/actions/workflows/test.yml)
58
+ [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
59
+ [![License](https://img.shields.io/github/license/ewoks-kit/ewoksppf)](https://github.com/ewoks-kit/ewoksppf/blob/main/LICENSE.md)
60
+ [![Coverage](https://codecov.io/gh/ewoks-kit/ewoksppf/branch/main/graph/badge.svg)](https://codecov.io/gh/ewoks-kit/ewoksppf)
61
+ [![Docs](https://readthedocs.org/projects/ewoksppf/badge/?version=latest)](https://ewoksppf.readthedocs.io/en/latest/?badge=latest)
62
+ [![PyPI](https://img.shields.io/pypi/v/ewoksppf)](https://pypi.org/project/ewoksppf/)
63
+
57
64
  ewoksppf provides task scheduling for cyclic [ewoks](https://ewoks.readthedocs.io/) workflows.
58
65
 
59
66
  ## Install
@@ -0,0 +1,26 @@
1
+ # ewoksppf
2
+
3
+ [![Pipeline](https://github.com/ewoks-kit/ewoksppf/actions/workflows/test.yml/badge.svg?branch=main)](https://github.com/ewoks-kit/ewoksppf/actions/workflows/test.yml)
4
+ [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
5
+ [![License](https://img.shields.io/github/license/ewoks-kit/ewoksppf)](https://github.com/ewoks-kit/ewoksppf/blob/main/LICENSE.md)
6
+ [![Coverage](https://codecov.io/gh/ewoks-kit/ewoksppf/branch/main/graph/badge.svg)](https://codecov.io/gh/ewoks-kit/ewoksppf)
7
+ [![Docs](https://readthedocs.org/projects/ewoksppf/badge/?version=latest)](https://ewoksppf.readthedocs.io/en/latest/?badge=latest)
8
+ [![PyPI](https://img.shields.io/pypi/v/ewoksppf)](https://pypi.org/project/ewoksppf/)
9
+
10
+ ewoksppf provides task scheduling for cyclic [ewoks](https://ewoks.readthedocs.io/) workflows.
11
+
12
+ ## Install
13
+
14
+ ```bash
15
+ pip install ewoksppf[test]
16
+ ```
17
+
18
+ ## Test
19
+
20
+ ```bash
21
+ pytest --pyargs ewoksppf.tests
22
+ ```
23
+
24
+ ## Documentation
25
+
26
+ https://ewoksppf.readthedocs.io/
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "ewoksppf"
7
- version = "2.0.1"
7
+ version = "3.0.0"
8
8
  authors = [{ name = "ESRF", email = "dau-pydev@esrf.fr" }]
9
9
  description = "Pypushflow binding for Ewoks"
10
10
  readme = { file = "README.md", content-type = "text/markdown" }
@@ -16,16 +16,16 @@ classifiers = [
16
16
  ]
17
17
  requires-python = ">=3.8"
18
18
  dependencies = [
19
- "ewokscore >=4.0.1",
20
- "pypushflow >=1.1.0"
19
+ "ewokscore >=5.0.0",
20
+ "pypushflow >=2.0.0",
21
21
  ]
22
22
 
23
23
  [project.urls]
24
- Homepage = "https://gitlab.esrf.fr/workflow/ewoks/ewoksppf/"
24
+ Homepage = "https://github.com/ewoks-kit/ewoksppf/"
25
25
  Documentation = "https://ewoksppf.readthedocs.io/"
26
- Repository = "https://gitlab.esrf.fr/workflow/ewoks/ewoksppf/"
27
- Issues = "https://gitlab.esrf.fr/workflow/ewoks/ewoksppf/issues"
28
- Changelog = "https://gitlab.esrf.fr/workflow/ewoks/ewoksppf/-/blob/main/CHANGELOG.md"
26
+ Repository = "https://github.com/ewoks-kit/ewoksppf/"
27
+ Issues = "https://github.com/ewoks-kit/ewoksppf/issues"
28
+ Changelog = "https://github.com/ewoks-kit/ewoksppf/blob/main/CHANGELOG.md"
29
29
 
30
30
  [project.optional-dependencies]
31
31
  test = [
@@ -1,7 +1,8 @@
1
1
  import os
2
- import pprint
2
+ import threading
3
3
  import warnings
4
4
  from contextlib import contextmanager
5
+ from typing import Dict
5
6
  from typing import Generator
6
7
  from typing import List
7
8
  from typing import Optional
@@ -81,13 +82,13 @@ class EwoksPythonActor(PythonActor):
81
82
  kw["name"] = ppfname(node_id)
82
83
  super().__init__(**kw)
83
84
 
84
- def trigger(self, inData: dict):
85
+ def _execute(self, inData: dict, _scope_id: Optional[str] = None) -> None:
85
86
  # Update the INFOKEY with node information
86
87
  infokey = ppfrunscript.INFOKEY
87
88
  inData[infokey] = dict(inData[infokey])
88
89
  inData[infokey]["node_id"] = self.node_id
89
90
  inData[infokey]["node_attrs"] = self.node_attrs
90
- return super().trigger(inData)
91
+ super()._execute(inData, _scope_id=_scope_id)
91
92
 
92
93
  def compileDownstreamData(self, result: dict) -> dict:
93
94
  # Merging inputs and outputs to trigger the next task
@@ -152,8 +153,7 @@ class ConditionalActor(AbstractActor):
152
153
  return False
153
154
  return True
154
155
 
155
- def trigger(self, inData):
156
- self.logger.info("triggered with inData =\n %s", pprint.pformat(inData))
156
+ def _execute(self, inData: dict, _scope_id: Optional[str] = None) -> None:
157
157
  self.setStarted()
158
158
  trigger = self._conditions_fulfilled(inData)
159
159
  self.setFinished()
@@ -174,6 +174,7 @@ class NameMapperActor(AbstractActor):
174
174
  name="Name mapper",
175
175
  trigger_on_error=False,
176
176
  required=False,
177
+ cache_if_optional=False,
177
178
  **kw,
178
179
  ):
179
180
  super().__init__(name=name, **kw)
@@ -181,14 +182,14 @@ class NameMapperActor(AbstractActor):
181
182
  self.map_all_data = map_all_data
182
183
  self.trigger_on_error = trigger_on_error
183
184
  self.required = required
185
+ self.cache_if_optional = cache_if_optional
184
186
 
185
187
  def connect(self, actor):
186
188
  super().connect(actor)
187
189
  if isinstance(actor, InputMergeActor):
188
- actor.require_input_from_actor(self)
190
+ actor.register_input_actor(self)
189
191
 
190
- def trigger(self, inData: dict):
191
- self.logger.info("triggered with inData =\n %s", pprint.pformat(inData))
192
+ def _execute(self, inData: dict, _scope_id: Optional[str] = None) -> None:
192
193
  is_error = "WorkflowExceptionInstance" in inData and inData.get(
193
194
  "_NewWorkflowException"
194
195
  )
@@ -218,55 +219,163 @@ class NameMapperActor(AbstractActor):
218
219
 
219
220
 
220
221
  class InputMergeActor(AbstractActor):
221
- """Requires triggers from some input actors before triggering
222
- the downstream actors.
223
-
224
- It remembers the last input from the required uptstream actors.
225
- Only the last non-required input is remembered.
222
+ """Requires triggers from some input actors before triggering the downstream actors.
223
+ Optional triggers are cached or buffered (before first execution) and the last one is retained.
226
224
  """
227
225
 
228
226
  def __init__(self, parent=None, name="Input merger", **kw):
229
227
  super().__init__(parent=parent, name=name, **kw)
230
- self.startInData = list()
231
- self.requiredInData = dict()
232
- self.nonrequiredInData = dict()
233
228
 
234
- def require_input_from_actor(self, actor):
229
+ # List of input dicts provided by the graph startargs (not part of the Ewoks SPEC)
230
+ self._cached_start_triggers: List[dict] = list()
231
+
232
+ # Map actor to input dict provided by that actor
233
+ self._cached_required_triggers: Dict[AbstractActor, dict] = dict()
234
+ self._cached_optional_triggers: Dict[AbstractActor, dict] = dict()
235
+
236
+ # List of input dicts provided by optional links without caching
237
+ # that arrived before all required triggers arrived
238
+ self._buffer_optional_triggers: List[dict] = list()
239
+ self._buffering = True
240
+
241
+ # Retain only one input dict provided by optional links without caching
242
+ # after all required triggers arrived
243
+ self._retained_optional_trigger: Optional[dict] = None
244
+
245
+ self._lock = threading.Lock()
246
+
247
+ def register_input_actor(self, actor: Optional[AbstractActor]):
235
248
  if actor.required:
236
- self.requiredInData[actor] = None
249
+ info = "(required): cache inputs"
250
+ self._cached_required_triggers[actor] = None
251
+ elif actor.cache_if_optional:
252
+ info = "(optional): cache inputs"
253
+ self._cached_optional_triggers[actor] = None
254
+ else:
255
+ info = "(optional): buffer inputs before first execution and then retain the last one"
256
+ # see self._buffer_optional_triggers
257
+ self.logger.info("%s %s", actor.name, info)
237
258
 
238
- def trigger(self, inData: dict, source=None):
239
- self.logger.info("triggered with inData =\n %s", pprint.pformat(inData))
240
- self.setStarted()
241
- self.setFinished()
242
- if source is None:
243
- self.startInData.append(inData)
259
+ def _execute(
260
+ self,
261
+ inData: dict,
262
+ _scope_id: Optional[str] = None,
263
+ source: Optional[AbstractActor] = None,
264
+ ) -> None:
265
+ with self._lock:
266
+ self.setStarted()
267
+ try:
268
+ self._cache_inputs(source, inData)
269
+ finally:
270
+ self.setFinished()
271
+
272
+ if not self._has_all_required_triggers():
273
+ return
274
+
275
+ self._propagate_cached_inputs()
276
+
277
+ def _propagate_cached_inputs(self) -> None:
278
+ if not self._buffering:
279
+ # Execute with the retained inputs from the last trigger
280
+ # of an optional link without caching. Might be `None`
281
+ # when there is none.
282
+ buffer = [self._retained_optional_trigger]
244
283
  else:
245
- if source in self.requiredInData:
246
- self.requiredInData[source] = inData
284
+ if self._buffer_optional_triggers:
285
+ # Execute for each retained inputs from optional links without caching.
286
+ buffer = list(self._buffer_optional_triggers)
287
+ else:
288
+ # Execute once without any retained inputs.
289
+ buffer = [None]
290
+
291
+ for i, retained_inputs in enumerate(buffer):
292
+ try:
293
+ self._trigger_downstream(retained_inputs)
294
+ except Exception:
295
+ if self._buffering:
296
+ # Keep the inputs not successfully propagated.
297
+ self._buffer_optional_triggers = buffer[i:]
298
+ raise
299
+
300
+ if self._buffering:
301
+ if buffer:
302
+ # Retain the last one for the next trigger.
303
+ # Might be `None` when there is none.
304
+ self._retained_optional_trigger = buffer[-1]
247
305
  else:
248
- self.nonrequiredInData = inData
249
- missing = {k: v for k, v in self.requiredInData.items() if v is None}
250
- if missing:
306
+ self._retained_optional_trigger = None
307
+
308
+ # No more buffering, only retain one.
309
+ self._buffering = False
310
+
311
+ # No longer needed so do not keep references.
312
+ self._buffer_optional_triggers.clear()
313
+
314
+ def _cache_inputs(self, source: Optional[AbstractActor], inData: dict) -> None:
315
+ if source is None:
316
+ self._cached_start_triggers.append(inData)
317
+ return
318
+
319
+ if source in self._cached_required_triggers:
320
+ # Cache inputs from required link
321
+ self._cached_required_triggers[source] = inData
322
+ elif source in self._cached_optional_triggers:
323
+ # Cache inputs from optional link
324
+ self._cached_optional_triggers[source] = inData
325
+ elif self._buffering:
326
+ # Did not execute yet
327
+ self._buffer_optional_triggers.append(inData)
328
+ else:
329
+ # Executed at least once
330
+ self._retained_optional_trigger = inData
331
+
332
+ def _has_all_required_triggers(self) -> bool:
333
+ missing_required = {
334
+ k: v for k, v in self._cached_required_triggers.items() if v is None
335
+ }
336
+ if missing_required:
251
337
  self.logger.info(
252
338
  "not triggering downstream actors because missing inputs from actors %s",
253
- [actor.name for actor in missing],
339
+ [actor.name for actor in missing_required],
254
340
  )
255
- return
256
- self.logger.info(
257
- "triggering downstream actors (%d start inputs, %d required inputs, %d optional inputs)",
258
- len(self.startInData),
259
- len(self.requiredInData),
260
- int(bool(self.nonrequiredInData)),
261
- )
262
- newInData = dict()
263
- for data in self.startInData:
264
- newInData.update(data)
265
- for data in self.requiredInData.values():
266
- newInData.update(data)
267
- newInData.update(self.nonrequiredInData)
341
+ return False
342
+ return True
343
+
344
+ def _trigger_downstream(self, retained_inputs: Optional[dict]):
345
+ merged_inputs = self._downstream_inputs(retained_inputs)
268
346
  for actor in self.listDownStreamActor:
269
- actor.trigger(newInData)
347
+ actor.trigger(merged_inputs)
348
+
349
+ def _downstream_inputs(self, retained_inputs: Optional[dict]) -> dict:
350
+ self.logger.debug(
351
+ "Trigger downstream actor with merged inputs from\n "
352
+ "%d graph start triggers\n "
353
+ "%d cached required links\n "
354
+ "%d cached optional links\n "
355
+ "%d retained optional links",
356
+ len(self._cached_start_triggers),
357
+ len(self._cached_required_triggers),
358
+ len(self._cached_optional_triggers),
359
+ int(retained_inputs is not None),
360
+ )
361
+
362
+ merged_inputs = dict()
363
+ for data in self._cached_start_triggers:
364
+ merged_inputs.update(data)
365
+
366
+ for data in self._cached_required_triggers.values():
367
+ merged_inputs.update(data)
368
+
369
+ for data in self._cached_optional_triggers.values():
370
+ if data is None:
371
+ # Optional link not triggered yet
372
+ continue
373
+ merged_inputs.update(data)
374
+
375
+ if retained_inputs:
376
+ merged_inputs.update(retained_inputs)
377
+
378
+ return merged_inputs
270
379
 
271
380
 
272
381
  class EwoksWorkflow(Workflow):
@@ -446,25 +555,35 @@ class EwoksWorkflow(Workflow):
446
555
  self, taskgraph: TaskGraph, source_id: NodeIdType, target_id: NodeIdType
447
556
  ) -> NameMapperActor:
448
557
  link_attrs = taskgraph.graph[source_id][target_id]
558
+
559
+ # Data mapping
449
560
  map_all_data = link_attrs.get("map_all_data", False)
450
561
  data_mapping = link_attrs.get("data_mapping", list())
451
562
  data_mapping = {
452
563
  item["target_input"]: item["source_output"] for item in data_mapping
453
564
  }
565
+
566
+ # Conditional link
454
567
  on_error = link_attrs.get("on_error", False)
568
+ cache_if_optional = link_attrs.get("cache_if_optional", False)
569
+
570
+ # Required link
455
571
  required = analysis.link_is_required(taskgraph.graph, source_id, target_id)
572
+
456
573
  source_label = ppfname(source_id)
457
574
  target_label = ppfname(target_id)
458
575
  if on_error:
459
576
  name = f"Name mapper <{source_label} -only on error- {target_label}>"
460
577
  else:
461
578
  name = f"Name mapper <{source_label} - {target_label}>"
579
+
462
580
  return NameMapperActor(
463
581
  name=name,
464
582
  namemap=data_mapping,
465
583
  map_all_data=map_all_data,
466
584
  trigger_on_error=on_error,
467
585
  required=required,
586
+ cache_if_optional=cache_if_optional,
468
587
  **self._actor_arguments,
469
588
  )
470
589
 
@@ -10,7 +10,7 @@ from ewokscore.tests.utils.results import assert_execute_graph_default_result
10
10
  def test_execute_graph(engine, graph_name, scheme, ppf_log_config, tmpdir):
11
11
  if graph_name == "self_trigger":
12
12
  pytest.skip(
13
- "Self-triggering workflow execution is inconsistent: https://gitlab.esrf.fr/workflow/ewoks/ewoksppf/-/issues/16"
13
+ "Self-triggering workflow execution is inconsistent: https://github.com/ewoks-kit/ewoksppf/issues/16"
14
14
  )
15
15
 
16
16
  graph, expected = get_graph(graph_name)
@@ -99,8 +99,10 @@ def submodel21_on_error():
99
99
  def workflow21(on_error):
100
100
  if on_error:
101
101
  submodel21 = submodel21_on_error
102
+ out1_required = False
102
103
  else:
103
104
  submodel21 = submodel21_conditions
105
+ out1_required = None
104
106
 
105
107
  nodes = [
106
108
  {"id": "in", "task_type": "method", "task_identifier": qualname(passthrough)},
@@ -132,6 +134,7 @@ def workflow21(on_error):
132
134
  {
133
135
  "source": "out1",
134
136
  "target": "out",
137
+ "required": out1_required,
135
138
  "data_mapping": [{"source_output": "return_value", "target_input": "a"}],
136
139
  },
137
140
  {
@@ -155,16 +158,13 @@ ARG_SUCCESS = {"inputs": {"a": 20}, "return_value": 1}
155
158
  ARG_FAILURE = {"inputs": {"a": 0}, "return_value": 2}
156
159
 
157
160
 
158
- @pytest.mark.parametrize(
159
- "args",
160
- [ARG_SUCCESS, ARG_FAILURE],
161
- )
162
- @pytest.mark.parametrize("on_error", [True, False])
163
- @pytest.mark.parametrize("persist", [True, False])
164
- def test_workflow21(args, on_error, persist, ppf_log_config, tmpdir):
161
+ @pytest.mark.parametrize("args", [ARG_SUCCESS, ARG_FAILURE], ids=["success", "failure"])
162
+ @pytest.mark.parametrize("on_error", [True, False], ids=["on_error", "-"])
163
+ @pytest.mark.parametrize("persist", [True, False], ids=["persist", "-"])
164
+ def test_workflow21(args, on_error, persist, ppf_log_config, tmp_path):
165
165
  """Test conditions in output nodes"""
166
166
  if persist:
167
- varinfo = {"root_uri": str(tmpdir)}
167
+ varinfo = {"root_uri": str(tmp_path)}
168
168
  else:
169
169
  varinfo = None
170
170
  graph = workflow21(on_error=on_error)
@@ -0,0 +1,172 @@
1
+ import itertools
2
+ import time
3
+
4
+ import pytest
5
+ from ewokscore.task import Task
6
+ from ewoksutils.import_utils import qualname
7
+
8
+ from ..bindings import execute_graph
9
+
10
+
11
+ class Required(Task, input_names=["compute_time"], output_names=["required"]):
12
+ def run(self):
13
+ time.sleep(self.inputs.compute_time)
14
+ self.outputs.required = True
15
+
16
+
17
+ class Optional(Task, input_names=["compute_time"], output_names=["optional"]):
18
+ def run(self):
19
+ time.sleep(self.inputs.compute_time)
20
+ self.outputs.optional = True
21
+
22
+
23
+ class Gather(
24
+ Task,
25
+ input_names=["required1", "required2"],
26
+ optional_input_names=["optional1", "optional2", "retained1", "retained2"],
27
+ output_names=["cached"],
28
+ ):
29
+ def run(self):
30
+ global _GATHER_CACHE
31
+ cached = self.get_input_values()
32
+ _GATHER_CACHE = cached
33
+ print(f"\nDecider executed with inputs: {cached}")
34
+ self.outputs.cached = cached
35
+
36
+
37
+ def workflow():
38
+ nodes = [
39
+ {
40
+ "id": "required1",
41
+ "task_type": "class",
42
+ "task_identifier": qualname(Required),
43
+ },
44
+ {
45
+ "id": "required2",
46
+ "task_type": "class",
47
+ "task_identifier": qualname(Required),
48
+ },
49
+ {
50
+ "id": "optional1",
51
+ "task_type": "class",
52
+ "task_identifier": qualname(Optional),
53
+ },
54
+ {
55
+ "id": "optional2",
56
+ "task_type": "class",
57
+ "task_identifier": qualname(Optional),
58
+ },
59
+ {
60
+ "id": "retained1",
61
+ "task_type": "class",
62
+ "task_identifier": qualname(Optional),
63
+ },
64
+ {
65
+ "id": "retained2",
66
+ "task_type": "class",
67
+ "task_identifier": qualname(Optional),
68
+ },
69
+ {
70
+ "id": "gather",
71
+ "task_type": "class",
72
+ "task_identifier": qualname(Gather),
73
+ },
74
+ ]
75
+ links = [
76
+ {
77
+ "source": "required1",
78
+ "target": "gather",
79
+ "data_mapping": [
80
+ {"source_output": "required", "target_input": "required1"}
81
+ ],
82
+ },
83
+ {
84
+ "source": "required2",
85
+ "target": "gather",
86
+ "data_mapping": [
87
+ {"source_output": "required", "target_input": "required2"}
88
+ ],
89
+ },
90
+ {
91
+ "source": "optional1",
92
+ "target": "gather",
93
+ "required": False,
94
+ "cache_if_optional": True,
95
+ "data_mapping": [
96
+ {"source_output": "optional", "target_input": "optional1"}
97
+ ],
98
+ },
99
+ {
100
+ "source": "optional2",
101
+ "target": "gather",
102
+ "required": False,
103
+ "cache_if_optional": True,
104
+ "data_mapping": [
105
+ {"source_output": "optional", "target_input": "optional2"}
106
+ ],
107
+ },
108
+ {
109
+ "source": "retained1",
110
+ "target": "gather",
111
+ "required": False,
112
+ "cache_if_optional": False,
113
+ "data_mapping": [
114
+ {"source_output": "optional", "target_input": "retained1"}
115
+ ],
116
+ },
117
+ {
118
+ "source": "retained2",
119
+ "target": "gather",
120
+ "required": False,
121
+ "cache_if_optional": False,
122
+ "data_mapping": [
123
+ {"source_output": "optional", "target_input": "retained2"}
124
+ ],
125
+ },
126
+ ]
127
+ return {"graph": {"id": "workflow"}, "nodes": nodes, "links": links}
128
+
129
+
130
+ def get_inputs(required, optional, retained):
131
+ return [
132
+ {"id": "required1", "name": "compute_time", "value": required},
133
+ {"id": "required2", "name": "compute_time", "value": required},
134
+ {"id": "optional1", "name": "compute_time", "value": optional},
135
+ {"id": "optional2", "name": "compute_time", "value": optional},
136
+ {"id": "retained1", "name": "compute_time", "value": retained},
137
+ {"id": "retained2", "name": "compute_time", "value": retained},
138
+ ]
139
+
140
+
141
+ _ORDER = list(itertools.permutations(["required", "optional", "retained"]))
142
+
143
+
144
+ @pytest.mark.parametrize("order", _ORDER, ids=["-".join(keys) for keys in _ORDER])
145
+ def test_ppf_workflow25(ppf_log_config, order):
146
+ """Test input caching for different types of links executed in different orders."""
147
+ global _GATHER_CACHE
148
+ _GATHER_CACHE = None
149
+ compute_times = [0, 0.5, 1]
150
+ inputs = get_inputs(**dict(zip(order, compute_times)))
151
+
152
+ # result = execute_graph(workflow(), inputs=inputs)
153
+ # cached = set(result["cached"])
154
+ #
155
+ # When
156
+ #
157
+ # order = ('retained', 'required', 'optional')
158
+ #
159
+ # the last two calls to "Gather" could be for example
160
+ #
161
+ # {'required1': True, 'required2': True, 'optional1': True, 'retained2': True}
162
+ # {'required1': True, 'required2': True, 'optional1': True, 'optional2': True, 'retained2': True}
163
+ #
164
+ # Since these calls happen in parallel and there is nothing in the workflow
165
+ # that guarantees we get one or the other as the final workflow result we
166
+ # cannot use the result to test the caching.
167
+
168
+ _ = execute_graph(workflow(), pool_type="thread", inputs=inputs)
169
+ cached = set(_GATHER_CACHE)
170
+ cached1 = {"required1", "required2", "optional1", "optional2", "retained1"}
171
+ cached2 = {"required1", "required2", "optional1", "optional2", "retained2"}
172
+ assert cached == cached1 or cached == cached2, cached
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ewoksppf
3
- Version: 2.0.1
3
+ Version: 3.0.0
4
4
  Summary: Pypushflow binding for Ewoks
5
5
  Author-email: ESRF <dau-pydev@esrf.fr>
6
6
  License: # MIT License
@@ -24,19 +24,19 @@ License: # MIT License
24
24
  IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
25
25
  CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26
26
 
27
- Project-URL: Homepage, https://gitlab.esrf.fr/workflow/ewoks/ewoksppf/
27
+ Project-URL: Homepage, https://github.com/ewoks-kit/ewoksppf/
28
28
  Project-URL: Documentation, https://ewoksppf.readthedocs.io/
29
- Project-URL: Repository, https://gitlab.esrf.fr/workflow/ewoks/ewoksppf/
30
- Project-URL: Issues, https://gitlab.esrf.fr/workflow/ewoks/ewoksppf/issues
31
- Project-URL: Changelog, https://gitlab.esrf.fr/workflow/ewoks/ewoksppf/-/blob/main/CHANGELOG.md
29
+ Project-URL: Repository, https://github.com/ewoks-kit/ewoksppf/
30
+ Project-URL: Issues, https://github.com/ewoks-kit/ewoksppf/issues
31
+ Project-URL: Changelog, https://github.com/ewoks-kit/ewoksppf/blob/main/CHANGELOG.md
32
32
  Classifier: Intended Audience :: Science/Research
33
33
  Classifier: License :: OSI Approved :: MIT License
34
34
  Classifier: Programming Language :: Python :: 3
35
35
  Requires-Python: >=3.8
36
36
  Description-Content-Type: text/markdown
37
37
  License-File: LICENSE.md
38
- Requires-Dist: ewokscore>=4.0.1
39
- Requires-Dist: pypushflow>=1.1.0
38
+ Requires-Dist: ewokscore>=5.0.0
39
+ Requires-Dist: pypushflow>=2.0.0
40
40
  Provides-Extra: test
41
41
  Requires-Dist: pytest>=7; extra == "test"
42
42
  Provides-Extra: dev
@@ -54,6 +54,13 @@ Dynamic: license-file
54
54
 
55
55
  # ewoksppf
56
56
 
57
+ [![Pipeline](https://github.com/ewoks-kit/ewoksppf/actions/workflows/test.yml/badge.svg?branch=main)](https://github.com/ewoks-kit/ewoksppf/actions/workflows/test.yml)
58
+ [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
59
+ [![License](https://img.shields.io/github/license/ewoks-kit/ewoksppf)](https://github.com/ewoks-kit/ewoksppf/blob/main/LICENSE.md)
60
+ [![Coverage](https://codecov.io/gh/ewoks-kit/ewoksppf/branch/main/graph/badge.svg)](https://codecov.io/gh/ewoks-kit/ewoksppf)
61
+ [![Docs](https://readthedocs.org/projects/ewoksppf/badge/?version=latest)](https://ewoksppf.readthedocs.io/en/latest/?badge=latest)
62
+ [![PyPI](https://img.shields.io/pypi/v/ewoksppf)](https://pypi.org/project/ewoksppf/)
63
+
57
64
  ewoksppf provides task scheduling for cyclic [ewoks](https://ewoks.readthedocs.io/) workflows.
58
65
 
59
66
  ## Install
@@ -32,6 +32,7 @@ src/ewoksppf/tests/test_ppf_workflow21.py
32
32
  src/ewoksppf/tests/test_ppf_workflow22.py
33
33
  src/ewoksppf/tests/test_ppf_workflow23.py
34
34
  src/ewoksppf/tests/test_ppf_workflow24.py
35
+ src/ewoksppf/tests/test_ppf_workflow25.py
35
36
  src/ewoksppf/tests/test_ppf_workflow3.py
36
37
  src/ewoksppf/tests/test_ppf_workflow6.py
37
38
  src/ewoksppf/tests/test_ppf_workflow7.py
@@ -1,5 +1,5 @@
1
- ewokscore>=4.0.1
2
- pypushflow>=1.1.0
1
+ ewokscore>=5.0.0
2
+ pypushflow>=2.0.0
3
3
 
4
4
  [dev]
5
5
  ewoksppf[test]
ewoksppf-2.0.1/README.md DELETED
@@ -1,19 +0,0 @@
1
- # ewoksppf
2
-
3
- ewoksppf provides task scheduling for cyclic [ewoks](https://ewoks.readthedocs.io/) workflows.
4
-
5
- ## Install
6
-
7
- ```bash
8
- pip install ewoksppf[test]
9
- ```
10
-
11
- ## Test
12
-
13
- ```bash
14
- pytest --pyargs ewoksppf.tests
15
- ```
16
-
17
- ## Documentation
18
-
19
- https://ewoksppf.readthedocs.io/
File without changes
File without changes