pypushflow 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/AbstractActor.py +89 -0
  2. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/ErrorHandler.py +38 -0
  3. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/ForkActor.py +31 -0
  4. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/JoinActor.py +81 -0
  5. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/PythonActor.py +331 -0
  6. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/RequestStatus.py +31 -0
  7. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/RouterActor.py +87 -0
  8. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/StartActor.py +35 -0
  9. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/StopActor.py +56 -0
  10. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/Submodel.py +108 -0
  11. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/UtilsMongoDb.py +114 -0
  12. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/Workflow.py +302 -0
  13. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/__init__.py +24 -0
  14. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/__pycache__/AbstractActor.cpython-37.pyc +0 -0
  15. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/__pycache__/ErrorHandler.cpython-37.pyc +0 -0
  16. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/__pycache__/ForkActor.cpython-37.pyc +0 -0
  17. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/__pycache__/JoinActor.cpython-37.pyc +0 -0
  18. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/__pycache__/PythonActor.cpython-37.pyc +0 -0
  19. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/__pycache__/RequestStatus.cpython-37.pyc +0 -0
  20. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/__pycache__/RouterActor.cpython-37.pyc +0 -0
  21. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/__pycache__/StartActor.cpython-37.pyc +0 -0
  22. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/__pycache__/StopActor.cpython-37.pyc +0 -0
  23. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/__pycache__/Submodel.cpython-37.pyc +0 -0
  24. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/__pycache__/UtilsMongoDb.cpython-37.pyc +0 -0
  25. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/__pycache__/Workflow.cpython-37.pyc +0 -0
  26. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/__pycache__/__init__.cpython-37.pyc +0 -0
  27. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/__pycache__/version.cpython-37.pyc +0 -0
  28. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/addon/__init__.py +1 -0
  29. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/addon/__pycache__/__init__.cpython-37.pyc +0 -0
  30. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/addon/__pycache__/classes.cpython-37.pyc +0 -0
  31. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/addon/__pycache__/utils.cpython-37.pyc +0 -0
  32. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/addon/classes.py +53 -0
  33. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/addon/utils.py +100 -0
  34. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/representation/__init__.py +3 -0
  35. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/representation/__pycache__/__init__.cpython-37.pyc +0 -0
  36. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/representation/scheme/__init__.py +0 -0
  37. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/representation/scheme/__pycache__/__init__.cpython-37.pyc +0 -0
  38. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/representation/scheme/__pycache__/json_parser.cpython-37.pyc +0 -0
  39. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/representation/scheme/__pycache__/link.cpython-37.pyc +0 -0
  40. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/representation/scheme/__pycache__/moml_parser.cpython-37.pyc +0 -0
  41. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/representation/scheme/__pycache__/node.cpython-37.pyc +0 -0
  42. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/representation/scheme/__pycache__/ows_parser.cpython-37.pyc +0 -0
  43. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/representation/scheme/__pycache__/parser.cpython-37.pyc +0 -0
  44. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/representation/scheme/__pycache__/scheme.cpython-37.pyc +0 -0
  45. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/representation/scheme/json_parser.py +0 -0
  46. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/representation/scheme/link.py +169 -0
  47. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/representation/scheme/moml_parser.py +151 -0
  48. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/representation/scheme/node.py +397 -0
  49. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/representation/scheme/ows_parser.py +241 -0
  50. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/representation/scheme/parser.py +129 -0
  51. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/representation/scheme/scheme.py +755 -0
  52. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/TestWorkflow.py +167 -0
  53. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/__init__.py +44 -0
  54. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/__pycache__/TestWorkflow.cpython-37.pyc +0 -0
  55. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/__pycache__/__init__.cpython-37.pyc +0 -0
  56. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/__pycache__/pythonActor1.cpython-37.pyc +0 -0
  57. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/__pycache__/pythonActor2.cpython-37.pyc +0 -0
  58. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/__pycache__/pythonActorTest.cpython-37.pyc +0 -0
  59. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/__pycache__/pythonErrorHandlerTest.cpython-37.pyc +0 -0
  60. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/__pycache__/test_Actors.cpython-37.pyc +0 -0
  61. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/__pycache__/test_UtilsMongoDb.cpython-37.pyc +0 -0
  62. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/__pycache__/test_Workflows.cpython-37.pyc +0 -0
  63. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/__pycache__/test_representation.cpython-37.pyc +0 -0
  64. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/__pycache__/utils.cpython-37.pyc +0 -0
  65. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/pythonActor1.py +29 -0
  66. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/pythonActor2.py +29 -0
  67. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/pythonActorTest.py +35 -0
  68. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/pythonErrorHandlerTest.py +31 -0
  69. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/test_Actors.py +105 -0
  70. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/test_UtilsMongoDb.py +77 -0
  71. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/test_Workflows.py +90 -0
  72. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/test_representation.py +126 -0
  73. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/test/utils.py +64 -0
  74. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow/version.py +132 -0
  75. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow-0.1.0-py3.7.egg-info/PKG-INFO +41 -0
  76. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow-0.1.0-py3.7.egg-info/SOURCES.txt +45 -0
  77. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow-0.1.0-py3.7.egg-info/dependency_links.txt +1 -0
  78. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow-0.1.0-py3.7.egg-info/not-zip-safe +1 -0
  79. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow-0.1.0-py3.7.egg-info/requires.txt +1 -0
  80. home/payno/.local/share/virtualenvs/pypushflow_venv/lib/python3.7/site-packages/pypushflow-0.1.0-py3.7.egg-info/top_level.txt +1 -0
@@ -0,0 +1,89 @@
1
+ #
2
+ # Copyright (c) European Synchrotron Radiation Facility (ESRF)
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining a copy of
5
+ # this software and associated documentation files (the "Software"), to deal in
6
+ # the Software without restriction, including without limitation the rights to
7
+ # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
8
+ # the Software, and to permit persons to whom the Software is furnished to do so,
9
+ # subject to the following conditions:
10
+ #
11
+ # The above copyright notice and this permission notice shall be included in all
12
+ # copies or substantial portions of the Software.
13
+ #
14
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
16
+ # FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
17
+ # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
18
+ # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
19
+ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
20
+ #
21
+
22
+ __authors__ = ["O. Svensson"]
23
+ __license__ = "MIT"
24
+ __date__ = "28/05/2019"
25
+
26
+ import logging
27
+ from pypushflow.addon import utils
28
+ from pypushflow.addon.classes import BaseActorAddOn
29
+
30
+
31
+ logger = logging.getLogger("pypushflow")
32
+
33
+
34
+ class AbstractActor(object):
35
+ """
36
+ TODO
37
+ """
38
+
39
+ def __init__(self, parent=None, name=None):
40
+ if name is None:
41
+ raise RuntimeError("Actor name is None!")
42
+ self.name = name
43
+ self.listDownStreamActor = []
44
+ self.parent = parent
45
+ self.actorId = None
46
+
47
+ self._add_ons = []
48
+ for add_on_class in self._getAddOnsClasses():
49
+ self._add_ons.append(add_on_class())
50
+
51
+ def connect(self, actor):
52
+ logger.debug(
53
+ 'Connecting actor "{0}" to actor "{1}"'.format(self.name, actor.name)
54
+ )
55
+ self.listDownStreamActor.append(actor)
56
+
57
+ def trigger(self, inData):
58
+ for actor in self.listDownStreamActor:
59
+ logger.debug(
60
+ 'In actor "{0}", triggering actor "{1}"'.format(self.name, actor.name)
61
+ )
62
+ self._process_pre_trigger_add_on(inData)
63
+ actor.trigger(inData)
64
+ self._process_post_trigger_add_on()
65
+
66
+ def getActorPath(self):
67
+ return self.parent.getActorPath()
68
+
69
+ def get_addons(self):
70
+ """Return the list of add-on that will be apply to this actor"""
71
+ raise NotImplementedError()
72
+
73
+ def _process_post_trigger_add_on(self):
74
+ for add_on in self._add_ons:
75
+ add_on.post_trigger_action(actor=self)
76
+
77
+ def _process_pre_trigger_add_on(self, in_data):
78
+ for add_on in self._add_ons:
79
+ add_on.pre_trigger_action(actor=self, in_data=in_data)
80
+
81
+ def _getAddOnsClasses(self):
82
+ add_ons = []
83
+ for _, classes in utils.get_registered_add_ons_classes().items():
84
+ for class_ in classes:
85
+ import inspect
86
+
87
+ if BaseActorAddOn in (inspect.getmro(class_)):
88
+ add_ons.append(class_)
89
+ return add_ons
@@ -0,0 +1,38 @@
1
+ #
2
+ # Copyright (c) European Synchrotron Radiation Facility (ESRF)
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining a copy of
5
+ # this software and associated documentation files (the "Software"), to deal in
6
+ # the Software without restriction, including without limitation the rights to
7
+ # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
8
+ # the Software, and to permit persons to whom the Software is furnished to do so,
9
+ # subject to the following conditions:
10
+ #
11
+ # The above copyright notice and this permission notice shall be included in all
12
+ # copies or substantial portions of the Software.
13
+ #
14
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
16
+ # FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
17
+ # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
18
+ # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
19
+ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
20
+ #
21
+
22
+ __authors__ = ["O. Svensson"]
23
+ __license__ = "MIT"
24
+ __date__ = "28/05/2019"
25
+
26
+ # from pypushflow import UtilsMongoDb
27
+
28
+ from pypushflow.AbstractActor import AbstractActor
29
+
30
+
31
+ class ErrorHandler(AbstractActor):
32
+ def __init__(self, parent=None, name="Error handler"):
33
+ AbstractActor.__init__(self, parent=parent, name=name)
34
+
35
+ def trigger(self, inData):
36
+ # if self.parent is not None and hasattr(self.parent, 'mongoId'):
37
+ # UtilsMongoDb.setMongoStatus(self.parent.mongoId, 'error')
38
+ AbstractActor.trigger(self, inData=inData)
@@ -0,0 +1,31 @@
1
+ #
2
+ # Copyright (c) European Synchrotron Radiation Facility (ESRF)
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining a copy of
5
+ # this software and associated documentation files (the "Software"), to deal in
6
+ # the Software without restriction, including without limitation the rights to
7
+ # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
8
+ # the Software, and to permit persons to whom the Software is furnished to do so,
9
+ # subject to the following conditions:
10
+ #
11
+ # The above copyright notice and this permission notice shall be included in all
12
+ # copies or substantial portions of the Software.
13
+ #
14
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
16
+ # FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
17
+ # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
18
+ # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
19
+ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
20
+ #
21
+
22
+ __authors__ = ["O. Svensson"]
23
+ __license__ = "MIT"
24
+ __date__ = "28/05/2019"
25
+
26
+ from pypushflow.AbstractActor import AbstractActor
27
+
28
+
29
+ class ForkActor(AbstractActor):
30
+ def __init__(self, parent=None, name="Fork actor"):
31
+ AbstractActor.__init__(self, parent=parent, name=name)
@@ -0,0 +1,81 @@
1
+ #
2
+ # Copyright (c) European Synchrotron Radiation Facility (ESRF)
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining a copy of
5
+ # this software and associated documentation files (the "Software"), to deal in
6
+ # the Software without restriction, including without limitation the rights to
7
+ # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
8
+ # the Software, and to permit persons to whom the Software is furnished to do so,
9
+ # subject to the following conditions:
10
+ #
11
+ # The above copyright notice and this permission notice shall be included in all
12
+ # copies or substantial portions of the Software.
13
+ #
14
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
16
+ # FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
17
+ # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
18
+ # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
19
+ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
20
+ #
21
+
22
+ __authors__ = ["O. Svensson"]
23
+ __license__ = "MIT"
24
+ __date__ = "28/05/2019"
25
+
26
+ from pypushflow.AbstractActor import AbstractActor
27
+
28
+
29
+ class JoinActor(AbstractActor):
30
+ def __init__(self, parent=None, name="Join actor"):
31
+ AbstractActor.__init__(self, parent=parent, name=name)
32
+ self.numberOfThreads = 0
33
+ self.listInData = []
34
+
35
+ def increaseNumberOfThreads(self):
36
+ self.numberOfThreads += 1
37
+
38
+ def trigger(self, in_data):
39
+ if in_data is None:
40
+ channel = data = None
41
+ else:
42
+ channel, data = in_data
43
+ self.listInData.append(data)
44
+ if len(self.listInData) == self.numberOfThreads:
45
+ newInData = {}
46
+ for data in self.listInData:
47
+ newInData.update(data)
48
+ for actor in self.listDownStreamActor:
49
+ actor.trigger((channel, data))
50
+
51
+
52
+ class JoinUntilStopSignal(AbstractActor):
53
+ def __init__(self, name):
54
+ self.name = name
55
+ self.listInData = []
56
+ self.listDownStreamActor = []
57
+ self._nprocess_received = 0
58
+ self._nprocess_waited = 0
59
+ self._can_stop = False
60
+
61
+ def connect(self, actor):
62
+ self.listDownStreamActor.append(actor)
63
+
64
+ def trigger(self, inData):
65
+ if (
66
+ type(inData) is dict
67
+ and "sig_type" in inData
68
+ and inData["sig_type"] == "stop"
69
+ ):
70
+ self._can_stop = True
71
+ self._nprocess_waited = inData["n_process"]
72
+ else:
73
+ self._nprocess_received += 1
74
+
75
+ self.listInData.append(inData)
76
+ if self._can_stop and self._nprocess_waited <= self._nprocess_received:
77
+ newInData = {}
78
+ for data in self.listInData:
79
+ newInData.update(data)
80
+ for actor in self.listDownStreamActor:
81
+ actor.trigger(newInData)
@@ -0,0 +1,331 @@
1
+ #
2
+ # Copyright (c) European Synchrotron Radiation Facility (ESRF)
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining a copy of
5
+ # this software and associated documentation files (the "Software"), to deal in
6
+ # the Software without restriction, including without limitation the rights to
7
+ # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
8
+ # the Software, and to permit persons to whom the Software is furnished to do so,
9
+ # subject to the following conditions:
10
+ #
11
+ # The above copyright notice and this permission notice shall be included in all
12
+ # copies or substantial portions of the Software.
13
+ #
14
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
16
+ # FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
17
+ # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
18
+ # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
19
+ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
20
+ #
21
+
22
+ __authors__ = ["O. Svensson"]
23
+ __license__ = "MIT"
24
+ __date__ = "28/05/2019"
25
+
26
+ import os
27
+ import pprint
28
+ import logging
29
+ import traceback
30
+ import functools
31
+ import multiprocessing
32
+ from multiprocessing.pool import Pool as _Pool
33
+ from pypushflow.AbstractActor import AbstractActor
34
+ from pypushflow.representation.scheme.node import Node
35
+
36
+
37
+ logger = logging.getLogger(__name__)
38
+
39
+
40
+ class WorkflowException(Exception):
41
+ def __init__(self, errorMessage="", traceBack="", data={}, msg=None):
42
+ super(WorkflowException, self).__init__(msg)
43
+ self.errorMessage = errorMessage
44
+ if data is None:
45
+ data = {}
46
+ self.data = data
47
+ self.traceBack = traceBack
48
+
49
+
50
+ def trace_unhandled_exceptions(func):
51
+ @functools.wraps(func)
52
+ def wrapped_func(*args, **kwargs):
53
+ try:
54
+ out_data = func(*args, **kwargs)
55
+ except Exception as e:
56
+ errorMessage = "{0}".format(e)
57
+ logger.exception(errorMessage)
58
+ traceBack = traceback.format_exc()
59
+ return WorkflowException(
60
+ errorMessage=errorMessage, traceBack=traceBack, data=args[1]
61
+ )
62
+ return out_data
63
+
64
+ return wrapped_func
65
+
66
+
67
+ #############################################################################
68
+ # Create no daemon processes
69
+ # See : https://stackoverflow.com/a/8963618
70
+ #
71
+
72
+
73
+ class NoDaemonProcess(multiprocessing.Process):
74
+ # make 'daemon' attribute always return False
75
+ def _get_daemon(self):
76
+ return False
77
+
78
+ def _set_daemon(self, value):
79
+ pass
80
+
81
+ daemon = property(_get_daemon, _set_daemon)
82
+
83
+
84
+ # We sub-class multiprocessing.pool.Pool instead of multiprocessing.Pool
85
+ # because the latter is only a wrapper function, not a proper class.
86
+
87
+
88
+ class Pool(_Pool):
89
+ Process = NoDaemonProcess
90
+
91
+
92
+ #
93
+ #
94
+ #############################################################################
95
+
96
+
97
+ @trace_unhandled_exceptions
98
+ def _exec_node(name: str, channel_name: str, data: dict, properties: dict):
99
+ """
100
+ Execute a node from the name of the process, input of the process and
101
+ properties of the process
102
+
103
+ :param str name: full name of the process to execute
104
+ :param data: data to process
105
+ :param properties: process properties / settings
106
+ :return: result of Node.execute
107
+ """
108
+ logger.debug(
109
+ "processing {0} on channel {1} with input {2} and {3} as "
110
+ "properties".format(str(name), str(channel_name), str(data), str(properties))
111
+ )
112
+ return Node.execute(
113
+ process_pt=name, properties=properties, input_data=data, input_name=channel_name
114
+ )
115
+
116
+
117
+ class AsyncFactory:
118
+ """
119
+ TODO
120
+ """
121
+
122
+ def __init__(self, node, callback=None, errorCallback=None):
123
+ self.node = node
124
+ self.callback = callback
125
+ self.errorCallback = errorCallback
126
+ self.pool = Pool(1)
127
+
128
+ def call(self, *args, **kwargs):
129
+ logger.debug(
130
+ "Before apply_async, func={0}, callback={1}, errorCallback={2}".format(
131
+ self.node, self.callback, self.errorCallback
132
+ )
133
+ )
134
+ logger.debug("args={0}, kwargs={1}".format(args, kwargs))
135
+ if self.callback is None:
136
+ self.pool.apply_async(_exec_node, args, kwargs)
137
+ elif self.errorCallback is None:
138
+ self.pool.apply_async(_exec_node, args, kwargs, self.callback)
139
+ else:
140
+ self.pool.apply_async(
141
+ _exec_node, args, kwargs, self.callback, self.errorCallback
142
+ )
143
+ logger.debug("After apply_async")
144
+
145
+ def wait(self):
146
+ self.pool.close()
147
+ self.pool.join()
148
+
149
+
150
+ class ActorWrapper(object):
151
+ """
152
+ TODO
153
+ """
154
+
155
+ def __init__(self, node):
156
+ self.node = node
157
+
158
+ @trace_unhandled_exceptions
159
+ def run(self, in_data):
160
+ logger.debug("In actor wrapper for {node}".format(node=self.node))
161
+ output_channel_name, out_data = self.node.execute(in_data)
162
+ if isinstance(out_data, WorkflowException):
163
+ return output_channel_name, out_data
164
+ else:
165
+ in_data.update(out_data)
166
+ return output_channel_name, out_data
167
+
168
+
169
+ class PythonActor(AbstractActor):
170
+ """
171
+ TODO
172
+
173
+ * find a way to avoid 'duplication' of input 'script/process_pt'
174
+ (should be done upstream)
175
+ * I don't think script should have a default value.
176
+
177
+ :param parent:
178
+ :param name:
179
+ :param errorHandler:
180
+ :param script: script originally used
181
+ :param node: Node from representation
182
+
183
+ """
184
+
185
+ def __init__(
186
+ self,
187
+ parent=None,
188
+ name="Python Actor",
189
+ errorHandler=None,
190
+ script=None,
191
+ node=None,
192
+ ):
193
+ if node is not None:
194
+ if script is not None:
195
+ raise ValueError(
196
+ "if a process pointer is provided, you "
197
+ "shouldn't provide a script"
198
+ )
199
+ elif script is None:
200
+ raise ValueError("no script provide to the python actor")
201
+
202
+ AbstractActor.__init__(self, parent=parent, name=name)
203
+ self.error_handler = errorHandler
204
+ self.list_error_handler = []
205
+ # Import script
206
+ self.script = script
207
+ if script is not None:
208
+ # module = importlib.import_module(os.path.splitext(script)[0])
209
+ if script.endswith(".py"):
210
+ script = "".join(os.path.splitext(script)[:-1])
211
+ node = Node(".".join((script, "run")))
212
+ self.actor_wrapper = ActorWrapper(node=node)
213
+ else:
214
+ self.actor_wrapper = ActorWrapper(node=node)
215
+
216
+ self.in_data = None
217
+ self.out_data = None
218
+ self.async_factory = None
219
+
220
+ def get_input_channel_name(self, type_):
221
+ return self.actor_wrapper.node.get_input_channel_name(type_)
222
+
223
+ def get_output_channel_name(self, type_):
224
+ return self.actor_wrapper.node.get_output_channel_name(type_)
225
+
226
+ def connectOnError(self, errorHandler):
227
+ self.list_error_handler.append(errorHandler)
228
+
229
+ def trigger(self, in_data):
230
+ """
231
+ 'callback' function when this function is triggered.
232
+
233
+ :param data: input data
234
+ """
235
+ channel, in_data = in_data
236
+ logging.info("On trigger channel is " + str(channel))
237
+ # cast data to dict if necessary
238
+ if hasattr(in_data, "to_dict"):
239
+ in_data = in_data.to_dict()
240
+
241
+ self.in_data = in_data
242
+ logger.debug(
243
+ "In trigger {0}, inData = {1}".format(self.name, pprint.pformat(in_data))
244
+ )
245
+ if isinstance(in_data, WorkflowException):
246
+ logger.error(
247
+ "Error from previous actor! Not running actor {0}".format(self.name)
248
+ )
249
+ if self.error_handler is not None:
250
+ workflowException = in_data
251
+ oldInData = workflowException.data
252
+ exceptionDict = {
253
+ "errorMessage": workflowException.errorMessage,
254
+ "traceBack": workflowException.traceBack.split("\n"),
255
+ }
256
+ oldInData["WorkflowException"] = exceptionDict
257
+ self.triggerOnError(oldInData)
258
+
259
+ self.async_factory = AsyncFactory(
260
+ self.actor_wrapper.run,
261
+ callback=self.triggerDownStreamActor,
262
+ errorCallback=self.error_handler,
263
+ )
264
+ self.async_factory.call(
265
+ self.actor_wrapper.node.process_pt,
266
+ channel,
267
+ in_data,
268
+ self.actor_wrapper.node.properties,
269
+ )
270
+
271
+ def errorHandler(self, exception):
272
+ logger.error("Error when running actor {0}!".format(self.name))
273
+ workflowException = WorkflowException(
274
+ errorMessage=exception, traceBack=None, data=None
275
+ )
276
+ inData = {"WorkflowException": workflowException}
277
+ logger.error(exception)
278
+ for errorHandler in self.list_error_handler:
279
+ errorHandler.trigger(inData)
280
+ if self.error_handler is not None:
281
+ self.error_handler.triggerOnError(inData)
282
+
283
+ def triggerDownStreamActor(self, output_last_processing=(None, {})):
284
+ logging.warning("---------------------")
285
+ logging.warning(output_last_processing)
286
+ logging.warning("---------------------")
287
+ try:
288
+ output_channel, inData = output_last_processing
289
+ except TypeError:
290
+ output_channel, inData = None, output_last_processing
291
+ logger.info(
292
+ "In triggerDownStreamActor for {0}, Output channel is {1}, "
293
+ "inData is {2}".format(self.name, output_channel, inData)
294
+ )
295
+ if isinstance(inData, WorkflowException):
296
+ logger.error(
297
+ "Error from previous actor! Not running down stream actors {0}".format(
298
+ [actor.name for actor in self.listDownStreamActor]
299
+ )
300
+ )
301
+ workflowException = inData
302
+ oldInData = workflowException.data
303
+ exceptionDict = {
304
+ "errorMessage": workflowException.errorMessage,
305
+ "traceBack": workflowException.traceBack.split("\n"),
306
+ }
307
+ logger.warning(
308
+ "oldInData type: {}, value: {}".format(type(oldInData), oldInData)
309
+ )
310
+ oldInData["WorkflowException"] = exceptionDict
311
+ for errorHandler in self.list_error_handler:
312
+ errorHandler.trigger((None, oldInData))
313
+ if self.error_handler is not None:
314
+ logger.error(
315
+ 'Trigger on error on errorHandler "{0}"'.format(
316
+ self.error_handler.name
317
+ )
318
+ )
319
+ self.error_handler.triggerOnError(inData=(None, oldInData))
320
+ else:
321
+ out_data = {}
322
+ if inData is not None:
323
+ for key, value in inData.items():
324
+ if key in self.in_data:
325
+ if self.in_data[key] != value:
326
+ out_data[key] = value
327
+ else:
328
+ out_data[key] = value
329
+
330
+ for downStreamActor in self.listDownStreamActor:
331
+ downStreamActor.trigger((output_channel, inData))
@@ -0,0 +1,31 @@
1
+ #
2
+ # Copyright (c) European Synchrotron Radiation Facility (ESRF)
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining a copy of
5
+ # this software and associated documentation files (the "Software"), to deal in
6
+ # the Software without restriction, including without limitation the rights to
7
+ # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
8
+ # the Software, and to permit persons to whom the Software is furnished to do so,
9
+ # subject to the following conditions:
10
+ #
11
+ # The above copyright notice and this permission notice shall be included in all
12
+ # copies or substantial portions of the Software.
13
+ #
14
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
16
+ # FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
17
+ # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
18
+ # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
19
+ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
20
+ #
21
+
22
+ __authors__ = ["O. Svensson"]
23
+ __license__ = "MIT"
24
+ __date__ = "28/05/2019"
25
+
26
+ from pypushflow.AbstractActor import AbstractActor
27
+
28
+
29
+ class RequestStatus(AbstractActor):
30
+ def __init__(self, parent, name="Request status"):
31
+ AbstractActor.__init__(self, parent=parent, name=name)
@@ -0,0 +1,87 @@
1
+ #
2
+ # Copyright (c) European Synchrotron Radiation Facility (ESRF)
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining a copy of
5
+ # this software and associated documentation files (the "Software"), to deal in
6
+ # the Software without restriction, including without limitation the rights to
7
+ # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
8
+ # the Software, and to permit persons to whom the Software is furnished to do so,
9
+ # subject to the following conditions:
10
+ #
11
+ # The above copyright notice and this permission notice shall be included in all
12
+ # copies or substantial portions of the Software.
13
+ #
14
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
16
+ # FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
17
+ # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
18
+ # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
19
+ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
20
+ #
21
+
22
+ __authors__ = ["O. Svensson"]
23
+ __license__ = "MIT"
24
+ __date__ = "28/05/2019"
25
+
26
+ from pypushflow.AbstractActor import AbstractActor
27
+
28
+ import logging
29
+
30
+ logger = logging.getLogger("pypushflow")
31
+
32
+
33
+ class RouterActor(AbstractActor):
34
+ def __init__(
35
+ self, parent, errorHandler=None, name="Router", itemName=None, listPort=[]
36
+ ):
37
+ AbstractActor.__init__(self, parent, name)
38
+ self.errorHandler = errorHandler
39
+ self.name = name
40
+ self.itemName = itemName
41
+ self.listPort = listPort
42
+ self.dictValues = {}
43
+
44
+ def connect(self, actor, expectedValue="other"):
45
+ if expectedValue != "other" and not expectedValue in self.listPort:
46
+ raise RuntimeError(
47
+ "Port {0} not defined for router actor {1}!".format(
48
+ expectedValue, self.name
49
+ )
50
+ )
51
+ if expectedValue in self.dictValues:
52
+ self.dictValues[expectedValue].append(actor)
53
+ else:
54
+ self.dictValues[expectedValue] = [actor]
55
+
56
+ def trigger(self, inData):
57
+ logger.debug('In router actor "{0}"'.format(self.name))
58
+ listActor = None
59
+ if self.itemName in inData and not inData[self.itemName] in [
60
+ None,
61
+ "None",
62
+ "null",
63
+ ]:
64
+ logger.debug(
65
+ 'In router actor "{0}", itemName {1} in inData'.format(
66
+ self.name, self.itemName
67
+ )
68
+ )
69
+ value = inData[self.itemName]
70
+ logger.debug('In router actor "{0}", value = {1}'.format(self.name, value))
71
+ if not isinstance(value, dict) and value in self.dictValues:
72
+ listActor = self.dictValues[value]
73
+ if listActor is None:
74
+ logger.debug('In router actor "{0}", actor is None')
75
+ if "other" in self.dictValues:
76
+ listActor = self.dictValues["other"]
77
+ else:
78
+ raise RuntimeError(
79
+ 'No "other" port for router actor "{0}"'.format(self.name)
80
+ )
81
+ for actor in listActor:
82
+ logger.debug(
83
+ 'In router actor "{0}", triggering actor "{1}"'.format(
84
+ self.name, actor.name
85
+ )
86
+ )
87
+ actor.trigger(inData)