taskflow 5.11.0__py3-none-any.whl → 6.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. taskflow/atom.py +3 -5
  2. taskflow/conductors/backends/__init__.py +0 -2
  3. taskflow/conductors/backends/impl_blocking.py +1 -3
  4. taskflow/conductors/backends/impl_executor.py +3 -6
  5. taskflow/conductors/backends/impl_nonblocking.py +1 -3
  6. taskflow/conductors/base.py +1 -3
  7. taskflow/deciders.py +0 -2
  8. taskflow/engines/__init__.py +0 -2
  9. taskflow/engines/action_engine/actions/base.py +1 -3
  10. taskflow/engines/action_engine/actions/retry.py +1 -3
  11. taskflow/engines/action_engine/actions/task.py +1 -3
  12. taskflow/engines/action_engine/builder.py +2 -4
  13. taskflow/engines/action_engine/compiler.py +9 -11
  14. taskflow/engines/action_engine/completer.py +5 -7
  15. taskflow/engines/action_engine/deciders.py +1 -3
  16. taskflow/engines/action_engine/engine.py +4 -26
  17. taskflow/engines/action_engine/executor.py +2 -4
  18. taskflow/engines/action_engine/runtime.py +1 -3
  19. taskflow/engines/action_engine/scheduler.py +3 -5
  20. taskflow/engines/action_engine/scopes.py +3 -5
  21. taskflow/engines/action_engine/selector.py +1 -3
  22. taskflow/engines/action_engine/traversal.py +0 -2
  23. taskflow/engines/base.py +1 -3
  24. taskflow/engines/helpers.py +0 -2
  25. taskflow/engines/worker_based/dispatcher.py +2 -4
  26. taskflow/engines/worker_based/endpoint.py +1 -3
  27. taskflow/engines/worker_based/engine.py +1 -4
  28. taskflow/engines/worker_based/executor.py +0 -2
  29. taskflow/engines/worker_based/protocol.py +1 -3
  30. taskflow/engines/worker_based/proxy.py +2 -4
  31. taskflow/engines/worker_based/server.py +3 -5
  32. taskflow/engines/worker_based/types.py +6 -7
  33. taskflow/engines/worker_based/worker.py +4 -6
  34. taskflow/examples/99_bottles.py +0 -2
  35. taskflow/examples/alphabet_soup.py +0 -2
  36. taskflow/examples/build_a_car.py +1 -3
  37. taskflow/examples/buildsystem.py +1 -3
  38. taskflow/examples/calculate_in_parallel.py +1 -3
  39. taskflow/examples/calculate_linear.py +3 -5
  40. taskflow/examples/create_parallel_volume.py +2 -5
  41. taskflow/examples/delayed_return.py +1 -3
  42. taskflow/examples/distance_calculator.py +12 -24
  43. taskflow/examples/dump_memory_backend.py +1 -3
  44. taskflow/examples/echo_listener.py +0 -2
  45. taskflow/examples/example_utils.py +1 -3
  46. taskflow/examples/fake_billing.py +9 -11
  47. taskflow/examples/graph_flow.py +2 -4
  48. taskflow/examples/hello_world.py +2 -4
  49. taskflow/examples/jobboard_produce_consume_colors.py +3 -5
  50. taskflow/examples/parallel_table_multiply.py +1 -3
  51. taskflow/examples/persistence_example.py +1 -3
  52. taskflow/examples/pseudo_scoping.py +2 -4
  53. taskflow/examples/resume_from_backend.py +1 -3
  54. taskflow/examples/resume_many_flows/my_flows.py +0 -2
  55. taskflow/examples/resume_many_flows/resume_all.py +1 -3
  56. taskflow/examples/resume_many_flows/run_flow.py +2 -4
  57. taskflow/examples/resume_many_flows.py +0 -2
  58. taskflow/examples/resume_vm_boot.py +12 -15
  59. taskflow/examples/resume_volume_create.py +3 -5
  60. taskflow/examples/retry_flow.py +0 -2
  61. taskflow/examples/reverting_linear.py +1 -3
  62. taskflow/examples/run_by_iter.py +0 -2
  63. taskflow/examples/run_by_iter_enumerate.py +1 -3
  64. taskflow/examples/share_engine_thread.py +2 -4
  65. taskflow/examples/simple_linear.py +0 -2
  66. taskflow/examples/simple_linear_listening.py +1 -3
  67. taskflow/examples/simple_linear_pass.py +0 -2
  68. taskflow/examples/simple_map_reduce.py +0 -2
  69. taskflow/examples/switch_graph_flow.py +1 -3
  70. taskflow/examples/timing_listener.py +1 -3
  71. taskflow/examples/tox_conductor.py +3 -5
  72. taskflow/examples/wbe_event_sender.py +0 -2
  73. taskflow/examples/wbe_mandelbrot.py +0 -2
  74. taskflow/examples/wbe_simple_linear.py +0 -2
  75. taskflow/examples/wrapped_exception.py +0 -2
  76. taskflow/exceptions.py +7 -9
  77. taskflow/flow.py +1 -3
  78. taskflow/formatters.py +22 -8
  79. taskflow/jobs/backends/__init__.py +0 -2
  80. taskflow/jobs/backends/impl_redis.py +22 -18
  81. taskflow/jobs/backends/impl_zookeeper.py +7 -8
  82. taskflow/jobs/base.py +5 -7
  83. taskflow/listeners/base.py +1 -3
  84. taskflow/listeners/capturing.py +1 -3
  85. taskflow/listeners/claims.py +1 -3
  86. taskflow/listeners/logging.py +10 -6
  87. taskflow/listeners/printing.py +1 -3
  88. taskflow/listeners/timing.py +10 -13
  89. taskflow/logging.py +0 -2
  90. taskflow/patterns/graph_flow.py +2 -4
  91. taskflow/patterns/linear_flow.py +4 -9
  92. taskflow/patterns/unordered_flow.py +4 -9
  93. taskflow/persistence/backends/__init__.py +1 -3
  94. taskflow/persistence/backends/impl_dir.py +3 -6
  95. taskflow/persistence/backends/impl_memory.py +6 -9
  96. taskflow/persistence/backends/impl_sqlalchemy.py +4 -6
  97. taskflow/persistence/backends/impl_zookeeper.py +2 -4
  98. taskflow/persistence/backends/sqlalchemy/alembic/env.py +0 -2
  99. taskflow/persistence/backends/sqlalchemy/alembic/versions/00af93df9d77_add_unique_into_all_indexes.py +80 -0
  100. taskflow/persistence/backends/sqlalchemy/alembic/versions/14b227d79a87_add_intention_column.py +0 -2
  101. taskflow/persistence/backends/sqlalchemy/alembic/versions/1c783c0c2875_replace_exception_an.py +0 -2
  102. taskflow/persistence/backends/sqlalchemy/alembic/versions/1cea328f0f65_initial_logbook_deta.py +0 -2
  103. taskflow/persistence/backends/sqlalchemy/alembic/versions/2ad4984f2864_switch_postgres_to_json_native.py +0 -2
  104. taskflow/persistence/backends/sqlalchemy/alembic/versions/3162c0f3f8e4_add_revert_results_and_revert_failure_.py +0 -2
  105. taskflow/persistence/backends/sqlalchemy/alembic/versions/589dccdf2b6e_rename_taskdetails_to_atomdetails.py +0 -2
  106. taskflow/persistence/backends/sqlalchemy/alembic/versions/84d6e888850_add_task_detail_type.py +0 -2
  107. taskflow/persistence/backends/sqlalchemy/tables.py +0 -2
  108. taskflow/persistence/base.py +2 -4
  109. taskflow/persistence/models.py +19 -24
  110. taskflow/persistence/path_based.py +1 -3
  111. taskflow/retry.py +12 -18
  112. taskflow/states.py +0 -2
  113. taskflow/storage.py +9 -11
  114. taskflow/task.py +15 -18
  115. taskflow/test.py +5 -5
  116. taskflow/tests/test_examples.py +0 -2
  117. taskflow/tests/unit/action_engine/test_builder.py +1 -3
  118. taskflow/tests/unit/action_engine/test_compile.py +4 -6
  119. taskflow/tests/unit/action_engine/test_creation.py +0 -21
  120. taskflow/tests/unit/action_engine/test_scoping.py +1 -3
  121. taskflow/tests/unit/jobs/base.py +1 -3
  122. taskflow/tests/unit/jobs/test_entrypoint.py +0 -2
  123. taskflow/tests/unit/jobs/test_redis_job.py +1 -3
  124. taskflow/tests/unit/jobs/test_zk_job.py +2 -4
  125. taskflow/tests/unit/patterns/test_graph_flow.py +18 -20
  126. taskflow/tests/unit/patterns/test_linear_flow.py +7 -9
  127. taskflow/tests/unit/patterns/test_unordered_flow.py +12 -14
  128. taskflow/tests/unit/persistence/base.py +2 -4
  129. taskflow/tests/unit/persistence/test_dir_persistence.py +2 -4
  130. taskflow/tests/unit/persistence/test_memory_persistence.py +2 -4
  131. taskflow/tests/unit/persistence/test_sql_persistence.py +4 -6
  132. taskflow/tests/unit/persistence/test_zk_persistence.py +2 -4
  133. taskflow/tests/unit/test_arguments_passing.py +3 -25
  134. taskflow/tests/unit/test_check_transition.py +3 -5
  135. taskflow/tests/unit/test_conductors.py +0 -2
  136. taskflow/tests/unit/test_deciders.py +1 -3
  137. taskflow/tests/unit/test_engine_helpers.py +0 -2
  138. taskflow/tests/unit/test_engines.py +19 -102
  139. taskflow/tests/unit/test_exceptions.py +4 -6
  140. taskflow/tests/unit/test_failure.py +10 -12
  141. taskflow/tests/unit/test_flow_dependencies.py +62 -64
  142. taskflow/tests/unit/test_formatters.py +0 -2
  143. taskflow/tests/unit/test_functor_task.py +1 -3
  144. taskflow/tests/unit/test_listeners.py +9 -11
  145. taskflow/tests/unit/test_mapfunctor_task.py +1 -3
  146. taskflow/tests/unit/test_notifier.py +1 -3
  147. taskflow/tests/unit/test_progress.py +2 -4
  148. taskflow/tests/unit/test_reducefunctor_task.py +1 -3
  149. taskflow/tests/unit/test_retries.py +1 -25
  150. taskflow/tests/unit/test_states.py +0 -2
  151. taskflow/tests/unit/test_storage.py +10 -12
  152. taskflow/tests/unit/test_suspend.py +2 -23
  153. taskflow/tests/unit/test_task.py +17 -19
  154. taskflow/tests/unit/test_types.py +4 -6
  155. taskflow/tests/unit/test_utils.py +9 -11
  156. taskflow/tests/unit/test_utils_async_utils.py +0 -2
  157. taskflow/tests/unit/test_utils_binary.py +12 -14
  158. taskflow/tests/unit/test_utils_iter_utils.py +1 -3
  159. taskflow/tests/unit/test_utils_kazoo_utils.py +0 -2
  160. taskflow/tests/unit/test_utils_threading_utils.py +1 -3
  161. taskflow/tests/unit/worker_based/test_creation.py +0 -2
  162. taskflow/tests/unit/worker_based/test_dispatcher.py +0 -2
  163. taskflow/tests/unit/worker_based/test_endpoint.py +2 -4
  164. taskflow/tests/unit/worker_based/test_executor.py +1 -3
  165. taskflow/tests/unit/worker_based/test_message_pump.py +0 -2
  166. taskflow/tests/unit/worker_based/test_pipeline.py +0 -2
  167. taskflow/tests/unit/worker_based/test_protocol.py +2 -4
  168. taskflow/tests/unit/worker_based/test_proxy.py +2 -4
  169. taskflow/tests/unit/worker_based/test_server.py +3 -5
  170. taskflow/tests/unit/worker_based/test_types.py +0 -2
  171. taskflow/tests/unit/worker_based/test_worker.py +1 -3
  172. taskflow/tests/utils.py +11 -15
  173. taskflow/types/entity.py +1 -3
  174. taskflow/types/failure.py +3 -6
  175. taskflow/types/graph.py +14 -18
  176. taskflow/types/latch.py +1 -3
  177. taskflow/types/notifier.py +6 -9
  178. taskflow/types/sets.py +2 -5
  179. taskflow/types/timing.py +1 -3
  180. taskflow/types/tree.py +6 -10
  181. taskflow/utils/async_utils.py +0 -2
  182. taskflow/utils/banner.py +8 -9
  183. taskflow/utils/eventlet_utils.py +2 -2
  184. taskflow/utils/iter_utils.py +0 -2
  185. taskflow/utils/kazoo_utils.py +7 -9
  186. taskflow/utils/kombu_utils.py +5 -7
  187. taskflow/utils/misc.py +4 -6
  188. taskflow/utils/persistence_utils.py +0 -2
  189. taskflow/utils/redis_utils.py +2 -4
  190. taskflow/utils/schema_utils.py +0 -2
  191. taskflow/utils/threading_utils.py +1 -3
  192. taskflow/version.py +1 -3
  193. {taskflow-5.11.0.dist-info → taskflow-6.0.0.dist-info}/AUTHORS +1 -0
  194. {taskflow-5.11.0.dist-info → taskflow-6.0.0.dist-info}/METADATA +2 -1
  195. taskflow-6.0.0.dist-info/RECORD +243 -0
  196. {taskflow-5.11.0.dist-info → taskflow-6.0.0.dist-info}/WHEEL +1 -1
  197. taskflow-6.0.0.dist-info/pbr.json +1 -0
  198. taskflow/engines/action_engine/process_executor.py +0 -720
  199. taskflow/tests/unit/action_engine/test_process_executor.py +0 -106
  200. taskflow-5.11.0.dist-info/RECORD +0 -244
  201. taskflow-5.11.0.dist-info/pbr.json +0 -1
  202. {taskflow-5.11.0.dist-info → taskflow-6.0.0.dist-info}/LICENSE +0 -0
  203. {taskflow-5.11.0.dist-info → taskflow-6.0.0.dist-info}/entry_points.txt +0 -0
  204. {taskflow-5.11.0.dist-info → taskflow-6.0.0.dist-info}/top_level.txt +0 -0
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2013 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -63,7 +61,7 @@ class PrintText(task.Task):
63
61
  """Just inserts some text print outs in a workflow."""
64
62
  def __init__(self, print_what, no_slow=False):
65
63
  content_hash = hashlib.md5(print_what.encode('utf-8')).hexdigest()[0:8]
66
- super(PrintText, self).__init__(name="Print: %s" % (content_hash))
64
+ super().__init__(name="Print: %s" % (content_hash))
67
65
  self._text = print_what
68
66
  self._no_slow = no_slow
69
67
 
@@ -78,7 +76,7 @@ class PrintText(task.Task):
78
76
  class DefineVMSpec(task.Task):
79
77
  """Defines a vm specification to be."""
80
78
  def __init__(self, name):
81
- super(DefineVMSpec, self).__init__(provides='vm_spec', name=name)
79
+ super().__init__(provides='vm_spec', name=name)
82
80
 
83
81
  def execute(self):
84
82
  return {
@@ -93,8 +91,7 @@ class DefineVMSpec(task.Task):
93
91
  class LocateImages(task.Task):
94
92
  """Locates where the vm images are."""
95
93
  def __init__(self, name):
96
- super(LocateImages, self).__init__(provides='image_locations',
97
- name=name)
94
+ super().__init__(provides='image_locations', name=name)
98
95
 
99
96
  def execute(self, vm_spec):
100
97
  image_locations = {}
@@ -107,13 +104,13 @@ class LocateImages(task.Task):
107
104
  class DownloadImages(task.Task):
108
105
  """Downloads all the vm images."""
109
106
  def __init__(self, name):
110
- super(DownloadImages, self).__init__(provides='download_paths',
111
- name=name)
107
+ super().__init__(provides='download_paths',
108
+ name=name)
112
109
 
113
110
  def execute(self, image_locations):
114
111
  for src, loc in image_locations.items():
115
112
  with slow_down(1):
116
- print("Downloading from %s => %s" % (src, loc))
113
+ print("Downloading from {} => {}".format(src, loc))
117
114
  return sorted(image_locations.values())
118
115
 
119
116
 
@@ -125,8 +122,8 @@ IPADDR=%s
125
122
  ONBOOT=yes"""
126
123
 
127
124
  def __init__(self, name):
128
- super(CreateNetworkTpl, self).__init__(provides='network_settings',
129
- name=name)
125
+ super().__init__(provides='network_settings',
126
+ name=name)
130
127
 
131
128
  def execute(self, ips):
132
129
  settings = []
@@ -138,7 +135,7 @@ ONBOOT=yes"""
138
135
  class AllocateIP(task.Task):
139
136
  """Allocates the ips for the given vm."""
140
137
  def __init__(self, name):
141
- super(AllocateIP, self).__init__(provides='ips', name=name)
138
+ super().__init__(provides='ips', name=name)
142
139
 
143
140
  def execute(self, vm_spec):
144
141
  ips = []
@@ -152,7 +149,7 @@ class WriteNetworkSettings(task.Task):
152
149
  def execute(self, download_paths, network_settings):
153
150
  for j, path in enumerate(download_paths):
154
151
  with slow_down(1):
155
- print("Mounting %s to /tmp/%s" % (path, j))
152
+ print("Mounting {} to /tmp/{}".format(path, j))
156
153
  for i, setting in enumerate(network_settings):
157
154
  filename = ("/tmp/etc/sysconfig/network-scripts/"
158
155
  "ifcfg-eth%s" % (i))
@@ -263,8 +260,8 @@ with eu.get_backend() as backend:
263
260
  backend=backend, book=book,
264
261
  engine='parallel',
265
262
  executor=executor)
266
- print("!! Your tracking id is: '%s+%s'" % (book.uuid,
267
- engine.storage.flow_uuid))
263
+ print("!! Your tracking id is: '{}+{}'".format(
264
+ book.uuid, engine.storage.flow_uuid))
268
265
  print("!! Please submit this on later runs for tracking purposes")
269
266
  else:
270
267
  # Attempt to load from a previously partially completed flow.
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2013 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -72,7 +70,7 @@ def find_flow_detail(backend, book_id, flow_id):
72
70
  class PrintText(task.Task):
73
71
  def __init__(self, print_what, no_slow=False):
74
72
  content_hash = hashlib.md5(print_what.encode('utf-8')).hexdigest()[0:8]
75
- super(PrintText, self).__init__(name="Print: %s" % (content_hash))
73
+ super().__init__(name="Print: %s" % (content_hash))
76
74
  self._text = print_what
77
75
  self._no_slow = no_slow
78
76
 
@@ -141,8 +139,8 @@ with example_utils.get_backend() as backend:
141
139
  book.add(flow_detail)
142
140
  with contextlib.closing(backend.get_connection()) as conn:
143
141
  conn.save_logbook(book)
144
- print("!! Your tracking id is: '%s+%s'" % (book.uuid,
145
- flow_detail.uuid))
142
+ print("!! Your tracking id is: '{}+{}'".format(book.uuid,
143
+ flow_detail.uuid))
146
144
  print("!! Please submit this on later runs for tracking purposes")
147
145
  else:
148
146
  flow_detail = find_flow_detail(backend, book_id, flow_id)
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -59,7 +57,7 @@ class CallJoe(task.Task):
59
57
 
60
58
  class CallSuzzie(task.Task):
61
59
  def execute(self, suzzie_number, *args, **kwargs):
62
- raise IOError("Suzzie not home right now.")
60
+ raise OSError("Suzzie not home right now.")
63
61
 
64
62
 
65
63
  # Create your flow and associated tasks (the work to be done).
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -51,4 +49,4 @@ e.compile()
51
49
  e.prepare()
52
50
 
53
51
  for i, st in enumerate(e.run_iter(), 1):
54
- print("Transition %s: %s" % (i, st))
52
+ print("Transition {}: {}".format(i, st))
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -43,11 +41,11 @@ from taskflow.utils import threading_utils as tu
43
41
 
44
42
  class DelayedTask(task.Task):
45
43
  def __init__(self, name):
46
- super(DelayedTask, self).__init__(name=name)
44
+ super().__init__(name=name)
47
45
  self._wait_for = random.random()
48
46
 
49
47
  def execute(self):
50
- print("Running '%s' in thread '%s'" % (self.name, tu.get_ident()))
48
+ print("Running '{}' in thread '{}'".format(self.name, tu.get_ident()))
51
49
  time.sleep(self._wait_for)
52
50
 
53
51
 
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -71,7 +69,7 @@ def flow_watch(state, details):
71
69
 
72
70
 
73
71
  def task_watch(state, details):
74
- print('Task %s => %s' % (details.get('task_name'), state))
72
+ print('Task {} => {}'.format(details.get('task_name'), state))
75
73
 
76
74
 
77
75
  # Wrap your functions into a task type that knows how to treat your functions
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -64,7 +62,7 @@ while entries:
64
62
  path = entries.pop()
65
63
  value = backend.memory[path]
66
64
  if value:
67
- print("%s -> %s" % (path, value))
65
+ print("{} -> {}".format(path, value))
68
66
  else:
69
67
  print("%s" % (path))
70
68
  entries.extend(os.path.join(path, child)
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -45,7 +43,7 @@ from taskflow import task
45
43
 
46
44
  class VariableTask(task.Task):
47
45
  def __init__(self, name):
48
- super(VariableTask, self).__init__(name)
46
+ super().__init__(name)
49
47
  self._sleepy_time = random.random()
50
48
 
51
49
  def execute(self):
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -60,7 +58,7 @@ from taskflow.utils import threading_utils
60
58
  RUN_TIME = 5
61
59
  REVIEW_CREATION_DELAY = 0.5
62
60
  SCAN_DELAY = 0.1
63
- NAME = "%s_%s" % (socket.getfqdn(), os.getpid())
61
+ NAME = "{}_{}".format(socket.getfqdn(), os.getpid())
64
62
 
65
63
  # This won't really use zookeeper but will use a local version of it using
66
64
  # the zake library that mimics an actual zookeeper cluster using threads and
@@ -74,7 +72,7 @@ class RunReview(task.Task):
74
72
  # A dummy task that clones the review and runs tox...
75
73
 
76
74
  def _clone_review(self, review, temp_dir):
77
- print("Cloning review '%s' into %s" % (review['id'], temp_dir))
75
+ print("Cloning review '{}' into {}".format(review['id'], temp_dir))
78
76
 
79
77
  def _run_tox(self, temp_dir):
80
78
  print("Running tox in %s" % temp_dir)
@@ -177,7 +175,7 @@ def generate_reviewer(client, saver, name=NAME):
177
175
  'review': review,
178
176
  },
179
177
  }
180
- job_name = "%s_%s" % (real_name, review['id'])
178
+ job_name = "{}_{}".format(real_name, review['id'])
181
179
  print("Posting review '%s'" % review['id'])
182
180
  jb.post(job_name,
183
181
  book=make_save_book(saver, review['id']),
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
taskflow/exceptions.py CHANGED
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -67,7 +65,7 @@ class TaskFlowException(Exception):
67
65
  this is not yet implemented/supported natively.
68
66
  """
69
67
  def __init__(self, message, cause=None):
70
- super(TaskFlowException, self).__init__(message)
68
+ super().__init__(message)
71
69
  self._cause = cause
72
70
 
73
71
  @property
@@ -192,7 +190,7 @@ class MissingDependencies(DependencyFailure):
192
190
  message = self.MESSAGE_TPL % {'who': who, 'requirements': requirements}
193
191
  if method:
194
192
  message = (self.METHOD_TPL % {'method': method}) + message
195
- super(MissingDependencies, self).__init__(message, cause=cause)
193
+ super().__init__(message, cause=cause)
196
194
  self.missing_requirements = requirements
197
195
 
198
196
 
@@ -228,7 +226,7 @@ class DisallowedAccess(TaskFlowException):
228
226
  """Raised when storage access is not possible due to state limitations."""
229
227
 
230
228
  def __init__(self, message, cause=None, state=None):
231
- super(DisallowedAccess, self).__init__(message, cause=cause)
229
+ super().__init__(message, cause=cause)
232
230
  self.state = state
233
231
 
234
232
 
@@ -261,7 +259,7 @@ class WrappedFailure(Exception):
261
259
  """
262
260
 
263
261
  def __init__(self, causes):
264
- super(WrappedFailure, self).__init__()
262
+ super().__init__()
265
263
  self._causes = []
266
264
  for cause in causes:
267
265
  if cause.check(type(self)) and cause.exception:
@@ -306,8 +304,8 @@ class WrappedFailure(Exception):
306
304
 
307
305
  def __str__(self):
308
306
  buf = io.StringIO()
309
- buf.write(u'WrappedFailure: [')
307
+ buf.write('WrappedFailure: [')
310
308
  causes_gen = (str(cause) for cause in self._causes)
311
- buf.write(u", ".join(causes_gen))
312
- buf.write(u']')
309
+ buf.write(", ".join(causes_gen))
310
+ buf.write(']')
313
311
  return buf.getvalue()
taskflow/flow.py CHANGED
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -42,7 +40,7 @@ _CHOP_PAT_LEN = len(_CHOP_PAT)
42
40
  LINK_DECIDER_DEPTH = 'decider_depth'
43
41
 
44
42
 
45
- class Flow(object, metaclass=abc.ABCMeta):
43
+ class Flow(metaclass=abc.ABCMeta):
46
44
  """The base abstract class of all flow implementations.
47
45
 
48
46
  A flow is a structure that defines relationships between tasks. You can
taskflow/formatters.py CHANGED
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -14,6 +12,7 @@
14
12
  # License for the specific language governing permissions and limitations
15
13
  # under the License.
16
14
 
15
+ import copy
17
16
  import functools
18
17
 
19
18
  from taskflow.engines.action_engine import compiler
@@ -58,17 +57,29 @@ def _fetch_predecessor_tree(graph, atom):
58
57
  return root
59
58
 
60
59
 
61
- class FailureFormatter(object):
60
+ class FailureFormatter:
62
61
  """Formats a failure and connects it to associated atoms & engine."""
63
62
 
64
63
  _BUILDERS = {
65
64
  states.EXECUTE: (_fetch_predecessor_tree, 'predecessors'),
66
65
  }
67
66
 
68
- def __init__(self, engine, hide_inputs_outputs_of=()):
67
+ def __init__(self, engine, hide_inputs_outputs_of=(),
68
+ mask_inputs_keys=(), mask_outputs_keys=()):
69
69
  self._hide_inputs_outputs_of = hide_inputs_outputs_of
70
+ self._mask_inputs_keys = mask_inputs_keys
71
+ self._mask_outputs_keys = mask_outputs_keys
70
72
  self._engine = engine
71
73
 
74
+ def _mask_keys(self, data, mask_keys):
75
+ if not data or not isinstance(data, dict):
76
+ return data
77
+ result = copy.deepcopy(data)
78
+ for k in mask_keys:
79
+ if k in result:
80
+ result[k] = '***'
81
+ return result
82
+
72
83
  def _format_node(self, storage, cache, node):
73
84
  """Formats a single tree node into a string version."""
74
85
  if node.metadata['kind'] == compiler.FLOW:
@@ -100,14 +111,16 @@ class FailureFormatter(object):
100
111
  atom_name,
101
112
  fetch_mapped_args)
102
113
  if requires_found:
103
- atom_attrs['requires'] = requires
114
+ atom_attrs['requires'] = self._mask_keys(
115
+ requires, self._mask_inputs_keys)
104
116
  provides, provides_found = _cached_get(
105
117
  cache, 'provides', atom_name,
106
118
  storage.get_execute_result, atom_name)
107
119
  if provides_found:
108
- atom_attrs['provides'] = provides
120
+ atom_attrs['provides'] = self._mask_keys(
121
+ provides, self._mask_outputs_keys)
109
122
  if atom_attrs:
110
- return "Atom '%s' %s" % (atom_name, atom_attrs)
123
+ return "Atom '{}' {}".format(atom_name, atom_attrs)
111
124
  else:
112
125
  return "Atom '%s'" % (atom_name)
113
126
  else:
@@ -156,7 +169,8 @@ class FailureFormatter(object):
156
169
  builder, kind = self._BUILDERS[atom_intention]
157
170
  rooted_tree = builder(graph, atom)
158
171
  child_count = rooted_tree.child_count(only_direct=False)
159
- buff.write_nl('%s %s (most recent first):' % (child_count, kind))
172
+ buff.write_nl(
173
+ '{} {} (most recent first):'.format(child_count, kind))
160
174
  formatter = functools.partial(self._format_node, storage, cache)
161
175
  direct_child_count = rooted_tree.child_count(only_direct=True)
162
176
  for i, child in enumerate(rooted_tree, 1):
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2015 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -17,7 +15,6 @@
17
15
  import contextlib
18
16
  import datetime
19
17
  import functools
20
- import re
21
18
  import string
22
19
  import threading
23
20
  import time
@@ -26,6 +23,7 @@ import fasteners
26
23
  import msgpack
27
24
  from oslo_serialization import msgpackutils
28
25
  from oslo_utils import excutils
26
+ from oslo_utils import netutils
29
27
  from oslo_utils import strutils
30
28
  from oslo_utils import timeutils
31
29
  from oslo_utils import uuidutils
@@ -69,10 +67,10 @@ class RedisJob(base.Job):
69
67
  created_on=None, backend=None,
70
68
  book=None, book_data=None,
71
69
  priority=base.JobPriority.NORMAL):
72
- super(RedisJob, self).__init__(board, name,
73
- uuid=uuid, details=details,
74
- backend=backend,
75
- book=book, book_data=book_data)
70
+ super().__init__(board, name,
71
+ uuid=uuid, details=details,
72
+ backend=backend,
73
+ book=book, book_data=book_data)
76
74
  self._created_on = created_on
77
75
  self._client = board._client
78
76
  self._redis_version = board._redis_version
@@ -561,15 +559,17 @@ return cmsgpack.pack(result)
561
559
 
562
560
  @classmethod
563
561
  def _parse_sentinel(cls, sentinel):
564
- # IPv6 (eg. [::1]:6379 )
565
- match = re.search(r'^\[(\S+)\]:(\d+)$', sentinel)
566
- if match:
567
- return (match[1], int(match[2]))
568
- # IPv4 or hostname (eg. 127.0.0.1:6379 or localhost:6379)
569
- match = re.search(r'^(\S+):(\d+)$', sentinel)
570
- if match:
571
- return (match[1], int(match[2]))
572
- raise ValueError('Malformed sentinel server format')
562
+ host, port = netutils.parse_host_port(sentinel)
563
+ if host is None or port is None:
564
+ raise ValueError('Malformed sentinel server format')
565
+ return (host, port)
566
+
567
+ @classmethod
568
+ def _filter_ssl_options(cls, opts):
569
+ if not opts.get('ssl', False):
570
+ return {k: v for (k, v) in opts.items()
571
+ if not k.startswith('ssl_')}
572
+ return opts
573
573
 
574
574
  @classmethod
575
575
  def _make_client(cls, conf):
@@ -584,8 +584,12 @@ return cmsgpack.pack(result)
584
584
  sentinels = [(client_conf.pop('host'), client_conf.pop('port'))]
585
585
  for fallback in conf.get('sentinel_fallbacks', []):
586
586
  sentinels.append(cls._parse_sentinel(fallback))
587
+ client_conf = cls._filter_ssl_options(client_conf)
588
+ sentinel_kwargs = conf.get('sentinel_kwargs')
589
+ if sentinel_kwargs is not None:
590
+ sentinel_kwargs = cls._filter_ssl_options(sentinel_kwargs)
587
591
  s = sentinel.Sentinel(sentinels,
588
- sentinel_kwargs=conf.get('sentinel_kwargs'),
592
+ sentinel_kwargs=sentinel_kwargs,
589
593
  **client_conf)
590
594
  return s.master_for(conf['sentinel'])
591
595
  else:
@@ -593,7 +597,7 @@ return cmsgpack.pack(result)
593
597
 
594
598
  def __init__(self, name, conf,
595
599
  client=None, persistence=None):
596
- super(RedisJobBoard, self).__init__(name, conf)
600
+ super().__init__(name, conf)
597
601
  self._closed = True
598
602
  if client is not None:
599
603
  self._client = client
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2013 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -51,10 +49,10 @@ class ZookeeperJob(base.Job):
51
49
  uuid=None, details=None, book=None, book_data=None,
52
50
  created_on=None, backend=None,
53
51
  priority=base.JobPriority.NORMAL):
54
- super(ZookeeperJob, self).__init__(board, name,
55
- uuid=uuid, details=details,
56
- backend=backend,
57
- book=book, book_data=book_data)
52
+ super().__init__(board, name,
53
+ uuid=uuid, details=details,
54
+ backend=backend,
55
+ book=book, book_data=book_data)
58
56
  self._client = client
59
57
  self._path = k_paths.normpath(path)
60
58
  self._lock_path = self._path + board.LOCK_POSTFIX
@@ -281,7 +279,7 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
281
279
 
282
280
  def __init__(self, name, conf,
283
281
  client=None, persistence=None, emit_notifications=True):
284
- super(ZookeeperJobBoard, self).__init__(name, conf)
282
+ super().__init__(name, conf)
285
283
  if client is not None:
286
284
  self._client = client
287
285
  self._owned = False
@@ -552,7 +550,8 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
552
550
  except Exception:
553
551
  owner = None
554
552
  if owner:
555
- message = "Job %s already claimed by '%s'" % (job.uuid, owner)
553
+ message = "Job {} already claimed by '{}'".format(
554
+ job.uuid, owner)
556
555
  else:
557
556
  message = "Job %s already claimed" % (job.uuid)
558
557
  excp.raise_with_cause(excp.UnclaimableJob,
taskflow/jobs/base.py CHANGED
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2013 Rackspace Hosting Inc. All Rights Reserved.
4
2
  # Copyright (C) 2013 Yahoo! Inc. All Rights Reserved.
5
3
  #
@@ -110,7 +108,7 @@ class JobPriority(enum.Enum):
110
108
  return tuple(values)
111
109
 
112
110
 
113
- class Job(object, metaclass=abc.ABCMeta):
111
+ class Job(metaclass=abc.ABCMeta):
114
112
  """A abstraction that represents a named and trackable unit of work.
115
113
 
116
114
  A job connects a logbook, a owner, a priority, last modified and created
@@ -277,12 +275,12 @@ class Job(object, metaclass=abc.ABCMeta):
277
275
  def __str__(self):
278
276
  """Pretty formats the job into something *more* meaningful."""
279
277
  cls_name = type(self).__name__
280
- return "%s: %s (priority=%s, uuid=%s, details=%s)" % (
278
+ return "{}: {} (priority={}, uuid={}, details={})".format(
281
279
  cls_name, self.name, self.priority,
282
280
  self.uuid, self.details)
283
281
 
284
282
 
285
- class JobBoardIterator(object):
283
+ class JobBoardIterator:
286
284
  """Iterator over a jobboard that iterates over potential jobs.
287
285
 
288
286
  It provides the following attributes:
@@ -355,7 +353,7 @@ class JobBoardIterator(object):
355
353
  return job
356
354
 
357
355
 
358
- class JobBoard(object, metaclass=abc.ABCMeta):
356
+ class JobBoard(metaclass=abc.ABCMeta):
359
357
  """A place where jobs can be posted, reposted, claimed and transferred.
360
358
 
361
359
  There can be multiple implementations of this job board, depending on the
@@ -565,7 +563,7 @@ class NotifyingJobBoard(JobBoard):
565
563
  registered are thread safe (and block for as little time as possible).
566
564
  """
567
565
  def __init__(self, name, conf):
568
- super(NotifyingJobBoard, self).__init__(name, conf)
566
+ super().__init__(name, conf)
569
567
  self.notifier = notifier.Notifier()
570
568
 
571
569
 
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2013 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -77,7 +75,7 @@ def _bulk_register(watch_states, notifier, cb, details_filter=None):
77
75
  return registered
78
76
 
79
77
 
80
- class Listener(object):
78
+ class Listener:
81
79
  """Base class for listeners.
82
80
 
83
81
  A listener can be attached to an engine to do various actions on flow and
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (C) 2015 Yahoo! Inc. All Rights Reserved.
4
2
  #
5
3
  # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -65,7 +63,7 @@ class CaptureListener(base.Listener):
65
63
  # Provide your own list (or previous list) to accumulate
66
64
  # into...
67
65
  values=None):
68
- super(CaptureListener, self).__init__(
66
+ super().__init__(
69
67
  engine,
70
68
  task_listen_for=task_listen_for,
71
69
  flow_listen_for=flow_listen_for,