taskflow 5.11.0__py3-none-any.whl → 6.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- taskflow/atom.py +3 -5
- taskflow/conductors/backends/__init__.py +0 -2
- taskflow/conductors/backends/impl_blocking.py +1 -3
- taskflow/conductors/backends/impl_executor.py +3 -6
- taskflow/conductors/backends/impl_nonblocking.py +1 -3
- taskflow/conductors/base.py +1 -3
- taskflow/deciders.py +0 -2
- taskflow/engines/__init__.py +0 -2
- taskflow/engines/action_engine/actions/base.py +1 -3
- taskflow/engines/action_engine/actions/retry.py +1 -3
- taskflow/engines/action_engine/actions/task.py +1 -3
- taskflow/engines/action_engine/builder.py +2 -4
- taskflow/engines/action_engine/compiler.py +9 -11
- taskflow/engines/action_engine/completer.py +5 -7
- taskflow/engines/action_engine/deciders.py +1 -3
- taskflow/engines/action_engine/engine.py +4 -26
- taskflow/engines/action_engine/executor.py +2 -4
- taskflow/engines/action_engine/runtime.py +1 -3
- taskflow/engines/action_engine/scheduler.py +3 -5
- taskflow/engines/action_engine/scopes.py +3 -5
- taskflow/engines/action_engine/selector.py +1 -3
- taskflow/engines/action_engine/traversal.py +0 -2
- taskflow/engines/base.py +1 -3
- taskflow/engines/helpers.py +0 -2
- taskflow/engines/worker_based/dispatcher.py +2 -4
- taskflow/engines/worker_based/endpoint.py +1 -3
- taskflow/engines/worker_based/engine.py +1 -4
- taskflow/engines/worker_based/executor.py +0 -2
- taskflow/engines/worker_based/protocol.py +1 -3
- taskflow/engines/worker_based/proxy.py +2 -4
- taskflow/engines/worker_based/server.py +3 -5
- taskflow/engines/worker_based/types.py +6 -7
- taskflow/engines/worker_based/worker.py +4 -6
- taskflow/examples/99_bottles.py +0 -2
- taskflow/examples/alphabet_soup.py +0 -2
- taskflow/examples/build_a_car.py +1 -3
- taskflow/examples/buildsystem.py +1 -3
- taskflow/examples/calculate_in_parallel.py +1 -3
- taskflow/examples/calculate_linear.py +3 -5
- taskflow/examples/create_parallel_volume.py +2 -5
- taskflow/examples/delayed_return.py +1 -3
- taskflow/examples/distance_calculator.py +12 -24
- taskflow/examples/dump_memory_backend.py +1 -3
- taskflow/examples/echo_listener.py +0 -2
- taskflow/examples/example_utils.py +1 -3
- taskflow/examples/fake_billing.py +9 -11
- taskflow/examples/graph_flow.py +2 -4
- taskflow/examples/hello_world.py +2 -4
- taskflow/examples/jobboard_produce_consume_colors.py +3 -5
- taskflow/examples/parallel_table_multiply.py +1 -3
- taskflow/examples/persistence_example.py +1 -3
- taskflow/examples/pseudo_scoping.py +2 -4
- taskflow/examples/resume_from_backend.py +1 -3
- taskflow/examples/resume_many_flows/my_flows.py +0 -2
- taskflow/examples/resume_many_flows/resume_all.py +1 -3
- taskflow/examples/resume_many_flows/run_flow.py +2 -4
- taskflow/examples/resume_many_flows.py +0 -2
- taskflow/examples/resume_vm_boot.py +12 -15
- taskflow/examples/resume_volume_create.py +3 -5
- taskflow/examples/retry_flow.py +0 -2
- taskflow/examples/reverting_linear.py +1 -3
- taskflow/examples/run_by_iter.py +0 -2
- taskflow/examples/run_by_iter_enumerate.py +1 -3
- taskflow/examples/share_engine_thread.py +2 -4
- taskflow/examples/simple_linear.py +0 -2
- taskflow/examples/simple_linear_listening.py +1 -3
- taskflow/examples/simple_linear_pass.py +0 -2
- taskflow/examples/simple_map_reduce.py +0 -2
- taskflow/examples/switch_graph_flow.py +1 -3
- taskflow/examples/timing_listener.py +1 -3
- taskflow/examples/tox_conductor.py +3 -5
- taskflow/examples/wbe_event_sender.py +0 -2
- taskflow/examples/wbe_mandelbrot.py +0 -2
- taskflow/examples/wbe_simple_linear.py +0 -2
- taskflow/examples/wrapped_exception.py +0 -2
- taskflow/exceptions.py +7 -9
- taskflow/flow.py +1 -3
- taskflow/formatters.py +22 -8
- taskflow/jobs/backends/__init__.py +0 -2
- taskflow/jobs/backends/impl_redis.py +22 -18
- taskflow/jobs/backends/impl_zookeeper.py +7 -8
- taskflow/jobs/base.py +5 -7
- taskflow/listeners/base.py +1 -3
- taskflow/listeners/capturing.py +1 -3
- taskflow/listeners/claims.py +1 -3
- taskflow/listeners/logging.py +10 -6
- taskflow/listeners/printing.py +1 -3
- taskflow/listeners/timing.py +10 -13
- taskflow/logging.py +0 -2
- taskflow/patterns/graph_flow.py +2 -4
- taskflow/patterns/linear_flow.py +4 -9
- taskflow/patterns/unordered_flow.py +4 -9
- taskflow/persistence/backends/__init__.py +1 -3
- taskflow/persistence/backends/impl_dir.py +3 -6
- taskflow/persistence/backends/impl_memory.py +6 -9
- taskflow/persistence/backends/impl_sqlalchemy.py +4 -6
- taskflow/persistence/backends/impl_zookeeper.py +2 -4
- taskflow/persistence/backends/sqlalchemy/alembic/env.py +0 -2
- taskflow/persistence/backends/sqlalchemy/alembic/versions/00af93df9d77_add_unique_into_all_indexes.py +80 -0
- taskflow/persistence/backends/sqlalchemy/alembic/versions/14b227d79a87_add_intention_column.py +0 -2
- taskflow/persistence/backends/sqlalchemy/alembic/versions/1c783c0c2875_replace_exception_an.py +0 -2
- taskflow/persistence/backends/sqlalchemy/alembic/versions/1cea328f0f65_initial_logbook_deta.py +0 -2
- taskflow/persistence/backends/sqlalchemy/alembic/versions/2ad4984f2864_switch_postgres_to_json_native.py +0 -2
- taskflow/persistence/backends/sqlalchemy/alembic/versions/3162c0f3f8e4_add_revert_results_and_revert_failure_.py +0 -2
- taskflow/persistence/backends/sqlalchemy/alembic/versions/589dccdf2b6e_rename_taskdetails_to_atomdetails.py +0 -2
- taskflow/persistence/backends/sqlalchemy/alembic/versions/84d6e888850_add_task_detail_type.py +0 -2
- taskflow/persistence/backends/sqlalchemy/tables.py +0 -2
- taskflow/persistence/base.py +2 -4
- taskflow/persistence/models.py +19 -24
- taskflow/persistence/path_based.py +1 -3
- taskflow/retry.py +12 -18
- taskflow/states.py +0 -2
- taskflow/storage.py +9 -11
- taskflow/task.py +15 -18
- taskflow/test.py +5 -5
- taskflow/tests/test_examples.py +0 -2
- taskflow/tests/unit/action_engine/test_builder.py +1 -3
- taskflow/tests/unit/action_engine/test_compile.py +4 -6
- taskflow/tests/unit/action_engine/test_creation.py +0 -21
- taskflow/tests/unit/action_engine/test_scoping.py +1 -3
- taskflow/tests/unit/jobs/base.py +1 -3
- taskflow/tests/unit/jobs/test_entrypoint.py +0 -2
- taskflow/tests/unit/jobs/test_redis_job.py +1 -3
- taskflow/tests/unit/jobs/test_zk_job.py +2 -4
- taskflow/tests/unit/patterns/test_graph_flow.py +18 -20
- taskflow/tests/unit/patterns/test_linear_flow.py +7 -9
- taskflow/tests/unit/patterns/test_unordered_flow.py +12 -14
- taskflow/tests/unit/persistence/base.py +2 -4
- taskflow/tests/unit/persistence/test_dir_persistence.py +2 -4
- taskflow/tests/unit/persistence/test_memory_persistence.py +2 -4
- taskflow/tests/unit/persistence/test_sql_persistence.py +4 -6
- taskflow/tests/unit/persistence/test_zk_persistence.py +2 -4
- taskflow/tests/unit/test_arguments_passing.py +3 -25
- taskflow/tests/unit/test_check_transition.py +3 -5
- taskflow/tests/unit/test_conductors.py +0 -2
- taskflow/tests/unit/test_deciders.py +1 -3
- taskflow/tests/unit/test_engine_helpers.py +0 -2
- taskflow/tests/unit/test_engines.py +19 -102
- taskflow/tests/unit/test_exceptions.py +4 -6
- taskflow/tests/unit/test_failure.py +10 -12
- taskflow/tests/unit/test_flow_dependencies.py +62 -64
- taskflow/tests/unit/test_formatters.py +0 -2
- taskflow/tests/unit/test_functor_task.py +1 -3
- taskflow/tests/unit/test_listeners.py +9 -11
- taskflow/tests/unit/test_mapfunctor_task.py +1 -3
- taskflow/tests/unit/test_notifier.py +1 -3
- taskflow/tests/unit/test_progress.py +2 -4
- taskflow/tests/unit/test_reducefunctor_task.py +1 -3
- taskflow/tests/unit/test_retries.py +1 -25
- taskflow/tests/unit/test_states.py +0 -2
- taskflow/tests/unit/test_storage.py +10 -12
- taskflow/tests/unit/test_suspend.py +2 -23
- taskflow/tests/unit/test_task.py +17 -19
- taskflow/tests/unit/test_types.py +4 -6
- taskflow/tests/unit/test_utils.py +9 -11
- taskflow/tests/unit/test_utils_async_utils.py +0 -2
- taskflow/tests/unit/test_utils_binary.py +12 -14
- taskflow/tests/unit/test_utils_iter_utils.py +1 -3
- taskflow/tests/unit/test_utils_kazoo_utils.py +0 -2
- taskflow/tests/unit/test_utils_threading_utils.py +1 -3
- taskflow/tests/unit/worker_based/test_creation.py +0 -2
- taskflow/tests/unit/worker_based/test_dispatcher.py +0 -2
- taskflow/tests/unit/worker_based/test_endpoint.py +2 -4
- taskflow/tests/unit/worker_based/test_executor.py +1 -3
- taskflow/tests/unit/worker_based/test_message_pump.py +0 -2
- taskflow/tests/unit/worker_based/test_pipeline.py +0 -2
- taskflow/tests/unit/worker_based/test_protocol.py +2 -4
- taskflow/tests/unit/worker_based/test_proxy.py +2 -4
- taskflow/tests/unit/worker_based/test_server.py +3 -5
- taskflow/tests/unit/worker_based/test_types.py +0 -2
- taskflow/tests/unit/worker_based/test_worker.py +1 -3
- taskflow/tests/utils.py +11 -15
- taskflow/types/entity.py +1 -3
- taskflow/types/failure.py +3 -6
- taskflow/types/graph.py +14 -18
- taskflow/types/latch.py +1 -3
- taskflow/types/notifier.py +6 -9
- taskflow/types/sets.py +2 -5
- taskflow/types/timing.py +1 -3
- taskflow/types/tree.py +6 -10
- taskflow/utils/async_utils.py +0 -2
- taskflow/utils/banner.py +8 -9
- taskflow/utils/eventlet_utils.py +2 -2
- taskflow/utils/iter_utils.py +0 -2
- taskflow/utils/kazoo_utils.py +7 -9
- taskflow/utils/kombu_utils.py +5 -7
- taskflow/utils/misc.py +4 -6
- taskflow/utils/persistence_utils.py +0 -2
- taskflow/utils/redis_utils.py +2 -4
- taskflow/utils/schema_utils.py +0 -2
- taskflow/utils/threading_utils.py +1 -3
- taskflow/version.py +1 -3
- {taskflow-5.11.0.dist-info → taskflow-6.0.0.dist-info}/AUTHORS +1 -0
- {taskflow-5.11.0.dist-info → taskflow-6.0.0.dist-info}/METADATA +2 -1
- taskflow-6.0.0.dist-info/RECORD +243 -0
- {taskflow-5.11.0.dist-info → taskflow-6.0.0.dist-info}/WHEEL +1 -1
- taskflow-6.0.0.dist-info/pbr.json +1 -0
- taskflow/engines/action_engine/process_executor.py +0 -720
- taskflow/tests/unit/action_engine/test_process_executor.py +0 -106
- taskflow-5.11.0.dist-info/RECORD +0 -244
- taskflow-5.11.0.dist-info/pbr.json +0 -1
- {taskflow-5.11.0.dist-info → taskflow-6.0.0.dist-info}/LICENSE +0 -0
- {taskflow-5.11.0.dist-info → taskflow-6.0.0.dist-info}/entry_points.txt +0 -0
- {taskflow-5.11.0.dist-info → taskflow-6.0.0.dist-info}/top_level.txt +0 -0
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2013 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -63,7 +61,7 @@ class PrintText(task.Task):
|
|
|
63
61
|
"""Just inserts some text print outs in a workflow."""
|
|
64
62
|
def __init__(self, print_what, no_slow=False):
|
|
65
63
|
content_hash = hashlib.md5(print_what.encode('utf-8')).hexdigest()[0:8]
|
|
66
|
-
super(
|
|
64
|
+
super().__init__(name="Print: %s" % (content_hash))
|
|
67
65
|
self._text = print_what
|
|
68
66
|
self._no_slow = no_slow
|
|
69
67
|
|
|
@@ -78,7 +76,7 @@ class PrintText(task.Task):
|
|
|
78
76
|
class DefineVMSpec(task.Task):
|
|
79
77
|
"""Defines a vm specification to be."""
|
|
80
78
|
def __init__(self, name):
|
|
81
|
-
super(
|
|
79
|
+
super().__init__(provides='vm_spec', name=name)
|
|
82
80
|
|
|
83
81
|
def execute(self):
|
|
84
82
|
return {
|
|
@@ -93,8 +91,7 @@ class DefineVMSpec(task.Task):
|
|
|
93
91
|
class LocateImages(task.Task):
|
|
94
92
|
"""Locates where the vm images are."""
|
|
95
93
|
def __init__(self, name):
|
|
96
|
-
super(
|
|
97
|
-
name=name)
|
|
94
|
+
super().__init__(provides='image_locations', name=name)
|
|
98
95
|
|
|
99
96
|
def execute(self, vm_spec):
|
|
100
97
|
image_locations = {}
|
|
@@ -107,13 +104,13 @@ class LocateImages(task.Task):
|
|
|
107
104
|
class DownloadImages(task.Task):
|
|
108
105
|
"""Downloads all the vm images."""
|
|
109
106
|
def __init__(self, name):
|
|
110
|
-
super(
|
|
111
|
-
|
|
107
|
+
super().__init__(provides='download_paths',
|
|
108
|
+
name=name)
|
|
112
109
|
|
|
113
110
|
def execute(self, image_locations):
|
|
114
111
|
for src, loc in image_locations.items():
|
|
115
112
|
with slow_down(1):
|
|
116
|
-
print("Downloading from
|
|
113
|
+
print("Downloading from {} => {}".format(src, loc))
|
|
117
114
|
return sorted(image_locations.values())
|
|
118
115
|
|
|
119
116
|
|
|
@@ -125,8 +122,8 @@ IPADDR=%s
|
|
|
125
122
|
ONBOOT=yes"""
|
|
126
123
|
|
|
127
124
|
def __init__(self, name):
|
|
128
|
-
super(
|
|
129
|
-
|
|
125
|
+
super().__init__(provides='network_settings',
|
|
126
|
+
name=name)
|
|
130
127
|
|
|
131
128
|
def execute(self, ips):
|
|
132
129
|
settings = []
|
|
@@ -138,7 +135,7 @@ ONBOOT=yes"""
|
|
|
138
135
|
class AllocateIP(task.Task):
|
|
139
136
|
"""Allocates the ips for the given vm."""
|
|
140
137
|
def __init__(self, name):
|
|
141
|
-
super(
|
|
138
|
+
super().__init__(provides='ips', name=name)
|
|
142
139
|
|
|
143
140
|
def execute(self, vm_spec):
|
|
144
141
|
ips = []
|
|
@@ -152,7 +149,7 @@ class WriteNetworkSettings(task.Task):
|
|
|
152
149
|
def execute(self, download_paths, network_settings):
|
|
153
150
|
for j, path in enumerate(download_paths):
|
|
154
151
|
with slow_down(1):
|
|
155
|
-
print("Mounting
|
|
152
|
+
print("Mounting {} to /tmp/{}".format(path, j))
|
|
156
153
|
for i, setting in enumerate(network_settings):
|
|
157
154
|
filename = ("/tmp/etc/sysconfig/network-scripts/"
|
|
158
155
|
"ifcfg-eth%s" % (i))
|
|
@@ -263,8 +260,8 @@ with eu.get_backend() as backend:
|
|
|
263
260
|
backend=backend, book=book,
|
|
264
261
|
engine='parallel',
|
|
265
262
|
executor=executor)
|
|
266
|
-
print("!! Your tracking id is: '
|
|
267
|
-
|
|
263
|
+
print("!! Your tracking id is: '{}+{}'".format(
|
|
264
|
+
book.uuid, engine.storage.flow_uuid))
|
|
268
265
|
print("!! Please submit this on later runs for tracking purposes")
|
|
269
266
|
else:
|
|
270
267
|
# Attempt to load from a previously partially completed flow.
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2013 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -72,7 +70,7 @@ def find_flow_detail(backend, book_id, flow_id):
|
|
|
72
70
|
class PrintText(task.Task):
|
|
73
71
|
def __init__(self, print_what, no_slow=False):
|
|
74
72
|
content_hash = hashlib.md5(print_what.encode('utf-8')).hexdigest()[0:8]
|
|
75
|
-
super(
|
|
73
|
+
super().__init__(name="Print: %s" % (content_hash))
|
|
76
74
|
self._text = print_what
|
|
77
75
|
self._no_slow = no_slow
|
|
78
76
|
|
|
@@ -141,8 +139,8 @@ with example_utils.get_backend() as backend:
|
|
|
141
139
|
book.add(flow_detail)
|
|
142
140
|
with contextlib.closing(backend.get_connection()) as conn:
|
|
143
141
|
conn.save_logbook(book)
|
|
144
|
-
print("!! Your tracking id is: '
|
|
145
|
-
|
|
142
|
+
print("!! Your tracking id is: '{}+{}'".format(book.uuid,
|
|
143
|
+
flow_detail.uuid))
|
|
146
144
|
print("!! Please submit this on later runs for tracking purposes")
|
|
147
145
|
else:
|
|
148
146
|
flow_detail = find_flow_detail(backend, book_id, flow_id)
|
taskflow/examples/retry_flow.py
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -59,7 +57,7 @@ class CallJoe(task.Task):
|
|
|
59
57
|
|
|
60
58
|
class CallSuzzie(task.Task):
|
|
61
59
|
def execute(self, suzzie_number, *args, **kwargs):
|
|
62
|
-
raise
|
|
60
|
+
raise OSError("Suzzie not home right now.")
|
|
63
61
|
|
|
64
62
|
|
|
65
63
|
# Create your flow and associated tasks (the work to be done).
|
taskflow/examples/run_by_iter.py
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -51,4 +49,4 @@ e.compile()
|
|
|
51
49
|
e.prepare()
|
|
52
50
|
|
|
53
51
|
for i, st in enumerate(e.run_iter(), 1):
|
|
54
|
-
print("Transition
|
|
52
|
+
print("Transition {}: {}".format(i, st))
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -43,11 +41,11 @@ from taskflow.utils import threading_utils as tu
|
|
|
43
41
|
|
|
44
42
|
class DelayedTask(task.Task):
|
|
45
43
|
def __init__(self, name):
|
|
46
|
-
super(
|
|
44
|
+
super().__init__(name=name)
|
|
47
45
|
self._wait_for = random.random()
|
|
48
46
|
|
|
49
47
|
def execute(self):
|
|
50
|
-
print("Running '
|
|
48
|
+
print("Running '{}' in thread '{}'".format(self.name, tu.get_ident()))
|
|
51
49
|
time.sleep(self._wait_for)
|
|
52
50
|
|
|
53
51
|
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2012-2013 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -71,7 +69,7 @@ def flow_watch(state, details):
|
|
|
71
69
|
|
|
72
70
|
|
|
73
71
|
def task_watch(state, details):
|
|
74
|
-
print('Task
|
|
72
|
+
print('Task {} => {}'.format(details.get('task_name'), state))
|
|
75
73
|
|
|
76
74
|
|
|
77
75
|
# Wrap your functions into a task type that knows how to treat your functions
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -64,7 +62,7 @@ while entries:
|
|
|
64
62
|
path = entries.pop()
|
|
65
63
|
value = backend.memory[path]
|
|
66
64
|
if value:
|
|
67
|
-
print("
|
|
65
|
+
print("{} -> {}".format(path, value))
|
|
68
66
|
else:
|
|
69
67
|
print("%s" % (path))
|
|
70
68
|
entries.extend(os.path.join(path, child)
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -45,7 +43,7 @@ from taskflow import task
|
|
|
45
43
|
|
|
46
44
|
class VariableTask(task.Task):
|
|
47
45
|
def __init__(self, name):
|
|
48
|
-
super(
|
|
46
|
+
super().__init__(name)
|
|
49
47
|
self._sleepy_time = random.random()
|
|
50
48
|
|
|
51
49
|
def execute(self):
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -60,7 +58,7 @@ from taskflow.utils import threading_utils
|
|
|
60
58
|
RUN_TIME = 5
|
|
61
59
|
REVIEW_CREATION_DELAY = 0.5
|
|
62
60
|
SCAN_DELAY = 0.1
|
|
63
|
-
NAME = "
|
|
61
|
+
NAME = "{}_{}".format(socket.getfqdn(), os.getpid())
|
|
64
62
|
|
|
65
63
|
# This won't really use zookeeper but will use a local version of it using
|
|
66
64
|
# the zake library that mimics an actual zookeeper cluster using threads and
|
|
@@ -74,7 +72,7 @@ class RunReview(task.Task):
|
|
|
74
72
|
# A dummy task that clones the review and runs tox...
|
|
75
73
|
|
|
76
74
|
def _clone_review(self, review, temp_dir):
|
|
77
|
-
print("Cloning review '
|
|
75
|
+
print("Cloning review '{}' into {}".format(review['id'], temp_dir))
|
|
78
76
|
|
|
79
77
|
def _run_tox(self, temp_dir):
|
|
80
78
|
print("Running tox in %s" % temp_dir)
|
|
@@ -177,7 +175,7 @@ def generate_reviewer(client, saver, name=NAME):
|
|
|
177
175
|
'review': review,
|
|
178
176
|
},
|
|
179
177
|
}
|
|
180
|
-
job_name = "
|
|
178
|
+
job_name = "{}_{}".format(real_name, review['id'])
|
|
181
179
|
print("Posting review '%s'" % review['id'])
|
|
182
180
|
jb.post(job_name,
|
|
183
181
|
book=make_save_book(saver, review['id']),
|
taskflow/exceptions.py
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -67,7 +65,7 @@ class TaskFlowException(Exception):
|
|
|
67
65
|
this is not yet implemented/supported natively.
|
|
68
66
|
"""
|
|
69
67
|
def __init__(self, message, cause=None):
|
|
70
|
-
super(
|
|
68
|
+
super().__init__(message)
|
|
71
69
|
self._cause = cause
|
|
72
70
|
|
|
73
71
|
@property
|
|
@@ -192,7 +190,7 @@ class MissingDependencies(DependencyFailure):
|
|
|
192
190
|
message = self.MESSAGE_TPL % {'who': who, 'requirements': requirements}
|
|
193
191
|
if method:
|
|
194
192
|
message = (self.METHOD_TPL % {'method': method}) + message
|
|
195
|
-
super(
|
|
193
|
+
super().__init__(message, cause=cause)
|
|
196
194
|
self.missing_requirements = requirements
|
|
197
195
|
|
|
198
196
|
|
|
@@ -228,7 +226,7 @@ class DisallowedAccess(TaskFlowException):
|
|
|
228
226
|
"""Raised when storage access is not possible due to state limitations."""
|
|
229
227
|
|
|
230
228
|
def __init__(self, message, cause=None, state=None):
|
|
231
|
-
super(
|
|
229
|
+
super().__init__(message, cause=cause)
|
|
232
230
|
self.state = state
|
|
233
231
|
|
|
234
232
|
|
|
@@ -261,7 +259,7 @@ class WrappedFailure(Exception):
|
|
|
261
259
|
"""
|
|
262
260
|
|
|
263
261
|
def __init__(self, causes):
|
|
264
|
-
super(
|
|
262
|
+
super().__init__()
|
|
265
263
|
self._causes = []
|
|
266
264
|
for cause in causes:
|
|
267
265
|
if cause.check(type(self)) and cause.exception:
|
|
@@ -306,8 +304,8 @@ class WrappedFailure(Exception):
|
|
|
306
304
|
|
|
307
305
|
def __str__(self):
|
|
308
306
|
buf = io.StringIO()
|
|
309
|
-
buf.write(
|
|
307
|
+
buf.write('WrappedFailure: [')
|
|
310
308
|
causes_gen = (str(cause) for cause in self._causes)
|
|
311
|
-
buf.write(
|
|
312
|
-
buf.write(
|
|
309
|
+
buf.write(", ".join(causes_gen))
|
|
310
|
+
buf.write(']')
|
|
313
311
|
return buf.getvalue()
|
taskflow/flow.py
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -42,7 +40,7 @@ _CHOP_PAT_LEN = len(_CHOP_PAT)
|
|
|
42
40
|
LINK_DECIDER_DEPTH = 'decider_depth'
|
|
43
41
|
|
|
44
42
|
|
|
45
|
-
class Flow(
|
|
43
|
+
class Flow(metaclass=abc.ABCMeta):
|
|
46
44
|
"""The base abstract class of all flow implementations.
|
|
47
45
|
|
|
48
46
|
A flow is a structure that defines relationships between tasks. You can
|
taskflow/formatters.py
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -14,6 +12,7 @@
|
|
|
14
12
|
# License for the specific language governing permissions and limitations
|
|
15
13
|
# under the License.
|
|
16
14
|
|
|
15
|
+
import copy
|
|
17
16
|
import functools
|
|
18
17
|
|
|
19
18
|
from taskflow.engines.action_engine import compiler
|
|
@@ -58,17 +57,29 @@ def _fetch_predecessor_tree(graph, atom):
|
|
|
58
57
|
return root
|
|
59
58
|
|
|
60
59
|
|
|
61
|
-
class FailureFormatter
|
|
60
|
+
class FailureFormatter:
|
|
62
61
|
"""Formats a failure and connects it to associated atoms & engine."""
|
|
63
62
|
|
|
64
63
|
_BUILDERS = {
|
|
65
64
|
states.EXECUTE: (_fetch_predecessor_tree, 'predecessors'),
|
|
66
65
|
}
|
|
67
66
|
|
|
68
|
-
def __init__(self, engine, hide_inputs_outputs_of=()
|
|
67
|
+
def __init__(self, engine, hide_inputs_outputs_of=(),
|
|
68
|
+
mask_inputs_keys=(), mask_outputs_keys=()):
|
|
69
69
|
self._hide_inputs_outputs_of = hide_inputs_outputs_of
|
|
70
|
+
self._mask_inputs_keys = mask_inputs_keys
|
|
71
|
+
self._mask_outputs_keys = mask_outputs_keys
|
|
70
72
|
self._engine = engine
|
|
71
73
|
|
|
74
|
+
def _mask_keys(self, data, mask_keys):
|
|
75
|
+
if not data or not isinstance(data, dict):
|
|
76
|
+
return data
|
|
77
|
+
result = copy.deepcopy(data)
|
|
78
|
+
for k in mask_keys:
|
|
79
|
+
if k in result:
|
|
80
|
+
result[k] = '***'
|
|
81
|
+
return result
|
|
82
|
+
|
|
72
83
|
def _format_node(self, storage, cache, node):
|
|
73
84
|
"""Formats a single tree node into a string version."""
|
|
74
85
|
if node.metadata['kind'] == compiler.FLOW:
|
|
@@ -100,14 +111,16 @@ class FailureFormatter(object):
|
|
|
100
111
|
atom_name,
|
|
101
112
|
fetch_mapped_args)
|
|
102
113
|
if requires_found:
|
|
103
|
-
atom_attrs['requires'] =
|
|
114
|
+
atom_attrs['requires'] = self._mask_keys(
|
|
115
|
+
requires, self._mask_inputs_keys)
|
|
104
116
|
provides, provides_found = _cached_get(
|
|
105
117
|
cache, 'provides', atom_name,
|
|
106
118
|
storage.get_execute_result, atom_name)
|
|
107
119
|
if provides_found:
|
|
108
|
-
atom_attrs['provides'] =
|
|
120
|
+
atom_attrs['provides'] = self._mask_keys(
|
|
121
|
+
provides, self._mask_outputs_keys)
|
|
109
122
|
if atom_attrs:
|
|
110
|
-
return "Atom '
|
|
123
|
+
return "Atom '{}' {}".format(atom_name, atom_attrs)
|
|
111
124
|
else:
|
|
112
125
|
return "Atom '%s'" % (atom_name)
|
|
113
126
|
else:
|
|
@@ -156,7 +169,8 @@ class FailureFormatter(object):
|
|
|
156
169
|
builder, kind = self._BUILDERS[atom_intention]
|
|
157
170
|
rooted_tree = builder(graph, atom)
|
|
158
171
|
child_count = rooted_tree.child_count(only_direct=False)
|
|
159
|
-
buff.write_nl(
|
|
172
|
+
buff.write_nl(
|
|
173
|
+
'{} {} (most recent first):'.format(child_count, kind))
|
|
160
174
|
formatter = functools.partial(self._format_node, storage, cache)
|
|
161
175
|
direct_child_count = rooted_tree.child_count(only_direct=True)
|
|
162
176
|
for i, child in enumerate(rooted_tree, 1):
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2015 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -17,7 +15,6 @@
|
|
|
17
15
|
import contextlib
|
|
18
16
|
import datetime
|
|
19
17
|
import functools
|
|
20
|
-
import re
|
|
21
18
|
import string
|
|
22
19
|
import threading
|
|
23
20
|
import time
|
|
@@ -26,6 +23,7 @@ import fasteners
|
|
|
26
23
|
import msgpack
|
|
27
24
|
from oslo_serialization import msgpackutils
|
|
28
25
|
from oslo_utils import excutils
|
|
26
|
+
from oslo_utils import netutils
|
|
29
27
|
from oslo_utils import strutils
|
|
30
28
|
from oslo_utils import timeutils
|
|
31
29
|
from oslo_utils import uuidutils
|
|
@@ -69,10 +67,10 @@ class RedisJob(base.Job):
|
|
|
69
67
|
created_on=None, backend=None,
|
|
70
68
|
book=None, book_data=None,
|
|
71
69
|
priority=base.JobPriority.NORMAL):
|
|
72
|
-
super(
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
70
|
+
super().__init__(board, name,
|
|
71
|
+
uuid=uuid, details=details,
|
|
72
|
+
backend=backend,
|
|
73
|
+
book=book, book_data=book_data)
|
|
76
74
|
self._created_on = created_on
|
|
77
75
|
self._client = board._client
|
|
78
76
|
self._redis_version = board._redis_version
|
|
@@ -561,15 +559,17 @@ return cmsgpack.pack(result)
|
|
|
561
559
|
|
|
562
560
|
@classmethod
|
|
563
561
|
def _parse_sentinel(cls, sentinel):
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
562
|
+
host, port = netutils.parse_host_port(sentinel)
|
|
563
|
+
if host is None or port is None:
|
|
564
|
+
raise ValueError('Malformed sentinel server format')
|
|
565
|
+
return (host, port)
|
|
566
|
+
|
|
567
|
+
@classmethod
|
|
568
|
+
def _filter_ssl_options(cls, opts):
|
|
569
|
+
if not opts.get('ssl', False):
|
|
570
|
+
return {k: v for (k, v) in opts.items()
|
|
571
|
+
if not k.startswith('ssl_')}
|
|
572
|
+
return opts
|
|
573
573
|
|
|
574
574
|
@classmethod
|
|
575
575
|
def _make_client(cls, conf):
|
|
@@ -584,8 +584,12 @@ return cmsgpack.pack(result)
|
|
|
584
584
|
sentinels = [(client_conf.pop('host'), client_conf.pop('port'))]
|
|
585
585
|
for fallback in conf.get('sentinel_fallbacks', []):
|
|
586
586
|
sentinels.append(cls._parse_sentinel(fallback))
|
|
587
|
+
client_conf = cls._filter_ssl_options(client_conf)
|
|
588
|
+
sentinel_kwargs = conf.get('sentinel_kwargs')
|
|
589
|
+
if sentinel_kwargs is not None:
|
|
590
|
+
sentinel_kwargs = cls._filter_ssl_options(sentinel_kwargs)
|
|
587
591
|
s = sentinel.Sentinel(sentinels,
|
|
588
|
-
sentinel_kwargs=
|
|
592
|
+
sentinel_kwargs=sentinel_kwargs,
|
|
589
593
|
**client_conf)
|
|
590
594
|
return s.master_for(conf['sentinel'])
|
|
591
595
|
else:
|
|
@@ -593,7 +597,7 @@ return cmsgpack.pack(result)
|
|
|
593
597
|
|
|
594
598
|
def __init__(self, name, conf,
|
|
595
599
|
client=None, persistence=None):
|
|
596
|
-
super(
|
|
600
|
+
super().__init__(name, conf)
|
|
597
601
|
self._closed = True
|
|
598
602
|
if client is not None:
|
|
599
603
|
self._client = client
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2013 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -51,10 +49,10 @@ class ZookeeperJob(base.Job):
|
|
|
51
49
|
uuid=None, details=None, book=None, book_data=None,
|
|
52
50
|
created_on=None, backend=None,
|
|
53
51
|
priority=base.JobPriority.NORMAL):
|
|
54
|
-
super(
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
52
|
+
super().__init__(board, name,
|
|
53
|
+
uuid=uuid, details=details,
|
|
54
|
+
backend=backend,
|
|
55
|
+
book=book, book_data=book_data)
|
|
58
56
|
self._client = client
|
|
59
57
|
self._path = k_paths.normpath(path)
|
|
60
58
|
self._lock_path = self._path + board.LOCK_POSTFIX
|
|
@@ -281,7 +279,7 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
|
|
|
281
279
|
|
|
282
280
|
def __init__(self, name, conf,
|
|
283
281
|
client=None, persistence=None, emit_notifications=True):
|
|
284
|
-
super(
|
|
282
|
+
super().__init__(name, conf)
|
|
285
283
|
if client is not None:
|
|
286
284
|
self._client = client
|
|
287
285
|
self._owned = False
|
|
@@ -552,7 +550,8 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
|
|
|
552
550
|
except Exception:
|
|
553
551
|
owner = None
|
|
554
552
|
if owner:
|
|
555
|
-
message = "Job
|
|
553
|
+
message = "Job {} already claimed by '{}'".format(
|
|
554
|
+
job.uuid, owner)
|
|
556
555
|
else:
|
|
557
556
|
message = "Job %s already claimed" % (job.uuid)
|
|
558
557
|
excp.raise_with_cause(excp.UnclaimableJob,
|
taskflow/jobs/base.py
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2013 Rackspace Hosting Inc. All Rights Reserved.
|
|
4
2
|
# Copyright (C) 2013 Yahoo! Inc. All Rights Reserved.
|
|
5
3
|
#
|
|
@@ -110,7 +108,7 @@ class JobPriority(enum.Enum):
|
|
|
110
108
|
return tuple(values)
|
|
111
109
|
|
|
112
110
|
|
|
113
|
-
class Job(
|
|
111
|
+
class Job(metaclass=abc.ABCMeta):
|
|
114
112
|
"""A abstraction that represents a named and trackable unit of work.
|
|
115
113
|
|
|
116
114
|
A job connects a logbook, a owner, a priority, last modified and created
|
|
@@ -277,12 +275,12 @@ class Job(object, metaclass=abc.ABCMeta):
|
|
|
277
275
|
def __str__(self):
|
|
278
276
|
"""Pretty formats the job into something *more* meaningful."""
|
|
279
277
|
cls_name = type(self).__name__
|
|
280
|
-
return "
|
|
278
|
+
return "{}: {} (priority={}, uuid={}, details={})".format(
|
|
281
279
|
cls_name, self.name, self.priority,
|
|
282
280
|
self.uuid, self.details)
|
|
283
281
|
|
|
284
282
|
|
|
285
|
-
class JobBoardIterator
|
|
283
|
+
class JobBoardIterator:
|
|
286
284
|
"""Iterator over a jobboard that iterates over potential jobs.
|
|
287
285
|
|
|
288
286
|
It provides the following attributes:
|
|
@@ -355,7 +353,7 @@ class JobBoardIterator(object):
|
|
|
355
353
|
return job
|
|
356
354
|
|
|
357
355
|
|
|
358
|
-
class JobBoard(
|
|
356
|
+
class JobBoard(metaclass=abc.ABCMeta):
|
|
359
357
|
"""A place where jobs can be posted, reposted, claimed and transferred.
|
|
360
358
|
|
|
361
359
|
There can be multiple implementations of this job board, depending on the
|
|
@@ -565,7 +563,7 @@ class NotifyingJobBoard(JobBoard):
|
|
|
565
563
|
registered are thread safe (and block for as little time as possible).
|
|
566
564
|
"""
|
|
567
565
|
def __init__(self, name, conf):
|
|
568
|
-
super(
|
|
566
|
+
super().__init__(name, conf)
|
|
569
567
|
self.notifier = notifier.Notifier()
|
|
570
568
|
|
|
571
569
|
|
taskflow/listeners/base.py
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2013 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -77,7 +75,7 @@ def _bulk_register(watch_states, notifier, cb, details_filter=None):
|
|
|
77
75
|
return registered
|
|
78
76
|
|
|
79
77
|
|
|
80
|
-
class Listener
|
|
78
|
+
class Listener:
|
|
81
79
|
"""Base class for listeners.
|
|
82
80
|
|
|
83
81
|
A listener can be attached to an engine to do various actions on flow and
|
taskflow/listeners/capturing.py
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2015 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -65,7 +63,7 @@ class CaptureListener(base.Listener):
|
|
|
65
63
|
# Provide your own list (or previous list) to accumulate
|
|
66
64
|
# into...
|
|
67
65
|
values=None):
|
|
68
|
-
super(
|
|
66
|
+
super().__init__(
|
|
69
67
|
engine,
|
|
70
68
|
task_listen_for=task_listen_for,
|
|
71
69
|
flow_listen_for=flow_listen_for,
|