taskflow 5.11.0__py3-none-any.whl → 6.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- taskflow/atom.py +3 -5
- taskflow/conductors/backends/__init__.py +0 -2
- taskflow/conductors/backends/impl_blocking.py +1 -3
- taskflow/conductors/backends/impl_executor.py +3 -6
- taskflow/conductors/backends/impl_nonblocking.py +1 -3
- taskflow/conductors/base.py +1 -3
- taskflow/deciders.py +0 -2
- taskflow/engines/__init__.py +0 -2
- taskflow/engines/action_engine/actions/base.py +1 -3
- taskflow/engines/action_engine/actions/retry.py +1 -3
- taskflow/engines/action_engine/actions/task.py +1 -3
- taskflow/engines/action_engine/builder.py +2 -4
- taskflow/engines/action_engine/compiler.py +9 -11
- taskflow/engines/action_engine/completer.py +5 -7
- taskflow/engines/action_engine/deciders.py +1 -3
- taskflow/engines/action_engine/engine.py +4 -26
- taskflow/engines/action_engine/executor.py +2 -4
- taskflow/engines/action_engine/runtime.py +1 -3
- taskflow/engines/action_engine/scheduler.py +3 -5
- taskflow/engines/action_engine/scopes.py +3 -5
- taskflow/engines/action_engine/selector.py +1 -3
- taskflow/engines/action_engine/traversal.py +0 -2
- taskflow/engines/base.py +1 -3
- taskflow/engines/helpers.py +0 -2
- taskflow/engines/worker_based/dispatcher.py +2 -4
- taskflow/engines/worker_based/endpoint.py +1 -3
- taskflow/engines/worker_based/engine.py +1 -4
- taskflow/engines/worker_based/executor.py +0 -2
- taskflow/engines/worker_based/protocol.py +1 -3
- taskflow/engines/worker_based/proxy.py +2 -4
- taskflow/engines/worker_based/server.py +3 -5
- taskflow/engines/worker_based/types.py +6 -7
- taskflow/engines/worker_based/worker.py +4 -6
- taskflow/examples/99_bottles.py +0 -2
- taskflow/examples/alphabet_soup.py +0 -2
- taskflow/examples/build_a_car.py +1 -3
- taskflow/examples/buildsystem.py +1 -3
- taskflow/examples/calculate_in_parallel.py +1 -3
- taskflow/examples/calculate_linear.py +3 -5
- taskflow/examples/create_parallel_volume.py +2 -5
- taskflow/examples/delayed_return.py +1 -3
- taskflow/examples/distance_calculator.py +12 -24
- taskflow/examples/dump_memory_backend.py +1 -3
- taskflow/examples/echo_listener.py +0 -2
- taskflow/examples/example_utils.py +1 -3
- taskflow/examples/fake_billing.py +9 -11
- taskflow/examples/graph_flow.py +2 -4
- taskflow/examples/hello_world.py +2 -4
- taskflow/examples/jobboard_produce_consume_colors.py +3 -5
- taskflow/examples/parallel_table_multiply.py +1 -3
- taskflow/examples/persistence_example.py +1 -3
- taskflow/examples/pseudo_scoping.py +2 -4
- taskflow/examples/resume_from_backend.py +1 -3
- taskflow/examples/resume_many_flows/my_flows.py +0 -2
- taskflow/examples/resume_many_flows/resume_all.py +1 -3
- taskflow/examples/resume_many_flows/run_flow.py +2 -4
- taskflow/examples/resume_many_flows.py +0 -2
- taskflow/examples/resume_vm_boot.py +12 -15
- taskflow/examples/resume_volume_create.py +3 -5
- taskflow/examples/retry_flow.py +0 -2
- taskflow/examples/reverting_linear.py +1 -3
- taskflow/examples/run_by_iter.py +0 -2
- taskflow/examples/run_by_iter_enumerate.py +1 -3
- taskflow/examples/share_engine_thread.py +2 -4
- taskflow/examples/simple_linear.py +0 -2
- taskflow/examples/simple_linear_listening.py +1 -3
- taskflow/examples/simple_linear_pass.py +0 -2
- taskflow/examples/simple_map_reduce.py +0 -2
- taskflow/examples/switch_graph_flow.py +1 -3
- taskflow/examples/timing_listener.py +1 -3
- taskflow/examples/tox_conductor.py +3 -5
- taskflow/examples/wbe_event_sender.py +0 -2
- taskflow/examples/wbe_mandelbrot.py +0 -2
- taskflow/examples/wbe_simple_linear.py +0 -2
- taskflow/examples/wrapped_exception.py +0 -2
- taskflow/exceptions.py +7 -9
- taskflow/flow.py +1 -3
- taskflow/formatters.py +22 -8
- taskflow/jobs/backends/__init__.py +0 -2
- taskflow/jobs/backends/impl_redis.py +22 -18
- taskflow/jobs/backends/impl_zookeeper.py +7 -8
- taskflow/jobs/base.py +5 -7
- taskflow/listeners/base.py +1 -3
- taskflow/listeners/capturing.py +1 -3
- taskflow/listeners/claims.py +1 -3
- taskflow/listeners/logging.py +10 -6
- taskflow/listeners/printing.py +1 -3
- taskflow/listeners/timing.py +10 -13
- taskflow/logging.py +0 -2
- taskflow/patterns/graph_flow.py +2 -4
- taskflow/patterns/linear_flow.py +4 -9
- taskflow/patterns/unordered_flow.py +4 -9
- taskflow/persistence/backends/__init__.py +1 -3
- taskflow/persistence/backends/impl_dir.py +3 -6
- taskflow/persistence/backends/impl_memory.py +6 -9
- taskflow/persistence/backends/impl_sqlalchemy.py +4 -6
- taskflow/persistence/backends/impl_zookeeper.py +2 -4
- taskflow/persistence/backends/sqlalchemy/alembic/env.py +0 -2
- taskflow/persistence/backends/sqlalchemy/alembic/versions/00af93df9d77_add_unique_into_all_indexes.py +80 -0
- taskflow/persistence/backends/sqlalchemy/alembic/versions/14b227d79a87_add_intention_column.py +0 -2
- taskflow/persistence/backends/sqlalchemy/alembic/versions/1c783c0c2875_replace_exception_an.py +0 -2
- taskflow/persistence/backends/sqlalchemy/alembic/versions/1cea328f0f65_initial_logbook_deta.py +0 -2
- taskflow/persistence/backends/sqlalchemy/alembic/versions/2ad4984f2864_switch_postgres_to_json_native.py +0 -2
- taskflow/persistence/backends/sqlalchemy/alembic/versions/3162c0f3f8e4_add_revert_results_and_revert_failure_.py +0 -2
- taskflow/persistence/backends/sqlalchemy/alembic/versions/589dccdf2b6e_rename_taskdetails_to_atomdetails.py +0 -2
- taskflow/persistence/backends/sqlalchemy/alembic/versions/84d6e888850_add_task_detail_type.py +0 -2
- taskflow/persistence/backends/sqlalchemy/tables.py +0 -2
- taskflow/persistence/base.py +2 -4
- taskflow/persistence/models.py +19 -24
- taskflow/persistence/path_based.py +1 -3
- taskflow/retry.py +12 -18
- taskflow/states.py +0 -2
- taskflow/storage.py +9 -11
- taskflow/task.py +15 -18
- taskflow/test.py +5 -5
- taskflow/tests/test_examples.py +0 -2
- taskflow/tests/unit/action_engine/test_builder.py +1 -3
- taskflow/tests/unit/action_engine/test_compile.py +4 -6
- taskflow/tests/unit/action_engine/test_creation.py +0 -21
- taskflow/tests/unit/action_engine/test_scoping.py +1 -3
- taskflow/tests/unit/jobs/base.py +1 -3
- taskflow/tests/unit/jobs/test_entrypoint.py +0 -2
- taskflow/tests/unit/jobs/test_redis_job.py +1 -3
- taskflow/tests/unit/jobs/test_zk_job.py +2 -4
- taskflow/tests/unit/patterns/test_graph_flow.py +18 -20
- taskflow/tests/unit/patterns/test_linear_flow.py +7 -9
- taskflow/tests/unit/patterns/test_unordered_flow.py +12 -14
- taskflow/tests/unit/persistence/base.py +2 -4
- taskflow/tests/unit/persistence/test_dir_persistence.py +2 -4
- taskflow/tests/unit/persistence/test_memory_persistence.py +2 -4
- taskflow/tests/unit/persistence/test_sql_persistence.py +4 -6
- taskflow/tests/unit/persistence/test_zk_persistence.py +2 -4
- taskflow/tests/unit/test_arguments_passing.py +3 -25
- taskflow/tests/unit/test_check_transition.py +3 -5
- taskflow/tests/unit/test_conductors.py +0 -2
- taskflow/tests/unit/test_deciders.py +1 -3
- taskflow/tests/unit/test_engine_helpers.py +0 -2
- taskflow/tests/unit/test_engines.py +19 -102
- taskflow/tests/unit/test_exceptions.py +4 -6
- taskflow/tests/unit/test_failure.py +10 -12
- taskflow/tests/unit/test_flow_dependencies.py +62 -64
- taskflow/tests/unit/test_formatters.py +0 -2
- taskflow/tests/unit/test_functor_task.py +1 -3
- taskflow/tests/unit/test_listeners.py +9 -11
- taskflow/tests/unit/test_mapfunctor_task.py +1 -3
- taskflow/tests/unit/test_notifier.py +1 -3
- taskflow/tests/unit/test_progress.py +2 -4
- taskflow/tests/unit/test_reducefunctor_task.py +1 -3
- taskflow/tests/unit/test_retries.py +1 -25
- taskflow/tests/unit/test_states.py +0 -2
- taskflow/tests/unit/test_storage.py +10 -12
- taskflow/tests/unit/test_suspend.py +2 -23
- taskflow/tests/unit/test_task.py +17 -19
- taskflow/tests/unit/test_types.py +4 -6
- taskflow/tests/unit/test_utils.py +9 -11
- taskflow/tests/unit/test_utils_async_utils.py +0 -2
- taskflow/tests/unit/test_utils_binary.py +12 -14
- taskflow/tests/unit/test_utils_iter_utils.py +1 -3
- taskflow/tests/unit/test_utils_kazoo_utils.py +0 -2
- taskflow/tests/unit/test_utils_threading_utils.py +1 -3
- taskflow/tests/unit/worker_based/test_creation.py +0 -2
- taskflow/tests/unit/worker_based/test_dispatcher.py +0 -2
- taskflow/tests/unit/worker_based/test_endpoint.py +2 -4
- taskflow/tests/unit/worker_based/test_executor.py +1 -3
- taskflow/tests/unit/worker_based/test_message_pump.py +0 -2
- taskflow/tests/unit/worker_based/test_pipeline.py +0 -2
- taskflow/tests/unit/worker_based/test_protocol.py +2 -4
- taskflow/tests/unit/worker_based/test_proxy.py +2 -4
- taskflow/tests/unit/worker_based/test_server.py +3 -5
- taskflow/tests/unit/worker_based/test_types.py +0 -2
- taskflow/tests/unit/worker_based/test_worker.py +1 -3
- taskflow/tests/utils.py +11 -15
- taskflow/types/entity.py +1 -3
- taskflow/types/failure.py +3 -6
- taskflow/types/graph.py +14 -18
- taskflow/types/latch.py +1 -3
- taskflow/types/notifier.py +6 -9
- taskflow/types/sets.py +2 -5
- taskflow/types/timing.py +1 -3
- taskflow/types/tree.py +6 -10
- taskflow/utils/async_utils.py +0 -2
- taskflow/utils/banner.py +8 -9
- taskflow/utils/eventlet_utils.py +2 -2
- taskflow/utils/iter_utils.py +0 -2
- taskflow/utils/kazoo_utils.py +7 -9
- taskflow/utils/kombu_utils.py +5 -7
- taskflow/utils/misc.py +4 -6
- taskflow/utils/persistence_utils.py +0 -2
- taskflow/utils/redis_utils.py +2 -4
- taskflow/utils/schema_utils.py +0 -2
- taskflow/utils/threading_utils.py +1 -3
- taskflow/version.py +1 -3
- {taskflow-5.11.0.dist-info → taskflow-6.0.0.dist-info}/AUTHORS +1 -0
- {taskflow-5.11.0.dist-info → taskflow-6.0.0.dist-info}/METADATA +2 -1
- taskflow-6.0.0.dist-info/RECORD +243 -0
- {taskflow-5.11.0.dist-info → taskflow-6.0.0.dist-info}/WHEEL +1 -1
- taskflow-6.0.0.dist-info/pbr.json +1 -0
- taskflow/engines/action_engine/process_executor.py +0 -720
- taskflow/tests/unit/action_engine/test_process_executor.py +0 -106
- taskflow-5.11.0.dist-info/RECORD +0 -244
- taskflow-5.11.0.dist-info/pbr.json +0 -1
- {taskflow-5.11.0.dist-info → taskflow-6.0.0.dist-info}/LICENSE +0 -0
- {taskflow-5.11.0.dist-info → taskflow-6.0.0.dist-info}/entry_points.txt +0 -0
- {taskflow-5.11.0.dist-info → taskflow-6.0.0.dist-info}/top_level.txt +0 -0
taskflow/persistence/models.py
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
# Copyright (C) 2013 Rackspace Hosting All Rights Reserved.
|
|
5
3
|
#
|
|
@@ -43,7 +41,7 @@ def _format_meta(metadata, indent):
|
|
|
43
41
|
# in percent format.
|
|
44
42
|
if k == 'progress' and isinstance(v, misc.NUMERIC_TYPES):
|
|
45
43
|
v = "%0.2f%%" % (v * 100.0)
|
|
46
|
-
lines.append("
|
|
44
|
+
lines.append("{}+ {} = {}".format(" " * (indent + 2), k, v))
|
|
47
45
|
return lines
|
|
48
46
|
|
|
49
47
|
|
|
@@ -55,8 +53,8 @@ def _format_shared(obj, indent):
|
|
|
55
53
|
for attr_name in ("uuid", "state"):
|
|
56
54
|
if not hasattr(obj, attr_name):
|
|
57
55
|
continue
|
|
58
|
-
lines.append("
|
|
59
|
-
|
|
56
|
+
lines.append("{}- {} = {}".format(" " * indent, attr_name,
|
|
57
|
+
getattr(obj, attr_name)))
|
|
60
58
|
return lines
|
|
61
59
|
|
|
62
60
|
|
|
@@ -98,7 +96,7 @@ def _fix_meta(data):
|
|
|
98
96
|
return meta
|
|
99
97
|
|
|
100
98
|
|
|
101
|
-
class LogBook
|
|
99
|
+
class LogBook:
|
|
102
100
|
"""A collection of flow details and associated metadata.
|
|
103
101
|
|
|
104
102
|
Typically this class contains a collection of flow detail entries
|
|
@@ -143,7 +141,7 @@ class LogBook(object):
|
|
|
143
141
|
- created_at = ...
|
|
144
142
|
"""
|
|
145
143
|
cls_name = self.__class__.__name__
|
|
146
|
-
lines = ["
|
|
144
|
+
lines = ["{}{}: '{}'".format(" " * indent, cls_name, self.name)]
|
|
147
145
|
lines.extend(_format_shared(self, indent=indent + 1))
|
|
148
146
|
lines.extend(_format_meta(self.meta, indent=indent + 1))
|
|
149
147
|
if self.created_at is not None:
|
|
@@ -258,8 +256,7 @@ class LogBook(object):
|
|
|
258
256
|
return self._name
|
|
259
257
|
|
|
260
258
|
def __iter__(self):
|
|
261
|
-
|
|
262
|
-
yield fd
|
|
259
|
+
yield from self._flowdetails_by_id.values()
|
|
263
260
|
|
|
264
261
|
def __len__(self):
|
|
265
262
|
return len(self._flowdetails_by_id)
|
|
@@ -288,7 +285,7 @@ class LogBook(object):
|
|
|
288
285
|
return clone
|
|
289
286
|
|
|
290
287
|
|
|
291
|
-
class FlowDetail
|
|
288
|
+
class FlowDetail:
|
|
292
289
|
"""A collection of atom details and associated metadata.
|
|
293
290
|
|
|
294
291
|
Typically this class contains a collection of atom detail entries that
|
|
@@ -345,7 +342,7 @@ class FlowDetail(object):
|
|
|
345
342
|
- state = ...
|
|
346
343
|
"""
|
|
347
344
|
cls_name = self.__class__.__name__
|
|
348
|
-
lines = ["
|
|
345
|
+
lines = ["{}{}: '{}'".format(" " * indent, cls_name, self.name)]
|
|
349
346
|
lines.extend(_format_shared(self, indent=indent + 1))
|
|
350
347
|
lines.extend(_format_meta(self.meta, indent=indent + 1))
|
|
351
348
|
for atom_detail in self:
|
|
@@ -463,14 +460,13 @@ class FlowDetail(object):
|
|
|
463
460
|
return self._name
|
|
464
461
|
|
|
465
462
|
def __iter__(self):
|
|
466
|
-
|
|
467
|
-
yield ad
|
|
463
|
+
yield from self._atomdetails_by_id.values()
|
|
468
464
|
|
|
469
465
|
def __len__(self):
|
|
470
466
|
return len(self._atomdetails_by_id)
|
|
471
467
|
|
|
472
468
|
|
|
473
|
-
class AtomDetail(
|
|
469
|
+
class AtomDetail(metaclass=abc.ABCMeta):
|
|
474
470
|
"""A collection of atom specific runtime information and metadata.
|
|
475
471
|
|
|
476
472
|
This is a base **abstract** class that contains attributes that are used
|
|
@@ -688,14 +684,14 @@ class AtomDetail(object, metaclass=abc.ABCMeta):
|
|
|
688
684
|
def pformat(self, indent=0, linesep=os.linesep):
|
|
689
685
|
"""Pretty formats this atom detail into a string."""
|
|
690
686
|
cls_name = self.__class__.__name__
|
|
691
|
-
lines = ["
|
|
687
|
+
lines = ["{}{}: '{}'".format(" " * (indent), cls_name, self.name)]
|
|
692
688
|
lines.extend(_format_shared(self, indent=indent + 1))
|
|
693
689
|
lines.append("%s- version = %s"
|
|
694
690
|
% (" " * (indent + 1), misc.get_version_string(self)))
|
|
695
691
|
lines.append("%s- results = %s"
|
|
696
692
|
% (" " * (indent + 1), self.results))
|
|
697
|
-
lines.append("
|
|
698
|
-
|
|
693
|
+
lines.append("{}- failure = {}".format(" " * (indent + 1),
|
|
694
|
+
bool(self.failure)))
|
|
699
695
|
lines.extend(_format_meta(self.meta, indent=indent + 1))
|
|
700
696
|
return linesep.join(lines)
|
|
701
697
|
|
|
@@ -793,7 +789,7 @@ class TaskDetail(AtomDetail):
|
|
|
793
789
|
" task details")
|
|
794
790
|
if other is self:
|
|
795
791
|
return self
|
|
796
|
-
super(
|
|
792
|
+
super().merge(other, deep_copy=deep_copy)
|
|
797
793
|
self.results = other.results
|
|
798
794
|
self.revert_results = other.revert_results
|
|
799
795
|
return self
|
|
@@ -834,7 +830,7 @@ class RetryDetail(AtomDetail):
|
|
|
834
830
|
"""
|
|
835
831
|
|
|
836
832
|
def __init__(self, name, uuid):
|
|
837
|
-
super(
|
|
833
|
+
super().__init__(name, uuid)
|
|
838
834
|
self.results = []
|
|
839
835
|
|
|
840
836
|
def reset(self, state):
|
|
@@ -983,7 +979,7 @@ class RetryDetail(AtomDetail):
|
|
|
983
979
|
new_results.append((data, new_failures))
|
|
984
980
|
return new_results
|
|
985
981
|
|
|
986
|
-
obj = super(
|
|
982
|
+
obj = super().from_dict(data)
|
|
987
983
|
obj.results = decode_results(obj.results)
|
|
988
984
|
return obj
|
|
989
985
|
|
|
@@ -1001,7 +997,7 @@ class RetryDetail(AtomDetail):
|
|
|
1001
997
|
new_results.append((data, new_failures))
|
|
1002
998
|
return new_results
|
|
1003
999
|
|
|
1004
|
-
base = super(
|
|
1000
|
+
base = super().to_dict()
|
|
1005
1001
|
base['results'] = encode_results(base.get('results'))
|
|
1006
1002
|
return base
|
|
1007
1003
|
|
|
@@ -1033,7 +1029,7 @@ class RetryDetail(AtomDetail):
|
|
|
1033
1029
|
" retry details")
|
|
1034
1030
|
if other is self:
|
|
1035
1031
|
return self
|
|
1036
|
-
super(
|
|
1032
|
+
super().merge(other, deep_copy=deep_copy)
|
|
1037
1033
|
results = []
|
|
1038
1034
|
# NOTE(imelnikov): we can't just deep copy Failures, as they
|
|
1039
1035
|
# contain tracebacks, which are not copyable.
|
|
@@ -1053,8 +1049,7 @@ _DETAIL_TO_NAME = {
|
|
|
1053
1049
|
RetryDetail: 'RETRY_DETAIL',
|
|
1054
1050
|
TaskDetail: 'TASK_DETAIL',
|
|
1055
1051
|
}
|
|
1056
|
-
_NAME_TO_DETAIL =
|
|
1057
|
-
for (cls, name) in _DETAIL_TO_NAME.items())
|
|
1052
|
+
_NAME_TO_DETAIL = {name: cls for (cls, name) in _DETAIL_TO_NAME.items()}
|
|
1058
1053
|
ATOM_TYPES = list(_NAME_TO_DETAIL.keys())
|
|
1059
1054
|
|
|
1060
1055
|
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2015 Rackspace Hosting All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -36,7 +34,7 @@ class PathBasedBackend(base.Backend, metaclass=abc.ABCMeta):
|
|
|
36
34
|
DEFAULT_PATH = None
|
|
37
35
|
|
|
38
36
|
def __init__(self, conf):
|
|
39
|
-
super(
|
|
37
|
+
super().__init__(conf)
|
|
40
38
|
self._path = self._conf.get('path', None)
|
|
41
39
|
if not self._path:
|
|
42
40
|
self._path = self.DEFAULT_PATH
|
taskflow/retry.py
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2013 Rackspace Hosting Inc. All Rights Reserved.
|
|
4
2
|
# Copyright (C) 2013 Yahoo! Inc. All Rights Reserved.
|
|
5
3
|
#
|
|
@@ -71,7 +69,7 @@ EXECUTE_REVERT_HISTORY = 'history'
|
|
|
71
69
|
REVERT_FLOW_FAILURES = 'flow_failures'
|
|
72
70
|
|
|
73
71
|
|
|
74
|
-
class History
|
|
72
|
+
class History:
|
|
75
73
|
"""Helper that simplifies interactions with retry historical contents."""
|
|
76
74
|
|
|
77
75
|
def __init__(self, contents, failure=None):
|
|
@@ -99,8 +97,7 @@ class History(object):
|
|
|
99
97
|
self._contents[index],
|
|
100
98
|
]
|
|
101
99
|
for (provided, outcomes) in contents:
|
|
102
|
-
|
|
103
|
-
yield (owner, outcome)
|
|
100
|
+
yield from outcomes.items()
|
|
104
101
|
|
|
105
102
|
def __len__(self):
|
|
106
103
|
return len(self._contents)
|
|
@@ -154,10 +151,10 @@ class Retry(atom.Atom, metaclass=abc.ABCMeta):
|
|
|
154
151
|
|
|
155
152
|
def __init__(self, name=None, provides=None, requires=None,
|
|
156
153
|
auto_extract=True, rebind=None):
|
|
157
|
-
super(
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
154
|
+
super().__init__(name=name, provides=provides,
|
|
155
|
+
requires=requires, rebind=rebind,
|
|
156
|
+
auto_extract=auto_extract,
|
|
157
|
+
ignore_list=[EXECUTE_REVERT_HISTORY])
|
|
161
158
|
|
|
162
159
|
@property
|
|
163
160
|
def name(self):
|
|
@@ -262,8 +259,7 @@ class Times(Retry):
|
|
|
262
259
|
|
|
263
260
|
def __init__(self, attempts=1, name=None, provides=None, requires=None,
|
|
264
261
|
auto_extract=True, rebind=None, revert_all=False):
|
|
265
|
-
super(
|
|
266
|
-
auto_extract, rebind)
|
|
262
|
+
super().__init__(name, provides, requires, auto_extract, rebind)
|
|
267
263
|
self._attempts = attempts
|
|
268
264
|
|
|
269
265
|
if revert_all:
|
|
@@ -285,8 +281,7 @@ class ForEachBase(Retry):
|
|
|
285
281
|
|
|
286
282
|
def __init__(self, name=None, provides=None, requires=None,
|
|
287
283
|
auto_extract=True, rebind=None, revert_all=False):
|
|
288
|
-
super(
|
|
289
|
-
auto_extract, rebind)
|
|
284
|
+
super().__init__(name, provides, requires, auto_extract, rebind)
|
|
290
285
|
|
|
291
286
|
if revert_all:
|
|
292
287
|
self._revert_action = REVERT_ALL
|
|
@@ -336,8 +331,8 @@ class ForEach(ForEachBase):
|
|
|
336
331
|
|
|
337
332
|
def __init__(self, values, name=None, provides=None, requires=None,
|
|
338
333
|
auto_extract=True, rebind=None, revert_all=False):
|
|
339
|
-
super(
|
|
340
|
-
|
|
334
|
+
super().__init__(name, provides, requires, auto_extract, rebind,
|
|
335
|
+
revert_all)
|
|
341
336
|
self._values = values
|
|
342
337
|
|
|
343
338
|
def on_failure(self, history, *args, **kwargs):
|
|
@@ -368,9 +363,8 @@ class ParameterizedForEach(ForEachBase):
|
|
|
368
363
|
|
|
369
364
|
def __init__(self, name=None, provides=None, requires=None,
|
|
370
365
|
auto_extract=True, rebind=None, revert_all=False):
|
|
371
|
-
super(
|
|
372
|
-
|
|
373
|
-
revert_all)
|
|
366
|
+
super().__init__(name, provides, requires, auto_extract, rebind,
|
|
367
|
+
revert_all)
|
|
374
368
|
|
|
375
369
|
def on_failure(self, values, history, *args, **kwargs):
|
|
376
370
|
return self._on_failure(values, history)
|
taskflow/states.py
CHANGED
taskflow/storage.py
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2013 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -87,7 +85,7 @@ META_PROGRESS = 'progress'
|
|
|
87
85
|
META_PROGRESS_DETAILS = 'progress_details'
|
|
88
86
|
|
|
89
87
|
|
|
90
|
-
class _ProviderLocator
|
|
88
|
+
class _ProviderLocator:
|
|
91
89
|
"""Helper to start to better decouple the finding logic from storage.
|
|
92
90
|
|
|
93
91
|
WIP: part of the larger effort to cleanup/refactor the finding of named
|
|
@@ -149,7 +147,7 @@ class _ProviderLocator(object):
|
|
|
149
147
|
return (searched_providers, providers_and_results)
|
|
150
148
|
if not atom_providers:
|
|
151
149
|
return (searched_providers, providers_and_results)
|
|
152
|
-
atom_providers_by_name =
|
|
150
|
+
atom_providers_by_name = {p.name: p for p in atom_providers}
|
|
153
151
|
for accessible_atom_names in iter(scope_walker):
|
|
154
152
|
# *Always* retain the scope ordering (if any matches
|
|
155
153
|
# happen); instead of retaining the possible provider match
|
|
@@ -199,7 +197,7 @@ class _ProviderLocator(object):
|
|
|
199
197
|
_searched_providers, providers_and_results = self._find(
|
|
200
198
|
looking_for, scope_walker=scope_walker,
|
|
201
199
|
short_circuit=False, find_potentials=True)
|
|
202
|
-
return
|
|
200
|
+
return {p for (p, _provider_results) in providers_and_results}
|
|
203
201
|
|
|
204
202
|
def find(self, looking_for, scope_walker=None, short_circuit=True):
|
|
205
203
|
"""Returns the accessible providers."""
|
|
@@ -208,7 +206,7 @@ class _ProviderLocator(object):
|
|
|
208
206
|
find_potentials=False)
|
|
209
207
|
|
|
210
208
|
|
|
211
|
-
class _Provider
|
|
209
|
+
class _Provider:
|
|
212
210
|
"""A named symbol provider that produces a output at the given index."""
|
|
213
211
|
|
|
214
212
|
def __init__(self, name, index):
|
|
@@ -270,7 +268,7 @@ def _item_from_first_of(providers, looking_for):
|
|
|
270
268
|
" extraction" % (looking_for, providers))
|
|
271
269
|
|
|
272
270
|
|
|
273
|
-
class Storage
|
|
271
|
+
class Storage:
|
|
274
272
|
"""Interface between engines and logbook and its backend (if any).
|
|
275
273
|
|
|
276
274
|
This class provides a simple interface to save atoms of a given flow and
|
|
@@ -326,8 +324,8 @@ class Storage(object):
|
|
|
326
324
|
fail_cache[states.REVERT] = ad.revert_failure
|
|
327
325
|
self._failures[ad.name] = fail_cache
|
|
328
326
|
|
|
329
|
-
self._atom_name_to_uuid =
|
|
330
|
-
|
|
327
|
+
self._atom_name_to_uuid = {ad.name: ad.uuid
|
|
328
|
+
for ad in self._flowdetail}
|
|
331
329
|
try:
|
|
332
330
|
source, _clone = self._atomdetail_by_name(
|
|
333
331
|
self.injector_name, expected_type=models.TaskDetail)
|
|
@@ -336,7 +334,7 @@ class Storage(object):
|
|
|
336
334
|
else:
|
|
337
335
|
names_iter = source.results.keys()
|
|
338
336
|
self._set_result_mapping(source.name,
|
|
339
|
-
|
|
337
|
+
{name: name for name in names_iter})
|
|
340
338
|
|
|
341
339
|
def _with_connection(self, functor, *args, **kwargs):
|
|
342
340
|
# Run the given functor with a backend connection as its first
|
|
@@ -911,7 +909,7 @@ class Storage(object):
|
|
|
911
909
|
provider_name, names = save_persistent()
|
|
912
910
|
|
|
913
911
|
self._set_result_mapping(provider_name,
|
|
914
|
-
|
|
912
|
+
{name: name for name in names})
|
|
915
913
|
|
|
916
914
|
def _fetch_providers(self, looking_for, providers=None):
|
|
917
915
|
"""Return pair of (default providers, atom providers)."""
|
taskflow/task.py
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright 2015 Hewlett-Packard Development Company, L.P.
|
|
4
2
|
# Copyright (C) 2013 Rackspace Hosting Inc. All Rights Reserved.
|
|
5
3
|
# Copyright (C) 2013 Yahoo! Inc. All Rights Reserved.
|
|
@@ -61,10 +59,10 @@ class Task(atom.Atom, metaclass=abc.ABCMeta):
|
|
|
61
59
|
ignore_list=None, revert_rebind=None, revert_requires=None):
|
|
62
60
|
if name is None:
|
|
63
61
|
name = reflection.get_class_name(self)
|
|
64
|
-
super(
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
62
|
+
super().__init__(name, provides=provides, requires=requires,
|
|
63
|
+
auto_extract=auto_extract, rebind=rebind,
|
|
64
|
+
inject=inject, revert_rebind=revert_rebind,
|
|
65
|
+
revert_requires=revert_requires)
|
|
68
66
|
self._notifier = notifier.RestrictedNotifier(self.TASK_EVENTS)
|
|
69
67
|
|
|
70
68
|
@property
|
|
@@ -131,8 +129,7 @@ class FunctorTask(Task):
|
|
|
131
129
|
" be callable")
|
|
132
130
|
if name is None:
|
|
133
131
|
name = reflection.get_callable_name(execute)
|
|
134
|
-
super(
|
|
135
|
-
inject=inject)
|
|
132
|
+
super().__init__(name, provides=provides, inject=inject)
|
|
136
133
|
self._execute = execute
|
|
137
134
|
self._revert = revert
|
|
138
135
|
if version is not None:
|
|
@@ -190,12 +187,12 @@ class ReduceFunctorTask(Task):
|
|
|
190
187
|
|
|
191
188
|
if name is None:
|
|
192
189
|
name = reflection.get_callable_name(functor)
|
|
193
|
-
super(
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
190
|
+
super().__init__(name=name,
|
|
191
|
+
provides=provides,
|
|
192
|
+
inject=inject,
|
|
193
|
+
requires=requires,
|
|
194
|
+
rebind=rebind,
|
|
195
|
+
auto_extract=auto_extract)
|
|
199
196
|
|
|
200
197
|
self._functor = functor
|
|
201
198
|
|
|
@@ -235,10 +232,10 @@ class MapFunctorTask(Task):
|
|
|
235
232
|
|
|
236
233
|
if name is None:
|
|
237
234
|
name = reflection.get_callable_name(functor)
|
|
238
|
-
super(
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
235
|
+
super().__init__(name=name, provides=provides,
|
|
236
|
+
inject=inject, requires=requires,
|
|
237
|
+
rebind=rebind,
|
|
238
|
+
auto_extract=auto_extract)
|
|
242
239
|
|
|
243
240
|
self._functor = functor
|
|
244
241
|
|
taskflow/test.py
CHANGED
|
@@ -27,7 +27,7 @@ from taskflow.tests import utils
|
|
|
27
27
|
from taskflow.utils import misc
|
|
28
28
|
|
|
29
29
|
|
|
30
|
-
class GreaterThanEqual
|
|
30
|
+
class GreaterThanEqual:
|
|
31
31
|
"""Matches if the item is geq than the matchers reference object."""
|
|
32
32
|
|
|
33
33
|
def __init__(self, source):
|
|
@@ -36,10 +36,10 @@ class GreaterThanEqual(object):
|
|
|
36
36
|
def match(self, other):
|
|
37
37
|
if other >= self.source:
|
|
38
38
|
return None
|
|
39
|
-
return matchers.Mismatch("
|
|
39
|
+
return matchers.Mismatch("{} was not >= {}".format(other, self.source))
|
|
40
40
|
|
|
41
41
|
|
|
42
|
-
class FailureRegexpMatcher
|
|
42
|
+
class FailureRegexpMatcher:
|
|
43
43
|
"""Matches if the failure was caused by the given exception and message.
|
|
44
44
|
|
|
45
45
|
This will match if a given failure contains and exception of the given
|
|
@@ -60,7 +60,7 @@ class FailureRegexpMatcher(object):
|
|
|
60
60
|
(failure, self.exc_class))
|
|
61
61
|
|
|
62
62
|
|
|
63
|
-
class ItemsEqual
|
|
63
|
+
class ItemsEqual:
|
|
64
64
|
"""Matches the items in two sequences.
|
|
65
65
|
|
|
66
66
|
This matcher will validate that the provided sequence has the same elements
|
|
@@ -166,7 +166,7 @@ class TestCase(base.BaseTestCase):
|
|
|
166
166
|
class MockTestCase(TestCase):
|
|
167
167
|
|
|
168
168
|
def setUp(self):
|
|
169
|
-
super(
|
|
169
|
+
super().setUp()
|
|
170
170
|
self.master_mock = mock.Mock(name='master_mock')
|
|
171
171
|
|
|
172
172
|
def patch(self, target, autospec=True, **kwargs):
|
taskflow/tests/test_examples.py
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -298,7 +296,7 @@ class BuildersTest(test.TestCase):
|
|
|
298
296
|
flow, initial_state=st.RUNNING)
|
|
299
297
|
transitions = list(machine_runner.run_iter(builder.START))
|
|
300
298
|
|
|
301
|
-
occurrences =
|
|
299
|
+
occurrences = {t: transitions.count(t) for t in transitions}
|
|
302
300
|
self.assertEqual(10, occurrences.get((st.SCHEDULING, st.WAITING)))
|
|
303
301
|
self.assertEqual(10, occurrences.get((st.WAITING, st.ANALYZING)))
|
|
304
302
|
self.assertEqual(9, occurrences.get((st.ANALYZING, st.SCHEDULING)))
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -110,7 +108,7 @@ class PatternCompileTest(test.TestCase):
|
|
|
110
108
|
('c', 'test[$]'),
|
|
111
109
|
('d', 'test[$]'),
|
|
112
110
|
])
|
|
113
|
-
self.assertEqual(
|
|
111
|
+
self.assertEqual({'test'}, set(g.no_predecessors_iter()))
|
|
114
112
|
|
|
115
113
|
def test_linear_nested(self):
|
|
116
114
|
a, b, c, d = test_utils.make_many(4)
|
|
@@ -283,7 +281,7 @@ class PatternCompileTest(test.TestCase):
|
|
|
283
281
|
self.assertEqual(4, len(g))
|
|
284
282
|
self.assertCountEqual(g.edges(data=True), [
|
|
285
283
|
('test', 'a', {'invariant': True}),
|
|
286
|
-
('a', 'b', {'reasons':
|
|
284
|
+
('a', 'b', {'reasons': {'x'}}),
|
|
287
285
|
('b', 'test[$]', {'invariant': True}),
|
|
288
286
|
])
|
|
289
287
|
self.assertCountEqual(['test'], g.no_predecessors_iter())
|
|
@@ -302,7 +300,7 @@ class PatternCompileTest(test.TestCase):
|
|
|
302
300
|
self.assertCountEqual(g.edges(data=True), [
|
|
303
301
|
('test', 'a', {'invariant': True}),
|
|
304
302
|
('test2', 'b', {'invariant': True}),
|
|
305
|
-
('a', 'test2', {'reasons':
|
|
303
|
+
('a', 'test2', {'reasons': {'x'}}),
|
|
306
304
|
('b', 'c', {'invariant': True}),
|
|
307
305
|
('c', 'test2[$]', {'invariant': True}),
|
|
308
306
|
('test2[$]', 'test[$]', {'invariant': True}),
|
|
@@ -325,7 +323,7 @@ class PatternCompileTest(test.TestCase):
|
|
|
325
323
|
('a', 'test[$]', {'invariant': True}),
|
|
326
324
|
|
|
327
325
|
# The 'x' requirement is produced out of test2...
|
|
328
|
-
('test2[$]', 'a', {'reasons':
|
|
326
|
+
('test2[$]', 'a', {'reasons': {'x'}}),
|
|
329
327
|
|
|
330
328
|
('test2', 'b', {'invariant': True}),
|
|
331
329
|
('b', 'c', {'invariant': True}),
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -26,11 +24,6 @@ from taskflow.tests import utils
|
|
|
26
24
|
from taskflow.utils import eventlet_utils as eu
|
|
27
25
|
from taskflow.utils import persistence_utils as pu
|
|
28
26
|
|
|
29
|
-
try:
|
|
30
|
-
from taskflow.engines.action_engine import process_executor as pe
|
|
31
|
-
except ImportError:
|
|
32
|
-
pe = None
|
|
33
|
-
|
|
34
27
|
|
|
35
28
|
class ParallelCreationTest(test.TestCase):
|
|
36
29
|
@staticmethod
|
|
@@ -48,26 +41,12 @@ class ParallelCreationTest(test.TestCase):
|
|
|
48
41
|
self.assertIsInstance(eng._task_executor,
|
|
49
42
|
executor.ParallelThreadTaskExecutor)
|
|
50
43
|
|
|
51
|
-
@testtools.skipIf(pe is None, 'process_executor is not available')
|
|
52
|
-
def test_process_string_creation(self):
|
|
53
|
-
for s in ['process', 'processes']:
|
|
54
|
-
eng = self._create_engine(executor=s)
|
|
55
|
-
self.assertIsInstance(eng._task_executor,
|
|
56
|
-
pe.ParallelProcessTaskExecutor)
|
|
57
|
-
|
|
58
44
|
def test_thread_executor_creation(self):
|
|
59
45
|
with futurist.ThreadPoolExecutor(1) as e:
|
|
60
46
|
eng = self._create_engine(executor=e)
|
|
61
47
|
self.assertIsInstance(eng._task_executor,
|
|
62
48
|
executor.ParallelThreadTaskExecutor)
|
|
63
49
|
|
|
64
|
-
@testtools.skipIf(pe is None, 'process_executor is not available')
|
|
65
|
-
def test_process_executor_creation(self):
|
|
66
|
-
with futurist.ProcessPoolExecutor(1) as e:
|
|
67
|
-
eng = self._create_engine(executor=e)
|
|
68
|
-
self.assertIsInstance(eng._task_executor,
|
|
69
|
-
pe.ParallelProcessTaskExecutor)
|
|
70
|
-
|
|
71
50
|
@testtools.skipIf(not eu.EVENTLET_AVAILABLE, 'eventlet is not available')
|
|
72
51
|
def test_green_executor_creation(self):
|
|
73
52
|
with futurist.GreenThreadPoolExecutor(1) as e:
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -272,7 +270,7 @@ class MixedPatternScopingTest(test.TestCase):
|
|
|
272
270
|
# This may be different after/if the following is resolved:
|
|
273
271
|
#
|
|
274
272
|
# https://github.com/networkx/networkx/issues/1181 (and a few others)
|
|
275
|
-
self.assertEqual(
|
|
273
|
+
self.assertEqual({'customer', 'customer2'},
|
|
276
274
|
set(_get_scopes(c, washer)[0]))
|
|
277
275
|
self.assertEqual([], _get_scopes(c, customer2))
|
|
278
276
|
self.assertEqual([], _get_scopes(c, customer))
|
taskflow/tests/unit/jobs/base.py
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -37,7 +35,7 @@ def connect_close(*args):
|
|
|
37
35
|
a.close()
|
|
38
36
|
|
|
39
37
|
|
|
40
|
-
class BoardTestMixin
|
|
38
|
+
class BoardTestMixin:
|
|
41
39
|
|
|
42
40
|
@contextlib.contextmanager
|
|
43
41
|
def flush(self, client):
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2013 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -101,7 +99,7 @@ class RedisJobboardTest(test.TestCase, base.BoardTestMixin):
|
|
|
101
99
|
self.assertEqual(0, len(possible_jobs))
|
|
102
100
|
|
|
103
101
|
def setUp(self):
|
|
104
|
-
super(
|
|
102
|
+
super().setUp()
|
|
105
103
|
self.client, self.board = self.create_board()
|
|
106
104
|
|
|
107
105
|
def test__make_client(self):
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
|
|
3
1
|
# Copyright (C) 2013 Yahoo! Inc. All Rights Reserved.
|
|
4
2
|
#
|
|
5
3
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
@@ -137,7 +135,7 @@ class ZookeeperJobboardTest(test.TestCase, ZookeeperBoardTestMixin):
|
|
|
137
135
|
return (client, board)
|
|
138
136
|
|
|
139
137
|
def setUp(self):
|
|
140
|
-
super(
|
|
138
|
+
super().setUp()
|
|
141
139
|
self.client, self.board = self.create_board()
|
|
142
140
|
|
|
143
141
|
|
|
@@ -152,7 +150,7 @@ class ZakeJobboardTest(test.TestCase, ZookeeperBoardTestMixin):
|
|
|
152
150
|
return (client, board)
|
|
153
151
|
|
|
154
152
|
def setUp(self):
|
|
155
|
-
super(
|
|
153
|
+
super().setUp()
|
|
156
154
|
self.client, self.board = self.create_board()
|
|
157
155
|
self.bad_paths = [self.board.path, self.board.trash_path]
|
|
158
156
|
self.bad_paths.extend(zake_utils.partition_path(self.board.path))
|