stepfunction 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stepfunction/constants/__init__.py +0 -0
- stepfunction/constants/enums.py +8 -0
- stepfunction/constants/visualizer.py +47 -0
- stepfunction/core/step_function/__init__.py +3 -0
- stepfunction/core/step_function/step_function.py +368 -0
- stepfunction/core/visualizer/__init__.py +3 -0
- stepfunction/core/visualizer/visualizer.py +129 -0
- stepfunction/exceptions/__init__.py +0 -0
- stepfunction/exceptions/step_errors.py +14 -0
- stepfunction/types/__init__.py +0 -0
- stepfunction/types/step_types.py +12 -0
- stepfunction/types/visualizer_types.py +7 -0
- stepfunction/utils/__init__.py +0 -0
- stepfunction/utils/constants.py +12 -0
- stepfunction/utils/logger.py +61 -0
- stepfunction/utils/utils.py +17 -0
- stepfunction-0.0.1.dist-info/LICENSE +21 -0
- stepfunction-0.0.1.dist-info/METADATA +67 -0
- stepfunction-0.0.1.dist-info/RECORD +21 -0
- stepfunction-0.0.1.dist-info/WHEEL +5 -0
- stepfunction-0.0.1.dist-info/top_level.txt +1 -0
|
File without changes
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Visualizer
|
|
2
|
+
|
|
3
|
+
# Render Configuration
|
|
4
|
+
|
|
5
|
+
DEFAULT_VISUALIZER_EXTENSION = "gv"
|
|
6
|
+
"""str: The default extension for the visualizer renderer."""
|
|
7
|
+
|
|
8
|
+
DEFAULT_VISUALIZER_FORMAT = "png"
|
|
9
|
+
"""str: The default format for the visualizer renderer."""
|
|
10
|
+
|
|
11
|
+
DEFAULT_VISUALIZER_RENDERER = "cairo"
|
|
12
|
+
"""str: The default renderer for the visualizer."""
|
|
13
|
+
|
|
14
|
+
DEFAULT_VISUALIZER_FOLDER = "workflow_renders"
|
|
15
|
+
"""str: The default folder where visualizer renders are stored."""
|
|
16
|
+
|
|
17
|
+
DEFAULT_VISUALIZER_STRING_ENCODING = "utf-8"
|
|
18
|
+
"""str: The default encoding for visualizer strings."""
|
|
19
|
+
|
|
20
|
+
# Edge labels and colors
|
|
21
|
+
|
|
22
|
+
DEFAULT_VISUALIZER_SUCCESS_EDGE_LABEL = "Success"
|
|
23
|
+
"""str: The default edge label for success transitions in the visualizer."""
|
|
24
|
+
|
|
25
|
+
DEFAULT_VISUALIZER_SUCCESS_EDGE_COLOR = "black"
|
|
26
|
+
"""str: The default edge color for success transitions in the visualizer."""
|
|
27
|
+
|
|
28
|
+
DEFAULT_VISUALIZER_FAILURE_EDGE_LABEL = "Failure"
|
|
29
|
+
"""str: The default edge label for failure transitions in the visualizer."""
|
|
30
|
+
|
|
31
|
+
DEFAULT_VISUALIZER_FAILURE_EDGE_COLOR = "black"
|
|
32
|
+
"""str: The default edge color for failure transitions in the visualizer."""
|
|
33
|
+
|
|
34
|
+
DEFAULT_VISUALIZER_STOP_ON_FAILURE_EDGE_LABEL = "Stop on Failure"
|
|
35
|
+
"""str: The default edge label for stop on failure transitions in the visualizer."""
|
|
36
|
+
|
|
37
|
+
DEFAULT_VISUALIZER_STOP_ON_FAILURE_EDGE_COLOR = "red"
|
|
38
|
+
"""str: The default edge color for stop on failure transitions in the visualizer."""
|
|
39
|
+
|
|
40
|
+
DEFAULT_VISUALIZER_PARALLEL_STEP_EDGE_STYLE = "dashed"
|
|
41
|
+
"""str: The default edge style for parallel steps in the visualizer."""
|
|
42
|
+
|
|
43
|
+
DEFAULT_VISUALIZER_SUB_STEP_FUNCTION_NODE_SHAPE = "boxed"
|
|
44
|
+
"""str: The default node shape for sub-step functions in the visualizer."""
|
|
45
|
+
|
|
46
|
+
DEFAULT_VISUALIZER_SUB_STEP_FUNCTION_NODE_STYLE = "dotted"
|
|
47
|
+
""" str: The default node style for sub-step functions in the visualizer."""
|
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
""" Module to define the StepFunction class.
|
|
2
|
+
|
|
3
|
+
Author: Vineeth Penugonda
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
8
|
+
from inspect import iscoroutinefunction
|
|
9
|
+
from typing import Any, Callable, Dict, Optional, Union
|
|
10
|
+
|
|
11
|
+
from stepfunction.constants.enums import StepFunctionStatus
|
|
12
|
+
from stepfunction.core.visualizer import Visualizer
|
|
13
|
+
from stepfunction.exceptions.step_errors import (ParallelStepExecutionError,
|
|
14
|
+
StepExecutionError)
|
|
15
|
+
from stepfunction.types.step_types import StepParams
|
|
16
|
+
from stepfunction.utils.logger import setup_logger
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class StepFunction:
|
|
20
|
+
"""
|
|
21
|
+
Class to represent a workflow consisting of multiple steps.
|
|
22
|
+
|
|
23
|
+
This class allows for sequential and parallel execution of steps in a workflow. Each step can have a next step to
|
|
24
|
+
proceed to upon success, an on_failure step to proceed to in case of failure, and optional branching based on the
|
|
25
|
+
result of a step. Parallel steps can be executed concurrently, with the option to stop all parallel executions
|
|
26
|
+
if one of them fails. The results of each step, including failures, are stored in the workflow's context, which can
|
|
27
|
+
be visualized or retrieved for further analysis.
|
|
28
|
+
|
|
29
|
+
Attributes:
|
|
30
|
+
current_step (Optional[str]): The current step being executed.
|
|
31
|
+
context (Dict[str, Any]): A dictionary that holds the results of each step, including exceptions if any occur.
|
|
32
|
+
|
|
33
|
+
Properties:
|
|
34
|
+
name (str): The name of the step function.
|
|
35
|
+
steps (StepParams): A dictionary containing the steps of the workflow.
|
|
36
|
+
last_result (Any): The result of the last step.
|
|
37
|
+
context (Dict[str, Any]): Stores step names and results, including exceptions if any occur.
|
|
38
|
+
status (StepFunctionStatus): The current status of the workflow. Possible values are:
|
|
39
|
+
- StepFunctionStatus.INITIALIZED: The workflow is initialized but not yet started.
|
|
40
|
+
- StepFunctionStatus.RUNNING: The workflow is currently in progress.
|
|
41
|
+
- StepFunctionStatus.COMPLETED: The workflow finished successfully.
|
|
42
|
+
- StepFunctionStatus.FAILED: The workflow encountered an error and went to a failure handler or failed entirely.
|
|
43
|
+
|
|
44
|
+
Methods:
|
|
45
|
+
add_step(name, func, next_step=None, on_failure=None, branch=None, parallel=False, stop_on_failure=False):
|
|
46
|
+
Add a step to the workflow.
|
|
47
|
+
|
|
48
|
+
set_start_step(name):
|
|
49
|
+
Set the start step of the workflow.
|
|
50
|
+
|
|
51
|
+
add_sub_step_function(name, sub_step_function, next_step=None, on_failure=None):
|
|
52
|
+
Add a sub-step function to be executed as a step.
|
|
53
|
+
|
|
54
|
+
execute(initial_input=None):
|
|
55
|
+
Execute the workflow starting from the specified start step. This will run each step in sequence or in parallel,
|
|
56
|
+
depending on the step configuration.
|
|
57
|
+
|
|
58
|
+
visualize():
|
|
59
|
+
Visualize the workflow structure and the relationships between steps.
|
|
60
|
+
|
|
61
|
+
visualize_to_string():
|
|
62
|
+
Return a string representation of the workflow for visualization.
|
|
63
|
+
|
|
64
|
+
Protected Methods:
|
|
65
|
+
_execute_step(func, input_value):
|
|
66
|
+
Execute a single step, handling asynchronous and synchronous functions appropriately.
|
|
67
|
+
|
|
68
|
+
_execute_parallel(func_dict, stop_on_failure=False):
|
|
69
|
+
Execute parallel steps concurrently. Store the results for each step in the context, including exceptions
|
|
70
|
+
if any step fails. If `stop_on_failure` is True, halt all parallel executions if one of the steps fails.
|
|
71
|
+
|
|
72
|
+
Raises:
|
|
73
|
+
ValueError:
|
|
74
|
+
Raised if a step with the same name already exists in the workflow or if an undefined step is set as the start step.
|
|
75
|
+
|
|
76
|
+
StepExecutionError:
|
|
77
|
+
Raised when a step fails and no `on_failure` step is defined.
|
|
78
|
+
|
|
79
|
+
ParallelStepExecutionError:
|
|
80
|
+
Raised when one or more parallel steps fail and `stop_on_failure` is set to True.
|
|
81
|
+
|
|
82
|
+
Example usage:
|
|
83
|
+
step_function = StepFunction("MyStepFunction")
|
|
84
|
+
step_function.add_step("Step1", func1, next_step="Step2")
|
|
85
|
+
step_function.add_step("Step2", func2, branch={"success": "Step3", "failure": "Step4"})
|
|
86
|
+
step_function.add_step("Step3", func3, on_failure="Step1")
|
|
87
|
+
step_function.add_step("Step4", func4)
|
|
88
|
+
step_function.set_start_step("Step1")
|
|
89
|
+
await step_function.execute()
|
|
90
|
+
|
|
91
|
+
Parallel Example:
|
|
92
|
+
step_function.add_step("Step1", func1, parallel=True)
|
|
93
|
+
step_function.add_step("ParallelStep", {
|
|
94
|
+
"task1": func2,
|
|
95
|
+
"task2": func3
|
|
96
|
+
}, next_step="Step2", parallel=True)
|
|
97
|
+
|
|
98
|
+
Sub-step function example:
|
|
99
|
+
sub_step_function = StepFunction("SubStepFunction")
|
|
100
|
+
# Define steps for sub_step_function...
|
|
101
|
+
step_function.add_sub_step_function("SubStep", sub_step_function, next_step="FinalStep")
|
|
102
|
+
await step_function.execute()
|
|
103
|
+
|
|
104
|
+
Status Example:
|
|
105
|
+
status = step_function.status # Will be StepFunctionStatus.INITIALIZED, StepFunctionStatus.RUNNING, StepFunctionStatus.COMPLETED, or StepFunctionStatus.FAILED.
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
def __init__(self, name: str):
|
|
109
|
+
self.__name = name # Name of the step function
|
|
110
|
+
|
|
111
|
+
self.__steps: StepParams = {} # Steps of the workflow
|
|
112
|
+
self.__current_step = None # The current step being executed
|
|
113
|
+
|
|
114
|
+
self.__last_result = None # To hold the result of the last step
|
|
115
|
+
self.__context = {} # To hold the step names and results of those steps
|
|
116
|
+
|
|
117
|
+
# Status of the step function
|
|
118
|
+
self.__status: StepFunctionStatus = StepFunctionStatus.INITIALIZED
|
|
119
|
+
|
|
120
|
+
self.__logger = setup_logger(__name__)
|
|
121
|
+
|
|
122
|
+
self.__logger.debug(
|
|
123
|
+
f"StepFunction - {self.__name} - Status - {self.__status.value}")
|
|
124
|
+
|
|
125
|
+
def add_step(
|
|
126
|
+
self,
|
|
127
|
+
name: str,
|
|
128
|
+
func: Union[Callable[[Any], Any], Dict[str, Callable[[Any], Any]]],
|
|
129
|
+
next_step: Optional[str] = None,
|
|
130
|
+
on_failure: Optional[str] = None,
|
|
131
|
+
branch: Optional[Dict[Any, str]] = None,
|
|
132
|
+
parallel: bool = False,
|
|
133
|
+
stop_on_failure: bool = False,
|
|
134
|
+
):
|
|
135
|
+
"""Add a step to the workflow."""
|
|
136
|
+
|
|
137
|
+
if name in self.__steps:
|
|
138
|
+
raise ValueError(f"Step '{name}' already exists in steps")
|
|
139
|
+
|
|
140
|
+
self.__steps[name] = {
|
|
141
|
+
"func": func,
|
|
142
|
+
"next_step": next_step,
|
|
143
|
+
"on_failure": on_failure,
|
|
144
|
+
"branch": branch,
|
|
145
|
+
"parallel": parallel,
|
|
146
|
+
"stop_on_failure": stop_on_failure,
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
def add_sub_step_function(self, name: str, sub_step_function: "StepFunction", next_step: Optional[str] = None, on_failure: Optional[str] = None):
|
|
150
|
+
"""Add a sub-step function to the workflow."""
|
|
151
|
+
|
|
152
|
+
if name in self.__steps:
|
|
153
|
+
raise ValueError(f"Step '{name}' already exists in steps")
|
|
154
|
+
|
|
155
|
+
async def sub_func(last_result):
|
|
156
|
+
await sub_step_function.execute(initial_input=last_result)
|
|
157
|
+
self.__context.update(sub_step_function.context)
|
|
158
|
+
|
|
159
|
+
if sub_step_function.status == StepFunctionStatus.FAILED:
|
|
160
|
+
self.__status = StepFunctionStatus.FAILED
|
|
161
|
+
|
|
162
|
+
return sub_step_function.last_result
|
|
163
|
+
|
|
164
|
+
self.__steps[name] = {
|
|
165
|
+
"func": sub_func,
|
|
166
|
+
"next_step": next_step,
|
|
167
|
+
"on_failure": on_failure,
|
|
168
|
+
"branch": None,
|
|
169
|
+
"parallel": False,
|
|
170
|
+
"stop_on_failure": False,
|
|
171
|
+
"is_sub_step_function": True
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
def set_start_step(self, name: str):
|
|
175
|
+
"""Set the start step of the workflow."""
|
|
176
|
+
if name not in self.__steps:
|
|
177
|
+
raise ValueError(f"Step '{name}' not found in steps")
|
|
178
|
+
|
|
179
|
+
self.__current_step = name
|
|
180
|
+
|
|
181
|
+
async def execute(self, initial_input: Any = None):
|
|
182
|
+
"""Execute the workflow."""
|
|
183
|
+
|
|
184
|
+
self.__status = StepFunctionStatus.RUNNING
|
|
185
|
+
|
|
186
|
+
self.__logger.debug(
|
|
187
|
+
f"StepFunction - {self.__name} - Status - {self.__status.value}")
|
|
188
|
+
|
|
189
|
+
self.__last_result = initial_input
|
|
190
|
+
|
|
191
|
+
while self.__current_step:
|
|
192
|
+
step = self.__steps[self.__current_step]
|
|
193
|
+
try:
|
|
194
|
+
if step["parallel"]:
|
|
195
|
+
results = self._execute_parallel(
|
|
196
|
+
step["func"], step["stop_on_failure"])
|
|
197
|
+
|
|
198
|
+
self.__last_result = results
|
|
199
|
+
self.__context[self.__current_step] = results
|
|
200
|
+
|
|
201
|
+
self.__logger.info(
|
|
202
|
+
f"Parallel step '{
|
|
203
|
+
self.__current_step}' succeeded with results: {results}"
|
|
204
|
+
)
|
|
205
|
+
else:
|
|
206
|
+
result = await self._execute_step(step["func"], self.__last_result)
|
|
207
|
+
|
|
208
|
+
self.__last_result = result
|
|
209
|
+
self.__context[self.__current_step] = result
|
|
210
|
+
|
|
211
|
+
self.__logger.info(
|
|
212
|
+
f"Step '{self.__current_step}' succeeded"
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
if step['branch'] and result in step['branch']:
|
|
216
|
+
self.__current_step = step['branch'][result]
|
|
217
|
+
else:
|
|
218
|
+
self.__current_step = step["next_step"]
|
|
219
|
+
|
|
220
|
+
except Exception as exc:
|
|
221
|
+
self.__logger.exception(
|
|
222
|
+
f"Step '{self.__current_step}' failed. Exception: {exc}"
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
self.__context[self.__current_step] = exc.args[0]
|
|
226
|
+
|
|
227
|
+
if step["on_failure"]:
|
|
228
|
+
self.__logger.exception(
|
|
229
|
+
f"Executing failure step: {
|
|
230
|
+
step['on_failure']} for '{self.__current_step}'"
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
self.__current_step = step["on_failure"]
|
|
234
|
+
self.__last_result = exc.args[0]
|
|
235
|
+
|
|
236
|
+
self.__status = StepFunctionStatus.FAILED
|
|
237
|
+
|
|
238
|
+
self.__logger.debug(
|
|
239
|
+
f"StepFunction - {self.__name} - Status - {self.__status.value}")
|
|
240
|
+
else:
|
|
241
|
+
self.__logger.exception(
|
|
242
|
+
f"No failure step defined for '{
|
|
243
|
+
self.__current_step}'. Raising Exception."
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
self.__status = StepFunctionStatus.FAILED
|
|
247
|
+
|
|
248
|
+
self.__logger.debug(
|
|
249
|
+
f"StepFunction - {self.__name} - Status - {self.__status.value}")
|
|
250
|
+
|
|
251
|
+
raise StepExecutionError(exc)
|
|
252
|
+
|
|
253
|
+
if not self.__current_step and self.__status != StepFunctionStatus.FAILED:
|
|
254
|
+
# If no more steps, mark as COMPLETED
|
|
255
|
+
|
|
256
|
+
self.__status = StepFunctionStatus.COMPLETED
|
|
257
|
+
|
|
258
|
+
self.__logger.debug(
|
|
259
|
+
f"StepFunction - {self.__name} - Status - {self.__status.value}")
|
|
260
|
+
|
|
261
|
+
async def _execute_step(self, func: Callable, input_value: Any):
|
|
262
|
+
""" Execute a single step, handling async functions. """
|
|
263
|
+
if iscoroutinefunction(func):
|
|
264
|
+
return await func(input_value)
|
|
265
|
+
else:
|
|
266
|
+
return func(input_value)
|
|
267
|
+
|
|
268
|
+
def _execute_parallel(self, func_dict: Dict[str, Callable[[Any], Any]], stop_on_failure: bool = False):
|
|
269
|
+
"""Execute the steps in parallel."""
|
|
270
|
+
results = {}
|
|
271
|
+
errors = []
|
|
272
|
+
should_stop_execution = False
|
|
273
|
+
|
|
274
|
+
with ThreadPoolExecutor() as executor:
|
|
275
|
+
futures = {executor.submit(
|
|
276
|
+
func, self.__last_result): step_name for step_name, func in func_dict.items()}
|
|
277
|
+
|
|
278
|
+
for future in as_completed(futures):
|
|
279
|
+
step_name = futures[future]
|
|
280
|
+
|
|
281
|
+
if should_stop_execution:
|
|
282
|
+
break
|
|
283
|
+
|
|
284
|
+
try:
|
|
285
|
+
result = future.result()
|
|
286
|
+
results[step_name] = result
|
|
287
|
+
except Exception as exc:
|
|
288
|
+
self.__logger.exception(
|
|
289
|
+
f"Parallel task '{step_name}' failed: {exc}")
|
|
290
|
+
|
|
291
|
+
results[step_name] = exc.args[0]
|
|
292
|
+
|
|
293
|
+
errors.append((step_name, exc))
|
|
294
|
+
|
|
295
|
+
if stop_on_failure:
|
|
296
|
+
should_stop_execution = True
|
|
297
|
+
|
|
298
|
+
if errors:
|
|
299
|
+
self.__logger.error(f"Some parallel tasks failed: {errors}")
|
|
300
|
+
|
|
301
|
+
if stop_on_failure:
|
|
302
|
+
self.__status = StepFunctionStatus.FAILED
|
|
303
|
+
raise ParallelStepExecutionError(errors)
|
|
304
|
+
|
|
305
|
+
return results
|
|
306
|
+
|
|
307
|
+
def visualize(self):
|
|
308
|
+
"""Visualize the workflow."""
|
|
309
|
+
visualizer = Visualizer(self.__name, self.__steps)
|
|
310
|
+
|
|
311
|
+
self.__logger.debug("Visualizing the step function")
|
|
312
|
+
|
|
313
|
+
visualizer.visualize_step_function()
|
|
314
|
+
|
|
315
|
+
self.__logger.debug("Rendering the step function")
|
|
316
|
+
|
|
317
|
+
visualizer.render_step_function()
|
|
318
|
+
|
|
319
|
+
output_file_name = visualizer.output_file_name
|
|
320
|
+
|
|
321
|
+
self.__logger.debug(f"Rendered the step function to file: {
|
|
322
|
+
output_file_name}")
|
|
323
|
+
|
|
324
|
+
def visualize_to_string(self):
|
|
325
|
+
"""Visualize the workflow as a string."""
|
|
326
|
+
visualizer = Visualizer(self.__name, self.__steps)
|
|
327
|
+
|
|
328
|
+
self.__logger.debug("Visualizing the step function")
|
|
329
|
+
|
|
330
|
+
visualizer.visualize_step_function()
|
|
331
|
+
|
|
332
|
+
return visualizer.render_step_function_to_string()
|
|
333
|
+
|
|
334
|
+
@property
|
|
335
|
+
def name(self):
|
|
336
|
+
""" Returns the name of the step function.
|
|
337
|
+
"""
|
|
338
|
+
return self.__name
|
|
339
|
+
|
|
340
|
+
@property
|
|
341
|
+
def steps(self):
|
|
342
|
+
""" Returns the steps of the step function.
|
|
343
|
+
"""
|
|
344
|
+
return self.__steps
|
|
345
|
+
|
|
346
|
+
@property
|
|
347
|
+
def last_result(self):
|
|
348
|
+
""" Returns the result of the last step.
|
|
349
|
+
"""
|
|
350
|
+
return self.__last_result
|
|
351
|
+
|
|
352
|
+
@property
|
|
353
|
+
def context(self):
|
|
354
|
+
""" Returns the context of the step function.
|
|
355
|
+
"""
|
|
356
|
+
return self.__context
|
|
357
|
+
|
|
358
|
+
@property
|
|
359
|
+
def status(self):
|
|
360
|
+
""" Returns the status of the step function.
|
|
361
|
+
"""
|
|
362
|
+
return self.__status
|
|
363
|
+
|
|
364
|
+
def __str__(self):
|
|
365
|
+
return f"StepFunction(name={self.__name}, NoOfSteps={len(self.__steps)}, CurrentStep={self.__current_step}, Status={self.__status})"
|
|
366
|
+
|
|
367
|
+
def __repr__(self):
|
|
368
|
+
return str(self)
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
""" This module contains the visualizer class for the graph model.
|
|
2
|
+
|
|
3
|
+
Author: Vineeth Penugonda
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from os import getcwd
|
|
7
|
+
|
|
8
|
+
from graphviz import Digraph
|
|
9
|
+
|
|
10
|
+
from stepfunction.constants.visualizer import (
|
|
11
|
+
DEFAULT_VISUALIZER_EXTENSION, DEFAULT_VISUALIZER_FAILURE_EDGE_COLOR,
|
|
12
|
+
DEFAULT_VISUALIZER_FAILURE_EDGE_LABEL, DEFAULT_VISUALIZER_FOLDER,
|
|
13
|
+
DEFAULT_VISUALIZER_FORMAT, DEFAULT_VISUALIZER_PARALLEL_STEP_EDGE_STYLE,
|
|
14
|
+
DEFAULT_VISUALIZER_RENDERER, DEFAULT_VISUALIZER_STOP_ON_FAILURE_EDGE_COLOR,
|
|
15
|
+
DEFAULT_VISUALIZER_STOP_ON_FAILURE_EDGE_LABEL,
|
|
16
|
+
DEFAULT_VISUALIZER_STRING_ENCODING,
|
|
17
|
+
DEFAULT_VISUALIZER_SUB_STEP_FUNCTION_NODE_SHAPE,
|
|
18
|
+
DEFAULT_VISUALIZER_SUB_STEP_FUNCTION_NODE_STYLE,
|
|
19
|
+
DEFAULT_VISUALIZER_SUCCESS_EDGE_LABEL)
|
|
20
|
+
from stepfunction.types.step_types import StepParams
|
|
21
|
+
from stepfunction.types.visualizer_types import RenderStepFunctionParams
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Visualizer:
|
|
25
|
+
""" This class is responsible for visualizing the graph model.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
def __init__(self, graph_name: str, steps: StepParams = None):
|
|
29
|
+
""" Initializes the visualizer."""
|
|
30
|
+
|
|
31
|
+
self.graph_name = graph_name
|
|
32
|
+
self.__steps = steps
|
|
33
|
+
|
|
34
|
+
self.__output_file_name = None
|
|
35
|
+
self.__output_file_path = None
|
|
36
|
+
|
|
37
|
+
self.__dot = Digraph(comment=self.graph_name)
|
|
38
|
+
|
|
39
|
+
def visualize_step_function(self):
|
|
40
|
+
""" Visualizes the graph model.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
if not self.__steps:
|
|
44
|
+
raise ValueError("No steps found to visualize.")
|
|
45
|
+
|
|
46
|
+
for step_name, step_info in self.__steps.items():
|
|
47
|
+
|
|
48
|
+
if 'is_sub_step_function' in step_info and step_info['is_sub_step_function']:
|
|
49
|
+
self.__dot.node(step_name, step_name, shape=DEFAULT_VISUALIZER_SUB_STEP_FUNCTION_NODE_SHAPE,
|
|
50
|
+
style=DEFAULT_VISUALIZER_SUB_STEP_FUNCTION_NODE_STYLE)
|
|
51
|
+
else:
|
|
52
|
+
self.__dot.node(step_name, step_name)
|
|
53
|
+
|
|
54
|
+
if step_info['next_step']:
|
|
55
|
+
self.__dot.edge(
|
|
56
|
+
step_name, step_info['next_step'], label=DEFAULT_VISUALIZER_SUCCESS_EDGE_LABEL)
|
|
57
|
+
|
|
58
|
+
if step_info['on_failure']:
|
|
59
|
+
failure_edge_label = DEFAULT_VISUALIZER_FAILURE_EDGE_LABEL
|
|
60
|
+
failure_edge_color = DEFAULT_VISUALIZER_FAILURE_EDGE_COLOR
|
|
61
|
+
|
|
62
|
+
if step_info.get('stop_on_failure'):
|
|
63
|
+
failure_edge_label = DEFAULT_VISUALIZER_STOP_ON_FAILURE_EDGE_LABEL
|
|
64
|
+
failure_edge_color = DEFAULT_VISUALIZER_STOP_ON_FAILURE_EDGE_COLOR
|
|
65
|
+
|
|
66
|
+
self.__dot.edge(
|
|
67
|
+
step_name, step_info['on_failure'], label=failure_edge_label, color=failure_edge_color)
|
|
68
|
+
|
|
69
|
+
if step_info['parallel']:
|
|
70
|
+
|
|
71
|
+
# func is a dictionary for parallel steps
|
|
72
|
+
parallel_function_names = step_info['func']
|
|
73
|
+
|
|
74
|
+
with self.__dot.subgraph() as s:
|
|
75
|
+
s.attr(rank='same')
|
|
76
|
+
|
|
77
|
+
for parallel_step_name, func in parallel_function_names.items():
|
|
78
|
+
self.__dot.node(parallel_step_name, parallel_step_name)
|
|
79
|
+
self.__dot.edge(
|
|
80
|
+
step_name, parallel_step_name, style=DEFAULT_VISUALIZER_PARALLEL_STEP_EDGE_STYLE)
|
|
81
|
+
|
|
82
|
+
if step_info['next_step']:
|
|
83
|
+
self.__dot.edge(parallel_step_name,
|
|
84
|
+
step_info['next_step'])
|
|
85
|
+
|
|
86
|
+
if step_info.get('branch'):
|
|
87
|
+
for result, next_step in step_info['branch'].items():
|
|
88
|
+
self.__dot.edge(step_name, next_step,
|
|
89
|
+
label=f"Branch: {result}")
|
|
90
|
+
|
|
91
|
+
def render_step_function(self, **kwargs: RenderStepFunctionParams):
|
|
92
|
+
""" Renders the graph model.
|
|
93
|
+
"""
|
|
94
|
+
current_dir = getcwd()
|
|
95
|
+
|
|
96
|
+
format = kwargs.get('format', DEFAULT_VISUALIZER_FORMAT)
|
|
97
|
+
renderer = kwargs.get('renderer', DEFAULT_VISUALIZER_RENDERER)
|
|
98
|
+
|
|
99
|
+
file_path = kwargs.get(
|
|
100
|
+
'file_path', f"{current_dir}/{DEFAULT_VISUALIZER_FOLDER}")
|
|
101
|
+
file_name = kwargs.get('file_name', f"{self.graph_name}.{
|
|
102
|
+
DEFAULT_VISUALIZER_EXTENSION}")
|
|
103
|
+
|
|
104
|
+
self.__output_file_path = file_path
|
|
105
|
+
self.__output_file_name = file_name
|
|
106
|
+
|
|
107
|
+
self.__dot.render(
|
|
108
|
+
filename=f"{self.__output_file_path}/{self.__output_file_name}", format=format, renderer=renderer)
|
|
109
|
+
|
|
110
|
+
def render_step_function_to_string(self, **kwargs: RenderStepFunctionParams):
|
|
111
|
+
""" Renders the graph model as a string.
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
format = kwargs.get('format', DEFAULT_VISUALIZER_FORMAT)
|
|
115
|
+
renderer = kwargs.get('renderer', DEFAULT_VISUALIZER_RENDERER)
|
|
116
|
+
|
|
117
|
+
return self.__dot.pipe(format=format, renderer=renderer).decode(DEFAULT_VISUALIZER_STRING_ENCODING)
|
|
118
|
+
|
|
119
|
+
@property
|
|
120
|
+
def output_file_name(self):
|
|
121
|
+
""" Returns the output file name.
|
|
122
|
+
"""
|
|
123
|
+
return self.__output_file_name
|
|
124
|
+
|
|
125
|
+
@property
|
|
126
|
+
def output_file_path(self):
|
|
127
|
+
""" Returns the output file path.
|
|
128
|
+
"""
|
|
129
|
+
return self.__output_file_path
|
|
File without changes
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
class StepExecutionError(Exception):
|
|
2
|
+
"""Error raised when a step fails."""
|
|
3
|
+
|
|
4
|
+
def __init__(self, exc: Exception):
|
|
5
|
+
self.message = f"Step generated an exception: {exc}"
|
|
6
|
+
super().__init__(self.message)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ParallelStepExecutionError(Exception):
|
|
10
|
+
"""Error raised when a parallel step fails."""
|
|
11
|
+
|
|
12
|
+
def __init__(self, exc: Exception):
|
|
13
|
+
self.message = f"Parallel step generated an exception: {exc}"
|
|
14
|
+
super().__init__(self.message)
|
|
File without changes
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
from typing import Any, Callable, Dict, Optional, TypedDict
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class StepParams(TypedDict, total=False):
|
|
5
|
+
|
|
6
|
+
func: Callable[[Any], Any]
|
|
7
|
+
next_step: Optional[str]
|
|
8
|
+
on_failure: Optional[str]
|
|
9
|
+
branch: Optional[Dict[Any, str]]
|
|
10
|
+
parallel: bool
|
|
11
|
+
stop_on_failure: bool
|
|
12
|
+
is_sub_step_function: bool
|
|
File without changes
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
""" Constants for the project. """
|
|
2
|
+
|
|
3
|
+
# Logging
|
|
4
|
+
DEFAULT_LOG_LEVEL = "INFO"
|
|
5
|
+
"""str: The default logging level for the project."""
|
|
6
|
+
|
|
7
|
+
DEFAULT_LOGGING_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
8
|
+
"""str: The default logging format for the project."""
|
|
9
|
+
|
|
10
|
+
# Environment variables
|
|
11
|
+
ENVIRONMENT_VARIABLE_LOG_LEVEL = "LOG_LEVEL"
|
|
12
|
+
""" str: The environment variable for the logging level."""
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
""" Module for setting up loggers. """
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import logging.config
|
|
5
|
+
|
|
6
|
+
from stepfunction.utils.constants import (DEFAULT_LOG_LEVEL,
|
|
7
|
+
DEFAULT_LOGGING_FORMAT,
|
|
8
|
+
ENVIRONMENT_VARIABLE_LOG_LEVEL)
|
|
9
|
+
from stepfunction.utils.utils import get_environment_variable
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def setup_logger(name: str | None = None, log_format: str = DEFAULT_LOGGING_FORMAT) -> logging.Logger:
|
|
13
|
+
"""
|
|
14
|
+
Set up and return a logger with the given name.
|
|
15
|
+
If no name is provided, return the root logger.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
LOG_LEVEL = get_environment_variable(
|
|
19
|
+
ENVIRONMENT_VARIABLE_LOG_LEVEL, DEFAULT_LOG_LEVEL)
|
|
20
|
+
|
|
21
|
+
logging_config = {
|
|
22
|
+
"version": 1,
|
|
23
|
+
"disable_existing_loggers": False,
|
|
24
|
+
"formatters": {
|
|
25
|
+
"default": {
|
|
26
|
+
"format": log_format,
|
|
27
|
+
},
|
|
28
|
+
},
|
|
29
|
+
"handlers": {
|
|
30
|
+
"console": {
|
|
31
|
+
"class": "logging.StreamHandler",
|
|
32
|
+
"formatter": "default",
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
"root": {
|
|
36
|
+
"level": LOG_LEVEL,
|
|
37
|
+
"handlers": ["console"],
|
|
38
|
+
},
|
|
39
|
+
"loggers": {
|
|
40
|
+
"filecentral": {
|
|
41
|
+
"level": LOG_LEVEL,
|
|
42
|
+
"handlers": ["console"],
|
|
43
|
+
"propagate": False,
|
|
44
|
+
},
|
|
45
|
+
|
|
46
|
+
"botocore": {
|
|
47
|
+
"level": "WARNING", # Set level to WARNING to ignore DEBUG logs
|
|
48
|
+
"handlers": ["console"],
|
|
49
|
+
"propagate": False,
|
|
50
|
+
},
|
|
51
|
+
"urllib3.connectionpool": {
|
|
52
|
+
"level": "ERROR", # Only log errors and above to reduce noise
|
|
53
|
+
"handlers": ["console"],
|
|
54
|
+
"propagate": False,
|
|
55
|
+
},
|
|
56
|
+
},
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
logging.config.dictConfig(logging_config)
|
|
60
|
+
|
|
61
|
+
return logging.getLogger(name)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
""" This module contains utility functions for the stepfunction package. """
|
|
2
|
+
|
|
3
|
+
from os import getenv
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def get_environment_variable(name: str, default: str = None) -> str:
|
|
7
|
+
"""
|
|
8
|
+
Returns the value of the environment variable with the given name.
|
|
9
|
+
|
|
10
|
+
Args:
|
|
11
|
+
name (str): The name of the environment variable.
|
|
12
|
+
default (str): The default value to return if the environment variable is not set.
|
|
13
|
+
|
|
14
|
+
Returns:
|
|
15
|
+
str: The value of the environment variable.
|
|
16
|
+
"""
|
|
17
|
+
return getenv(name, default)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 Vineeth Penugonda
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: stepfunction
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Step Function Workflow Orchestration Library
|
|
5
|
+
Author: Vineeth Penugonda
|
|
6
|
+
Project-URL: Homepage, https://github.com/vinecodes/stepfunction
|
|
7
|
+
Project-URL: Issues, https://github.com/vinecodes/stepfunction/issues
|
|
8
|
+
Project-URL: Blog_Post, https://blog.vineethp.com/posts/introducingstepfunction/
|
|
9
|
+
Keywords: StepFunction,Workflow,Orchestration,Library
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
16
|
+
Requires-Python: >=3.12
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: graphviz ===0.20.3
|
|
20
|
+
|
|
21
|
+
# StepFunction
|
|
22
|
+
|
|
23
|
+

|
|
24
|
+

|
|
25
|
+

|
|
26
|
+
|
|
27
|
+
`StepFunction` is a Python library for orchestrating complex workflows through a step-by-step process. It allows for easy management of sequential and parallel tasks with robust error handling and branching. The library is inspired by AWS Step Functions but implemented as a Python-native solution to orchestrate workflows across any environment.
|
|
28
|
+
|
|
29
|
+
## Features
|
|
30
|
+
|
|
31
|
+
- **Sequential and Parallel Execution**: Run steps one after another or in parallel.
|
|
32
|
+
- **Error Handling**: Define failure paths and handle exceptions gracefully.
|
|
33
|
+
- **Branching Logic**: Direct workflows based on conditions.
|
|
34
|
+
- **Sub-Step Functions**: Modularize workflows by embedding sub-step functions.
|
|
35
|
+
- **Visualization**: Integrated support for visualizing the workflow graph.
|
|
36
|
+
|
|
37
|
+
## Installation
|
|
38
|
+
|
|
39
|
+
You can install the `StepFunction` package using `pip`:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
pip install stepfunction
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Documentation and Further Reading
|
|
46
|
+
|
|
47
|
+
For more detailed usage examples and advanced use cases, please read the [blog post](https://blog.vineethp.com/posts/introducingstepfunction/) that explains the design and usage of the library in-depth. This blog post covers advanced concepts like error handling, sub-step functions, and visualizing workflows in a real-world context.
|
|
48
|
+
|
|
49
|
+
## Dependencies
|
|
50
|
+
|
|
51
|
+
The StepFunction package requires the following dependencies:
|
|
52
|
+
|
|
53
|
+
- `graphviz == 0.20.3`
|
|
54
|
+
|
|
55
|
+
These will be automatically installed when you install the package via pip.
|
|
56
|
+
|
|
57
|
+
## Contributing
|
|
58
|
+
|
|
59
|
+
If you'd like to contribute to StepFunction, feel free to submit issues or pull requests. Contributions are welcome!
|
|
60
|
+
|
|
61
|
+
## License
|
|
62
|
+
|
|
63
|
+
This project is licensed under the MIT License - see the LICENSE file for details.
|
|
64
|
+
|
|
65
|
+
## Author
|
|
66
|
+
Created and maintained by **Vineeth Penugonda**.
|
|
67
|
+
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
stepfunction/constants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
stepfunction/constants/enums.py,sha256=NnJ_388IDnKQhjk_692TjL5O-KFVOA3b_PMhggyKGpA,166
|
|
3
|
+
stepfunction/constants/visualizer.py,sha256=99isL2AnwQln_Rc_-asIu3PCDEscipCf5HMlWLU7PWw,1763
|
|
4
|
+
stepfunction/core/step_function/__init__.py,sha256=AGg68Zm_9qud_2JORuQ2fdb1hJkc-t9J3iMXTOnbUJk,67
|
|
5
|
+
stepfunction/core/step_function/step_function.py,sha256=W9d3BKojyt0T7E6Ue2-fsJUSrz-_eGD_Qw139IIz46o,13867
|
|
6
|
+
stepfunction/core/visualizer/__init__.py,sha256=sZWiVnJKdgm_yv-G9nLeGHhNdYbbuCh72SwcJ6kgYBQ,60
|
|
7
|
+
stepfunction/core/visualizer/visualizer.py,sha256=RyaR8rFRJ8JJNpGe6C_FEfx67D0OBJ5c0RFUibBkQNc,5013
|
|
8
|
+
stepfunction/exceptions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
stepfunction/exceptions/step_errors.py,sha256=9CQGL18IfnXvRLTnBmESuW6p_1yzMB3Jx1PrqWoQfPQ,468
|
|
10
|
+
stepfunction/types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
+
stepfunction/types/step_types.py,sha256=IdxcIM9WDVozQPUFwTUuhZUikz2LAq-o_6zxqBp4ePQ,308
|
|
12
|
+
stepfunction/types/visualizer_types.py,sha256=kAytibRU7eTm2MI02zJLYH-ZkRa4kosGfDjggCAFR94,187
|
|
13
|
+
stepfunction/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
|
+
stepfunction/utils/constants.py,sha256=cCeOZUJdoaSoBV5lM5ZOGg7d5sh1oFTum26nNOopKeo,392
|
|
15
|
+
stepfunction/utils/logger.py,sha256=5beoM6rxSKbv_4nWOfwGJ93V4nR9ewXAnRjTwQsIO3U,1819
|
|
16
|
+
stepfunction/utils/utils.py,sha256=tIAgd6HEGJn8MMSkgrmyKVvBzHrFRyZF4jY1uP55BZ8,541
|
|
17
|
+
stepfunction-0.0.1.dist-info/LICENSE,sha256=8YO69WbB5N-EJjk-po0P7LiWUmnxdsycwqEqgG5F4JY,1073
|
|
18
|
+
stepfunction-0.0.1.dist-info/METADATA,sha256=b1t1Ey-51t4vHjYoeXAJI7ylD9N1cRSIvy5o6JXcrbM,2733
|
|
19
|
+
stepfunction-0.0.1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
|
|
20
|
+
stepfunction-0.0.1.dist-info/top_level.txt,sha256=XqLJSUeYSmzTNMiPj-IESxp1om-LdZjefT4Un2orQD4,13
|
|
21
|
+
stepfunction-0.0.1.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
stepfunction
|