ddeutil-workflow 0.0.81__py3-none-any.whl → 0.0.82__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +2 -1
- ddeutil/workflow/__init__.py +19 -6
- ddeutil/workflow/__main__.py +280 -1
- ddeutil/workflow/api/routes/job.py +2 -2
- ddeutil/workflow/api/routes/logs.py +8 -61
- ddeutil/workflow/audits.py +46 -17
- ddeutil/workflow/conf.py +45 -25
- ddeutil/workflow/errors.py +9 -0
- ddeutil/workflow/job.py +70 -16
- ddeutil/workflow/result.py +33 -11
- ddeutil/workflow/stages.py +172 -134
- ddeutil/workflow/traces.py +64 -24
- ddeutil/workflow/utils.py +7 -4
- ddeutil/workflow/workflow.py +66 -75
- {ddeutil_workflow-0.0.81.dist-info → ddeutil_workflow-0.0.82.dist-info}/METADATA +1 -1
- ddeutil_workflow-0.0.82.dist-info/RECORD +35 -0
- ddeutil/workflow/cli.py +0 -284
- ddeutil_workflow-0.0.81.dist-info/RECORD +0 -36
- {ddeutil_workflow-0.0.81.dist-info → ddeutil_workflow-0.0.82.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.81.dist-info → ddeutil_workflow-0.0.82.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.81.dist-info → ddeutil_workflow-0.0.82.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.81.dist-info → ddeutil_workflow-0.0.82.dist-info}/top_level.txt +0 -0
ddeutil/workflow/stages.py
CHANGED
@@ -36,16 +36,6 @@ the stage execution method.
|
|
36
36
|
execute method receives a `params={"params": {...}}` value for passing template
|
37
37
|
searching.
|
38
38
|
|
39
|
-
All stages model inherit from `BaseStage` or `AsyncBaseStage` models that has the
|
40
|
-
base fields:
|
41
|
-
|
42
|
-
| field | alias | data type | default | description |
|
43
|
-
|-----------|-------|-------------|:--------:|-----------------------------------------------------------------------|
|
44
|
-
| id | | str \| None | `None` | A stage ID that use to keep execution output or getting by job owner. |
|
45
|
-
| name | | str | | A stage name that want to log when start execution. |
|
46
|
-
| condition | if | str \| None | `None` | A stage condition statement to allow stage executable. |
|
47
|
-
| extras | | dict | `dict()` | An extra parameter that override core config values. |
|
48
|
-
|
49
39
|
It has a special base class is `BaseRetryStage` that inherit from `AsyncBaseStage`
|
50
40
|
that use to handle retry execution when it got any error with `retry` field.
|
51
41
|
"""
|
@@ -92,9 +82,18 @@ from pydantic import BaseModel, Field, ValidationError
|
|
92
82
|
from pydantic.functional_validators import field_validator, model_validator
|
93
83
|
from typing_extensions import Self
|
94
84
|
|
85
|
+
from .__about__ import __python_version__
|
95
86
|
from .__types import DictData, DictStr, StrOrInt, StrOrNone, TupleStr
|
96
87
|
from .conf import dynamic, pass_env
|
97
|
-
from .errors import
|
88
|
+
from .errors import (
|
89
|
+
StageCancelError,
|
90
|
+
StageError,
|
91
|
+
StageNestedCancelError,
|
92
|
+
StageNestedError,
|
93
|
+
StageNestedSkipError,
|
94
|
+
StageSkipError,
|
95
|
+
to_dict,
|
96
|
+
)
|
98
97
|
from .result import (
|
99
98
|
CANCEL,
|
100
99
|
FAILED,
|
@@ -114,7 +113,7 @@ from .reusables import (
|
|
114
113
|
not_in_template,
|
115
114
|
param2template,
|
116
115
|
)
|
117
|
-
from .traces import
|
116
|
+
from .traces import Trace, get_trace
|
118
117
|
from .utils import (
|
119
118
|
delay,
|
120
119
|
dump_all,
|
@@ -155,13 +154,11 @@ class BaseStage(BaseModel, ABC):
|
|
155
154
|
process: Main execution logic that must be implemented by subclasses
|
156
155
|
|
157
156
|
Example:
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
# Custom execution logic
|
164
|
-
return Result(status=SUCCESS)
|
157
|
+
>>> class CustomStage(BaseStage):
|
158
|
+
... custom_param: str = Field(description="Custom parameter")
|
159
|
+
...
|
160
|
+
... def process(self, params: DictData, **kwargs) -> Result:
|
161
|
+
... return Result(status=SUCCESS)
|
165
162
|
```
|
166
163
|
"""
|
167
164
|
|
@@ -208,7 +205,8 @@ class BaseStage(BaseModel, ABC):
|
|
208
205
|
def ___prepare_desc__(cls, value: str) -> str:
|
209
206
|
"""Prepare description string that was created on a template.
|
210
207
|
|
211
|
-
:
|
208
|
+
Returns:
|
209
|
+
str: A dedent and left strip newline of description string.
|
212
210
|
"""
|
213
211
|
return dedent(value.lstrip("\n"))
|
214
212
|
|
@@ -218,16 +216,17 @@ class BaseStage(BaseModel, ABC):
|
|
218
216
|
method will validate name and id fields should not contain any template
|
219
217
|
parameter (exclude matrix template).
|
220
218
|
|
221
|
-
:
|
222
|
-
|
219
|
+
Raises:
|
220
|
+
ValueError: When the ID and name fields include matrix parameter
|
221
|
+
template with the 'matrix.' string value.
|
223
222
|
|
224
|
-
:
|
223
|
+
Returns: Self
|
225
224
|
"""
|
226
225
|
# VALIDATE: Validate stage id and name should not dynamic with params
|
227
226
|
# template. (allow only matrix)
|
228
227
|
if not_in_template(self.id) or not_in_template(self.name):
|
229
228
|
raise ValueError(
|
230
|
-
"Stage name and ID should only template with 'matrix.
|
229
|
+
"Stage name and ID should only template with 'matrix.?'."
|
231
230
|
)
|
232
231
|
return self
|
233
232
|
|
@@ -302,9 +301,9 @@ class BaseStage(BaseModel, ABC):
|
|
302
301
|
"""
|
303
302
|
ts: float = time.monotonic()
|
304
303
|
parent_run_id: str = run_id
|
305
|
-
run_id: str =
|
304
|
+
run_id: str = gen_id(self.iden, unique=True, extras=self.extras)
|
306
305
|
context: DictData = {"status": WAIT}
|
307
|
-
trace:
|
306
|
+
trace: Trace = get_trace(
|
308
307
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
309
308
|
)
|
310
309
|
try:
|
@@ -349,13 +348,21 @@ class BaseStage(BaseModel, ABC):
|
|
349
348
|
# this exception class at other location.
|
350
349
|
except (
|
351
350
|
StageSkipError,
|
352
|
-
|
351
|
+
StageNestedSkipError,
|
352
|
+
StageNestedError,
|
353
353
|
StageError,
|
354
354
|
) as e: # pragma: no cov
|
355
|
-
|
356
|
-
f"[STAGE]: Handler
|
357
|
-
|
358
|
-
|
355
|
+
if isinstance(e, StageNestedError):
|
356
|
+
trace.info(f"[STAGE]: Handler: {e}")
|
357
|
+
else:
|
358
|
+
emoji: str = (
|
359
|
+
"⏭️"
|
360
|
+
if isinstance(e, (StageSkipError, StageNestedSkipError))
|
361
|
+
else "🚨"
|
362
|
+
)
|
363
|
+
trace.info(
|
364
|
+
f"[STAGE]: Handler:||{emoji} {traceback.format_exc()}"
|
365
|
+
)
|
359
366
|
st: Status = get_status_from_error(e)
|
360
367
|
return Result(
|
361
368
|
run_id=run_id,
|
@@ -366,7 +373,7 @@ class BaseStage(BaseModel, ABC):
|
|
366
373
|
status=st,
|
367
374
|
updated=(
|
368
375
|
None
|
369
|
-
if isinstance(e, StageSkipError)
|
376
|
+
if isinstance(e, (StageSkipError, StageNestedSkipError))
|
370
377
|
else {"errors": e.to_dict()}
|
371
378
|
),
|
372
379
|
),
|
@@ -374,10 +381,7 @@ class BaseStage(BaseModel, ABC):
|
|
374
381
|
extras=self.extras,
|
375
382
|
)
|
376
383
|
except Exception as e:
|
377
|
-
trace.error(
|
378
|
-
f"[STAGE]: Error Handler:||{e.__class__.__name__}: {e}||"
|
379
|
-
f"{traceback.format_exc()}"
|
380
|
-
)
|
384
|
+
trace.error(f"[STAGE]: Error Handler:||🚨 {traceback.format_exc()}")
|
381
385
|
return Result(
|
382
386
|
run_id=run_id,
|
383
387
|
parent_run_id=parent_run_id,
|
@@ -632,9 +636,9 @@ class BaseAsyncStage(BaseStage, ABC):
|
|
632
636
|
"""
|
633
637
|
ts: float = time.monotonic()
|
634
638
|
parent_run_id: StrOrNone = run_id
|
635
|
-
run_id: str =
|
639
|
+
run_id: str = gen_id(self.iden, unique=True, extras=self.extras)
|
636
640
|
context: DictData = {}
|
637
|
-
trace:
|
641
|
+
trace: Trace = get_trace(
|
638
642
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
639
643
|
)
|
640
644
|
try:
|
@@ -677,13 +681,21 @@ class BaseAsyncStage(BaseStage, ABC):
|
|
677
681
|
# this exception class at other location.
|
678
682
|
except (
|
679
683
|
StageSkipError,
|
680
|
-
|
684
|
+
StageNestedSkipError,
|
685
|
+
StageNestedError,
|
681
686
|
StageError,
|
682
687
|
) as e: # pragma: no cov
|
683
|
-
|
684
|
-
f"[STAGE]:
|
685
|
-
|
686
|
-
|
688
|
+
if isinstance(e, StageNestedError):
|
689
|
+
await trace.ainfo(f"[STAGE]: Handler: {e}")
|
690
|
+
else:
|
691
|
+
emoji: str = (
|
692
|
+
"⏭️"
|
693
|
+
if isinstance(e, (StageSkipError, StageNestedSkipError))
|
694
|
+
else "🚨"
|
695
|
+
)
|
696
|
+
await trace.ainfo(
|
697
|
+
f"[STAGE]:Handler:||{emoji} {traceback.format_exc()}"
|
698
|
+
)
|
687
699
|
st: Status = get_status_from_error(e)
|
688
700
|
return Result(
|
689
701
|
run_id=run_id,
|
@@ -703,8 +715,7 @@ class BaseAsyncStage(BaseStage, ABC):
|
|
703
715
|
)
|
704
716
|
except Exception as e:
|
705
717
|
await trace.aerror(
|
706
|
-
f"[STAGE]: Error Handler
|
707
|
-
f"{traceback.format_exc()}"
|
718
|
+
f"[STAGE]: Error Handler:||🚨 {traceback.format_exc()}"
|
708
719
|
)
|
709
720
|
return Result(
|
710
721
|
run_id=run_id,
|
@@ -777,7 +788,7 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
777
788
|
current_retry: int = 0
|
778
789
|
exception: Exception
|
779
790
|
catch(context, status=WAIT)
|
780
|
-
trace:
|
791
|
+
trace: Trace = get_trace(
|
781
792
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
782
793
|
)
|
783
794
|
|
@@ -823,6 +834,7 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
823
834
|
f"( {e.__class__.__name__} )"
|
824
835
|
)
|
825
836
|
exception = e
|
837
|
+
time.sleep(1.2**current_retry)
|
826
838
|
|
827
839
|
trace.error(
|
828
840
|
f"[STAGE]: Reach the maximum of retry number: {self.retry}."
|
@@ -850,7 +862,7 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
850
862
|
current_retry: int = 0
|
851
863
|
exception: Exception
|
852
864
|
catch(context, status=WAIT)
|
853
|
-
trace:
|
865
|
+
trace: Trace = get_trace(
|
854
866
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
855
867
|
)
|
856
868
|
|
@@ -896,6 +908,7 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
896
908
|
f"( {e.__class__.__name__} )"
|
897
909
|
)
|
898
910
|
exception = e
|
911
|
+
await asyncio.sleep(1.2**current_retry)
|
899
912
|
|
900
913
|
await trace.aerror(
|
901
914
|
f"[STAGE]: Reach the maximum of retry number: {self.retry}."
|
@@ -908,19 +921,15 @@ class EmptyStage(BaseAsyncStage):
|
|
908
921
|
|
909
922
|
EmptyStage is a utility stage that performs no actual work but provides
|
910
923
|
logging output and optional delays. It's commonly used for:
|
911
|
-
|
912
|
-
|
913
|
-
|
914
|
-
|
924
|
+
- Debugging workflow execution flow
|
925
|
+
- Adding informational messages to workflows
|
926
|
+
- Creating delays between stages
|
927
|
+
- Testing template parameter resolution
|
915
928
|
|
916
929
|
The stage outputs the echo message to stdout and can optionally sleep
|
917
930
|
for a specified duration, making it useful for workflow timing control
|
918
931
|
and debugging scenarios.
|
919
932
|
|
920
|
-
Attributes:
|
921
|
-
echo (str, optional): Message to display during execution
|
922
|
-
sleep (float): Duration to sleep after logging (0-1800 seconds)
|
923
|
-
|
924
933
|
Example:
|
925
934
|
```yaml
|
926
935
|
stages:
|
@@ -932,24 +941,25 @@ class EmptyStage(BaseAsyncStage):
|
|
932
941
|
echo: "Processing file: ${{ params.filename }}"
|
933
942
|
```
|
934
943
|
|
935
|
-
|
936
|
-
|
937
|
-
|
938
|
-
|
939
|
-
|
940
|
-
)
|
941
|
-
```
|
944
|
+
>>> stage = EmptyStage(
|
945
|
+
... name="Status Update",
|
946
|
+
... echo="Processing completed successfully",
|
947
|
+
... sleep=1.0
|
948
|
+
... )
|
942
949
|
"""
|
943
950
|
|
944
951
|
echo: StrOrNone = Field(
|
945
952
|
default=None,
|
946
|
-
description=
|
953
|
+
description=(
|
954
|
+
"A message that want to display on the stdout during execution. "
|
955
|
+
"By default, it do not show any message."
|
956
|
+
),
|
947
957
|
)
|
948
958
|
sleep: float = Field(
|
949
959
|
default=0,
|
950
960
|
description=(
|
951
|
-
"A second value to sleep
|
952
|
-
"
|
961
|
+
"A duration in second value to sleep after logging. This value "
|
962
|
+
"should between 0 - 1800 seconds."
|
953
963
|
),
|
954
964
|
ge=0,
|
955
965
|
lt=1800,
|
@@ -982,7 +992,7 @@ class EmptyStage(BaseAsyncStage):
|
|
982
992
|
Returns:
|
983
993
|
Result: The execution result with status and context data.
|
984
994
|
"""
|
985
|
-
trace:
|
995
|
+
trace: Trace = get_trace(
|
986
996
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
987
997
|
)
|
988
998
|
message: str = (
|
@@ -1035,7 +1045,7 @@ class EmptyStage(BaseAsyncStage):
|
|
1035
1045
|
Returns:
|
1036
1046
|
Result: The execution result with status and context data.
|
1037
1047
|
"""
|
1038
|
-
trace:
|
1048
|
+
trace: Trace = get_trace(
|
1039
1049
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1040
1050
|
)
|
1041
1051
|
message: str = (
|
@@ -1200,7 +1210,7 @@ class BashStage(BaseRetryStage):
|
|
1200
1210
|
Returns:
|
1201
1211
|
Result: The execution result with status and context data.
|
1202
1212
|
"""
|
1203
|
-
trace:
|
1213
|
+
trace: Trace = get_trace(
|
1204
1214
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1205
1215
|
)
|
1206
1216
|
bash: str = param2template(
|
@@ -1264,7 +1274,7 @@ class BashStage(BaseRetryStage):
|
|
1264
1274
|
Returns:
|
1265
1275
|
Result: The execution result with status and context data.
|
1266
1276
|
"""
|
1267
|
-
trace:
|
1277
|
+
trace: Trace = get_trace(
|
1268
1278
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1269
1279
|
)
|
1270
1280
|
bash: str = param2template(
|
@@ -1394,21 +1404,21 @@ class PyStage(BaseRetryStage):
|
|
1394
1404
|
to globals argument on `exec` build-in function.
|
1395
1405
|
|
1396
1406
|
Args:
|
1397
|
-
params: A parameter data that want to use in this
|
1407
|
+
params (DictData): A parameter data that want to use in this
|
1398
1408
|
execution.
|
1399
|
-
run_id: A running stage ID.
|
1409
|
+
run_id (str): A running stage ID.
|
1400
1410
|
context: A context data.
|
1401
|
-
parent_run_id: A parent running ID.
|
1411
|
+
parent_run_id (str | None, default None): A parent running ID.
|
1402
1412
|
event: An event manager that use to track parent process
|
1403
1413
|
was not force stopped.
|
1404
1414
|
|
1405
1415
|
Returns:
|
1406
1416
|
Result: The execution result with status and context data.
|
1407
1417
|
"""
|
1408
|
-
trace:
|
1418
|
+
trace: Trace = get_trace(
|
1409
1419
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1410
1420
|
)
|
1411
|
-
trace.
|
1421
|
+
trace.debug("[STAGE]: Prepare `globals` and `locals` variables.")
|
1412
1422
|
lc: DictData = {}
|
1413
1423
|
gb: DictData = (
|
1414
1424
|
globals()
|
@@ -1486,7 +1496,7 @@ class PyStage(BaseRetryStage):
|
|
1486
1496
|
Returns:
|
1487
1497
|
Result: The execution result with status and context data.
|
1488
1498
|
"""
|
1489
|
-
trace:
|
1499
|
+
trace: Trace = get_trace(
|
1490
1500
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1491
1501
|
)
|
1492
1502
|
await trace.ainfo("[STAGE]: Prepare `globals` and `locals` variables.")
|
@@ -1631,7 +1641,7 @@ class CallStage(BaseRetryStage):
|
|
1631
1641
|
Returns:
|
1632
1642
|
Result: The execution result with status and context data.
|
1633
1643
|
"""
|
1634
|
-
trace:
|
1644
|
+
trace: Trace = get_trace(
|
1635
1645
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1636
1646
|
)
|
1637
1647
|
call_func: TagFunc = self.get_caller(params=params)()
|
@@ -1750,7 +1760,7 @@ class CallStage(BaseRetryStage):
|
|
1750
1760
|
Returns:
|
1751
1761
|
Result: The execution result with status and context data.
|
1752
1762
|
"""
|
1753
|
-
trace:
|
1763
|
+
trace: Trace = get_trace(
|
1754
1764
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1755
1765
|
)
|
1756
1766
|
call_func: TagFunc = self.get_caller(params=params)()
|
@@ -1884,7 +1894,7 @@ class CallStage(BaseRetryStage):
|
|
1884
1894
|
"Validate argument from the caller function raise invalid type."
|
1885
1895
|
) from e
|
1886
1896
|
except TypeError as e:
|
1887
|
-
trace:
|
1897
|
+
trace: Trace = get_trace(
|
1888
1898
|
run_id, parent_run_id=parent_run_id, extras=extras
|
1889
1899
|
)
|
1890
1900
|
trace.warning(
|
@@ -1963,6 +1973,9 @@ class TriggerStage(BaseNestedStage):
|
|
1963
1973
|
execute method. This is the stage that allow you to create the reusable
|
1964
1974
|
Workflow template with dynamic parameters.
|
1965
1975
|
|
1976
|
+
This stage does not allow to pass the workflow model directly to the
|
1977
|
+
trigger field. A trigger workflow name should exist on the config path only.
|
1978
|
+
|
1966
1979
|
Data Validate:
|
1967
1980
|
>>> stage = {
|
1968
1981
|
... "name": "Trigger workflow stage execution",
|
@@ -2009,11 +2022,22 @@ class TriggerStage(BaseNestedStage):
|
|
2009
2022
|
"""
|
2010
2023
|
from .workflow import Workflow
|
2011
2024
|
|
2012
|
-
trace:
|
2025
|
+
trace: Trace = get_trace(
|
2013
2026
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2014
2027
|
)
|
2015
2028
|
_trigger: str = param2template(self.trigger, params, extras=self.extras)
|
2016
|
-
|
2029
|
+
# if _trigger in self.extras.get("stop_circle_workflow_name", []):
|
2030
|
+
# raise StageError(
|
2031
|
+
# "[NESTED]: Circle execution via trigger itself workflow name."
|
2032
|
+
# )
|
2033
|
+
trace.info(f"[NESTED]: Load Workflow Config: {_trigger!r}")
|
2034
|
+
|
2035
|
+
# # NOTE: add noted key for cancel circle execution.
|
2036
|
+
# if "stop_circle_workflow_name" in self.extras:
|
2037
|
+
# self.extras["stop_circle_workflow_name"].append(_trigger)
|
2038
|
+
# else:
|
2039
|
+
# self.extras.update({"stop_circle_workflow_name": [_trigger]})
|
2040
|
+
|
2017
2041
|
result: Result = Workflow.from_conf(
|
2018
2042
|
name=pass_env(_trigger),
|
2019
2043
|
extras=self.extras,
|
@@ -2024,16 +2048,16 @@ class TriggerStage(BaseNestedStage):
|
|
2024
2048
|
event=event,
|
2025
2049
|
)
|
2026
2050
|
if result.status == FAILED:
|
2027
|
-
err_msg:
|
2051
|
+
err_msg: str = (
|
2028
2052
|
f" with:\n{msg}"
|
2029
2053
|
if (msg := result.context.get("errors", {}).get("message"))
|
2030
2054
|
else "."
|
2031
2055
|
)
|
2032
|
-
raise
|
2056
|
+
raise StageNestedError(f"Trigger workflow was failed{err_msg}")
|
2033
2057
|
elif result.status == CANCEL:
|
2034
|
-
raise
|
2058
|
+
raise StageNestedCancelError("Trigger workflow was cancel.")
|
2035
2059
|
elif result.status == SKIP:
|
2036
|
-
raise
|
2060
|
+
raise StageNestedSkipError("Trigger workflow was skipped.")
|
2037
2061
|
return result
|
2038
2062
|
|
2039
2063
|
|
@@ -2098,26 +2122,29 @@ class ParallelStage(BaseNestedStage):
|
|
2098
2122
|
"""Execute branch that will execute all nested-stage that was set in
|
2099
2123
|
this stage with specific branch ID.
|
2100
2124
|
|
2101
|
-
:
|
2102
|
-
|
2103
|
-
|
2104
|
-
|
2105
|
-
|
2106
|
-
|
2107
|
-
|
2108
|
-
|
2125
|
+
Args:
|
2126
|
+
branch (str): A branch ID.
|
2127
|
+
params (DictData): A parameter data.
|
2128
|
+
run_id (str): A running ID.
|
2129
|
+
context (DictData):
|
2130
|
+
parent_run_id (str | None, default None): A parent running ID.
|
2131
|
+
event: (Event) An Event manager instance that use to cancel this
|
2132
|
+
execution if it forces stopped by parent execution.
|
2133
|
+
(Default is None)
|
2109
2134
|
|
2110
|
-
:
|
2111
|
-
|
2112
|
-
|
2113
|
-
|
2135
|
+
Raises:
|
2136
|
+
StageCancelError: If event was set.
|
2137
|
+
StageCancelError: If result from a nested-stage return canceled
|
2138
|
+
status.
|
2139
|
+
StageError: If result from a nested-stage return failed status.
|
2114
2140
|
|
2115
|
-
:
|
2141
|
+
Returns:
|
2142
|
+
tuple[Status, DictData]: A pair of status and result context data.
|
2116
2143
|
"""
|
2117
|
-
trace:
|
2144
|
+
trace: Trace = get_trace(
|
2118
2145
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2119
2146
|
)
|
2120
|
-
trace.debug(f"[
|
2147
|
+
trace.debug(f"[NESTED]: Execute Branch: {branch!r}")
|
2121
2148
|
|
2122
2149
|
# NOTE: Create nested-context
|
2123
2150
|
current_context: DictData = copy.deepcopy(params)
|
@@ -2244,11 +2271,11 @@ class ParallelStage(BaseNestedStage):
|
|
2244
2271
|
Returns:
|
2245
2272
|
Result: The execution result with status and context data.
|
2246
2273
|
"""
|
2247
|
-
trace:
|
2274
|
+
trace: Trace = get_trace(
|
2248
2275
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2249
2276
|
)
|
2250
2277
|
event: Event = event or Event()
|
2251
|
-
trace.info(f"[
|
2278
|
+
trace.info(f"[NESTED]: Parallel with {self.max_workers} workers.")
|
2252
2279
|
catch(
|
2253
2280
|
context=context,
|
2254
2281
|
status=WAIT,
|
@@ -2383,10 +2410,10 @@ class ForEachStage(BaseNestedStage):
|
|
2383
2410
|
|
2384
2411
|
:rtype: tuple[Status, Result]
|
2385
2412
|
"""
|
2386
|
-
trace:
|
2413
|
+
trace: Trace = get_trace(
|
2387
2414
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2388
2415
|
)
|
2389
|
-
trace.debug(f"[
|
2416
|
+
trace.debug(f"[NESTED]: Execute Item: {item!r}")
|
2390
2417
|
key: StrOrInt = index if self.use_index_as_key else item
|
2391
2418
|
|
2392
2419
|
# NOTE: Create nested-context data from the passing context.
|
@@ -2442,7 +2469,7 @@ class ForEachStage(BaseNestedStage):
|
|
2442
2469
|
f"Item execution was break because its nested-stage, "
|
2443
2470
|
f"{stage.iden!r}, failed."
|
2444
2471
|
)
|
2445
|
-
trace.warning(f"[
|
2472
|
+
trace.warning(f"[NESTED]: {error_msg}")
|
2446
2473
|
catch(
|
2447
2474
|
context=context,
|
2448
2475
|
status=FAILED,
|
@@ -2520,7 +2547,7 @@ class ForEachStage(BaseNestedStage):
|
|
2520
2547
|
Returns:
|
2521
2548
|
Result: The execution result with status and context data.
|
2522
2549
|
"""
|
2523
|
-
trace:
|
2550
|
+
trace: Trace = get_trace(
|
2524
2551
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2525
2552
|
)
|
2526
2553
|
event: Event = event or Event()
|
@@ -2552,7 +2579,7 @@ class ForEachStage(BaseNestedStage):
|
|
2552
2579
|
"duplicate item, it should set `use_index_as_key: true`."
|
2553
2580
|
)
|
2554
2581
|
|
2555
|
-
trace.info(f"[
|
2582
|
+
trace.info(f"[NESTED]: Foreach: {foreach!r}.")
|
2556
2583
|
catch(
|
2557
2584
|
context=context,
|
2558
2585
|
status=WAIT,
|
@@ -2586,7 +2613,7 @@ class ForEachStage(BaseNestedStage):
|
|
2586
2613
|
done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
|
2587
2614
|
if len(list(done)) != len(futures):
|
2588
2615
|
trace.warning(
|
2589
|
-
"[
|
2616
|
+
"[NESTED]: Set the event for stop pending for-each stage."
|
2590
2617
|
)
|
2591
2618
|
event.set()
|
2592
2619
|
for future in not_done:
|
@@ -2601,7 +2628,7 @@ class ForEachStage(BaseNestedStage):
|
|
2601
2628
|
if not_done
|
2602
2629
|
else ""
|
2603
2630
|
)
|
2604
|
-
trace.debug(f"[
|
2631
|
+
trace.debug(f"[NESTED]: ... Foreach-Stage set failed event{nd}")
|
2605
2632
|
done: Iterator[Future] = as_completed(futures)
|
2606
2633
|
fail_fast = True
|
2607
2634
|
|
@@ -2705,10 +2732,10 @@ class UntilStage(BaseNestedStage):
|
|
2705
2732
|
:rtype: tuple[Status, DictData, T]
|
2706
2733
|
:return: Return a pair of Result and changed item.
|
2707
2734
|
"""
|
2708
|
-
trace:
|
2735
|
+
trace: Trace = get_trace(
|
2709
2736
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2710
2737
|
)
|
2711
|
-
trace.debug(f"[
|
2738
|
+
trace.debug(f"[NESTED]: Execute Loop: {loop} (Item {item!r})")
|
2712
2739
|
|
2713
2740
|
# NOTE: Create nested-context
|
2714
2741
|
current_context: DictData = copy.deepcopy(params)
|
@@ -2777,11 +2804,11 @@ class UntilStage(BaseNestedStage):
|
|
2777
2804
|
"stages": filter_func(
|
2778
2805
|
nestet_context.pop("stages", {})
|
2779
2806
|
),
|
2780
|
-
"errors":
|
2807
|
+
"errors": StageNestedError(error_msg).to_dict(),
|
2781
2808
|
}
|
2782
2809
|
},
|
2783
2810
|
)
|
2784
|
-
raise
|
2811
|
+
raise StageNestedError(error_msg, refs=loop)
|
2785
2812
|
|
2786
2813
|
elif rs.status == CANCEL:
|
2787
2814
|
error_msg: str = (
|
@@ -2799,11 +2826,13 @@ class UntilStage(BaseNestedStage):
|
|
2799
2826
|
"stages": filter_func(
|
2800
2827
|
nestet_context.pop("stages", {})
|
2801
2828
|
),
|
2802
|
-
"errors":
|
2829
|
+
"errors": StageNestedCancelError(
|
2830
|
+
error_msg
|
2831
|
+
).to_dict(),
|
2803
2832
|
}
|
2804
2833
|
},
|
2805
2834
|
)
|
2806
|
-
raise
|
2835
|
+
raise StageNestedCancelError(error_msg, refs=loop)
|
2807
2836
|
|
2808
2837
|
status: Status = SKIP if sum(skips) == total_stage else SUCCESS
|
2809
2838
|
return (
|
@@ -2847,11 +2876,11 @@ class UntilStage(BaseNestedStage):
|
|
2847
2876
|
Returns:
|
2848
2877
|
Result: The execution result with status and context data.
|
2849
2878
|
"""
|
2850
|
-
trace:
|
2879
|
+
trace: Trace = get_trace(
|
2851
2880
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2852
2881
|
)
|
2853
2882
|
event: Event = event or Event()
|
2854
|
-
trace.info(f"[
|
2883
|
+
trace.info(f"[NESTED]: Until: {self.until!r}")
|
2855
2884
|
item: Union[str, int, bool] = pass_env(
|
2856
2885
|
param2template(self.item, params, extras=self.extras)
|
2857
2886
|
)
|
@@ -2881,7 +2910,7 @@ class UntilStage(BaseNestedStage):
|
|
2881
2910
|
if item is None:
|
2882
2911
|
item: int = loop
|
2883
2912
|
trace.warning(
|
2884
|
-
f"[
|
2913
|
+
f"[NESTED]: Return loop not set the item. It uses loop: "
|
2885
2914
|
f"{loop} by default."
|
2886
2915
|
)
|
2887
2916
|
|
@@ -2999,10 +3028,10 @@ class CaseStage(BaseNestedStage):
|
|
2999
3028
|
|
3000
3029
|
:rtype: DictData
|
3001
3030
|
"""
|
3002
|
-
trace:
|
3031
|
+
trace: Trace = get_trace(
|
3003
3032
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
3004
3033
|
)
|
3005
|
-
trace.debug(f"[
|
3034
|
+
trace.debug(f"[NESTED]: Execute Case: {case!r}")
|
3006
3035
|
current_context: DictData = copy.deepcopy(params)
|
3007
3036
|
current_context.update({"case": case})
|
3008
3037
|
output: DictData = {"case": case, "stages": {}}
|
@@ -3080,23 +3109,27 @@ class CaseStage(BaseNestedStage):
|
|
3080
3109
|
Returns:
|
3081
3110
|
Result: The execution result with status and context data.
|
3082
3111
|
"""
|
3083
|
-
trace:
|
3112
|
+
trace: Trace = get_trace(
|
3084
3113
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
3085
3114
|
)
|
3086
3115
|
|
3087
3116
|
_case: StrOrNone = param2template(self.case, params, extras=self.extras)
|
3117
|
+
trace.info(f"[NESTED]: Get Case: {_case!r}.")
|
3088
3118
|
|
3089
|
-
trace.info(f"[STAGE]: Case: {_case!r}.")
|
3090
3119
|
_else: Optional[Match] = None
|
3091
3120
|
stages: Optional[list[Stage]] = None
|
3121
|
+
|
3122
|
+
# NOTE: Start check the condition of each stage match with this case.
|
3092
3123
|
for match in self.match:
|
3124
|
+
# NOTE: Store the else case.
|
3093
3125
|
if (c := match.case) == "_":
|
3094
3126
|
_else: Match = match
|
3095
3127
|
continue
|
3096
3128
|
|
3097
3129
|
_condition: str = param2template(c, params, extras=self.extras)
|
3098
|
-
if
|
3130
|
+
if pass_env(_case) == pass_env(_condition):
|
3099
3131
|
stages: list[Stage] = match.stages
|
3132
|
+
break
|
3100
3133
|
|
3101
3134
|
if stages is None:
|
3102
3135
|
if _else is None:
|
@@ -3110,6 +3143,7 @@ class CaseStage(BaseNestedStage):
|
|
3110
3143
|
"case and the else condition does not set too."
|
3111
3144
|
)
|
3112
3145
|
|
3146
|
+
# NOTE: Force to use the else when it does not match any case.
|
3113
3147
|
_case: str = "_"
|
3114
3148
|
stages: list[Stage] = _else.stages
|
3115
3149
|
|
@@ -3178,7 +3212,7 @@ class RaiseStage(BaseAsyncStage):
|
|
3178
3212
|
Returns:
|
3179
3213
|
Result: The execution result with status and context data.
|
3180
3214
|
"""
|
3181
|
-
trace:
|
3215
|
+
trace: Trace = get_trace(
|
3182
3216
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
3183
3217
|
)
|
3184
3218
|
message: str = param2template(self.message, params, extras=self.extras)
|
@@ -3209,7 +3243,7 @@ class RaiseStage(BaseAsyncStage):
|
|
3209
3243
|
Returns:
|
3210
3244
|
Result: The execution result with status and context data.
|
3211
3245
|
"""
|
3212
|
-
trace:
|
3246
|
+
trace: Trace = get_trace(
|
3213
3247
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
3214
3248
|
)
|
3215
3249
|
message: str = param2template(self.message, params, extras=self.extras)
|
@@ -3290,7 +3324,7 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
3290
3324
|
"by `pip install docker` first."
|
3291
3325
|
) from None
|
3292
3326
|
|
3293
|
-
trace:
|
3327
|
+
trace: Trace = get_trace(
|
3294
3328
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
3295
3329
|
)
|
3296
3330
|
client = DockerClient(
|
@@ -3390,7 +3424,7 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
3390
3424
|
Returns:
|
3391
3425
|
Result: The execution result with status and context data.
|
3392
3426
|
"""
|
3393
|
-
trace:
|
3427
|
+
trace: Trace = get_trace(
|
3394
3428
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
3395
3429
|
)
|
3396
3430
|
trace.info(f"[STAGE]: Docker: {self.image}:{self.tag}")
|
@@ -3403,8 +3437,11 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
3403
3437
|
"""
|
3404
3438
|
|
3405
3439
|
version: str = Field(
|
3406
|
-
default=
|
3407
|
-
description=
|
3440
|
+
default=__python_version__,
|
3441
|
+
description=(
|
3442
|
+
"A Python version that want to run. It will use supported version "
|
3443
|
+
f"of this package by default, {__python_version__}."
|
3444
|
+
),
|
3408
3445
|
)
|
3409
3446
|
deps: list[str] = Field(
|
3410
3447
|
description=(
|
@@ -3427,11 +3464,12 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
3427
3464
|
The format of Python dependency was followed by the `uv`
|
3428
3465
|
recommended.
|
3429
3466
|
|
3430
|
-
:
|
3431
|
-
|
3432
|
-
|
3433
|
-
|
3434
|
-
|
3467
|
+
Args:
|
3468
|
+
py: A Python string statement.
|
3469
|
+
values: A variable that want to set before running this
|
3470
|
+
deps: An additional Python dependencies that want install before
|
3471
|
+
run this python stage.
|
3472
|
+
run_id: (StrOrNone) A running ID of this stage execution.
|
3435
3473
|
"""
|
3436
3474
|
run_id: str = run_id or uuid.uuid4()
|
3437
3475
|
f_name: str = f"{run_id}.py"
|
@@ -3500,7 +3538,7 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
3500
3538
|
Returns:
|
3501
3539
|
Result: The execution result with status and context data.
|
3502
3540
|
"""
|
3503
|
-
trace:
|
3541
|
+
trace: Trace = get_trace(
|
3504
3542
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
3505
3543
|
)
|
3506
3544
|
run: str = param2template(dedent(self.run), params, extras=self.extras)
|