ddeutil-workflow 0.0.84__py3-none-any.whl → 0.0.85__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +4 -4
- ddeutil/workflow/audits.py +8 -6
- ddeutil/workflow/conf.py +4 -17
- ddeutil/workflow/errors.py +31 -19
- ddeutil/workflow/job.py +276 -156
- ddeutil/workflow/plugins/providers/az.py +2 -2
- ddeutil/workflow/stages.py +404 -314
- ddeutil/workflow/traces.py +125 -185
- ddeutil/workflow/workflow.py +4 -1
- {ddeutil_workflow-0.0.84.dist-info → ddeutil_workflow-0.0.85.dist-info}/METADATA +13 -16
- {ddeutil_workflow-0.0.84.dist-info → ddeutil_workflow-0.0.85.dist-info}/RECORD +16 -16
- {ddeutil_workflow-0.0.84.dist-info → ddeutil_workflow-0.0.85.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.84.dist-info → ddeutil_workflow-0.0.85.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.84.dist-info → ddeutil_workflow-0.0.85.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.84.dist-info → ddeutil_workflow-0.0.85.dist-info}/top_level.txt +0 -0
ddeutil/workflow/stages.py
CHANGED
@@ -138,10 +138,10 @@ class BaseStage(BaseModel, ABC):
|
|
138
138
|
implement, ensuring consistent behavior across different stage types.
|
139
139
|
|
140
140
|
This abstract class handles core stage functionality including:
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
141
|
+
- Stage identification and naming
|
142
|
+
- Conditional execution logic
|
143
|
+
- Output management and templating
|
144
|
+
- Execution lifecycle management
|
145
145
|
|
146
146
|
Custom stages should inherit from this class and implement the abstract
|
147
147
|
`process()` method to define their specific execution behavior.
|
@@ -162,7 +162,6 @@ class BaseStage(BaseModel, ABC):
|
|
162
162
|
...
|
163
163
|
... def process(self, params: DictData, **kwargs) -> Result:
|
164
164
|
... return Result(status=SUCCESS)
|
165
|
-
```
|
166
165
|
"""
|
167
166
|
|
168
167
|
action_stage: ClassVar[bool] = False
|
@@ -240,7 +239,8 @@ class BaseStage(BaseModel, ABC):
|
|
240
239
|
|
241
240
|
Args:
|
242
241
|
value (Any): An any value.
|
243
|
-
params (DictData):
|
242
|
+
params (DictData): A parameter data that want to use in this
|
243
|
+
execution.
|
244
244
|
|
245
245
|
Returns:
|
246
246
|
Any: A templated value.
|
@@ -264,10 +264,11 @@ class BaseStage(BaseModel, ABC):
|
|
264
264
|
params (DictData): A parameter data that want to use in this
|
265
265
|
execution.
|
266
266
|
run_id (str): A running stage ID.
|
267
|
-
context (DictData): A context data
|
268
|
-
|
269
|
-
|
270
|
-
|
267
|
+
context (DictData): A context data that was passed from handler
|
268
|
+
method.
|
269
|
+
parent_run_id (str, default None): A parent running ID.
|
270
|
+
event (Event, default None): An event manager that use to track
|
271
|
+
parent process was not force stopped.
|
271
272
|
|
272
273
|
Returns:
|
273
274
|
Result: The execution result with status and context data.
|
@@ -308,10 +309,10 @@ class BaseStage(BaseModel, ABC):
|
|
308
309
|
object from the current stage ID before release the final result.
|
309
310
|
|
310
311
|
Args:
|
311
|
-
params: A parameter data.
|
312
|
-
run_id: A running
|
313
|
-
event: An event manager that pass to the stage
|
314
|
-
|
312
|
+
params (DictData): A parameter data.
|
313
|
+
run_id (str, default None): A running ID.
|
314
|
+
event (Event, default None): An event manager that pass to the stage
|
315
|
+
execution.
|
315
316
|
|
316
317
|
Returns:
|
317
318
|
Result: The execution result with updated status and context.
|
@@ -370,28 +371,26 @@ class BaseStage(BaseModel, ABC):
|
|
370
371
|
StageNestedError,
|
371
372
|
StageError,
|
372
373
|
) as e: # pragma: no cov
|
374
|
+
updated: Optional[DictData] = {"errors": e.to_dict()}
|
373
375
|
if isinstance(e, StageNestedError):
|
374
|
-
trace.
|
376
|
+
trace.error(f"[STAGE]: Nested: {e}")
|
375
377
|
elif isinstance(e, (StageSkipError, StageNestedSkipError)):
|
376
|
-
trace.
|
377
|
-
|
378
|
-
|
378
|
+
trace.error(f"[STAGE]: ⏭️ Skip: {e}")
|
379
|
+
updated = None
|
380
|
+
elif e.allow_traceback:
|
381
|
+
trace.error(
|
379
382
|
f"[STAGE]: Stage Failed:||🚨 {traceback.format_exc()}||"
|
380
383
|
)
|
384
|
+
else:
|
385
|
+
trace.error(
|
386
|
+
f"[STAGE]: 🤫 Stage Failed with disable traceback:||{e}"
|
387
|
+
)
|
381
388
|
st: Status = get_status_from_error(e)
|
382
389
|
return Result(
|
383
390
|
run_id=run_id,
|
384
391
|
parent_run_id=parent_run_id,
|
385
392
|
status=st,
|
386
|
-
context=catch(
|
387
|
-
context,
|
388
|
-
status=st,
|
389
|
-
updated=(
|
390
|
-
None
|
391
|
-
if isinstance(e, (StageSkipError, StageNestedSkipError))
|
392
|
-
else {"errors": e.to_dict()}
|
393
|
-
),
|
394
|
-
),
|
393
|
+
context=catch(context, status=st, updated=updated),
|
395
394
|
info={"execution_time": time.monotonic() - ts},
|
396
395
|
extras=self.extras,
|
397
396
|
)
|
@@ -409,6 +408,8 @@ class BaseStage(BaseModel, ABC):
|
|
409
408
|
info={"execution_time": time.monotonic() - ts},
|
410
409
|
extras=self.extras,
|
411
410
|
)
|
411
|
+
finally:
|
412
|
+
trace.debug("[STAGE]: End Handler stage execution.")
|
412
413
|
|
413
414
|
def _execute(
|
414
415
|
self,
|
@@ -421,8 +422,10 @@ class BaseStage(BaseModel, ABC):
|
|
421
422
|
"""Wrapped the process method before returning to handler execution.
|
422
423
|
|
423
424
|
Args:
|
424
|
-
params: A parameter data that want to use in this
|
425
|
-
|
425
|
+
params: A parameter data that want to use in this execution.
|
426
|
+
run_id (str):
|
427
|
+
context:
|
428
|
+
parent_run_id:
|
426
429
|
event: An event manager that use to track parent process
|
427
430
|
was not force stopped.
|
428
431
|
|
@@ -643,7 +646,7 @@ class BaseStage(BaseModel, ABC):
|
|
643
646
|
*,
|
644
647
|
parent_run_id: Optional[str] = None,
|
645
648
|
event: Optional[Event] = None,
|
646
|
-
) ->
|
649
|
+
) -> Result:
|
647
650
|
"""Pre-process method that will use to run with dry-run mode, and it
|
648
651
|
should be used replace of process method when workflow release set with
|
649
652
|
DRYRUN mode.
|
@@ -689,6 +692,8 @@ class BaseStage(BaseModel, ABC):
|
|
689
692
|
"""Convert the current Stage model to the EmptyStage model for dry-run
|
690
693
|
mode if the `action_stage` class attribute has set.
|
691
694
|
|
695
|
+
Some use-case for this method is use for deactivate.
|
696
|
+
|
692
697
|
Args:
|
693
698
|
sleep (int, default 0.35): An adjustment sleep time.
|
694
699
|
message (str, default None): A message that want to override default
|
@@ -863,6 +868,8 @@ class BaseAsyncStage(BaseStage, ABC):
|
|
863
868
|
info={"execution_time": time.monotonic() - ts},
|
864
869
|
extras=self.extras,
|
865
870
|
)
|
871
|
+
finally:
|
872
|
+
trace.debug("[STAGE]: End Handler stage process.")
|
866
873
|
|
867
874
|
async def _axecute(
|
868
875
|
self,
|
@@ -954,9 +961,19 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
954
961
|
parent_run_id=parent_run_id,
|
955
962
|
event=event,
|
956
963
|
)
|
964
|
+
except (
|
965
|
+
StageNestedSkipError,
|
966
|
+
StageNestedCancelError,
|
967
|
+
StageSkipError,
|
968
|
+
StageCancelError,
|
969
|
+
):
|
970
|
+
trace.debug("[STAGE]: process raise skip or cancel error.")
|
971
|
+
raise
|
957
972
|
except Exception as e:
|
958
973
|
current_retry += 1
|
959
974
|
exception = e
|
975
|
+
finally:
|
976
|
+
trace.debug("[STAGE]: Failed at the first execution.")
|
960
977
|
|
961
978
|
if self.retry == 0:
|
962
979
|
raise exception
|
@@ -1060,9 +1077,19 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
1060
1077
|
parent_run_id=parent_run_id,
|
1061
1078
|
event=event,
|
1062
1079
|
)
|
1080
|
+
except (
|
1081
|
+
StageNestedSkipError,
|
1082
|
+
StageNestedCancelError,
|
1083
|
+
StageSkipError,
|
1084
|
+
StageCancelError,
|
1085
|
+
):
|
1086
|
+
await trace.adebug("[STAGE]: process raise skip or cancel error.")
|
1087
|
+
raise
|
1063
1088
|
except Exception as e:
|
1064
1089
|
current_retry += 1
|
1065
1090
|
exception = e
|
1091
|
+
finally:
|
1092
|
+
await trace.adebug("[STAGE]: Failed at the first execution.")
|
1066
1093
|
|
1067
1094
|
if self.retry == 0:
|
1068
1095
|
raise exception
|
@@ -1136,22 +1163,13 @@ class EmptyStage(BaseAsyncStage):
|
|
1136
1163
|
for a specified duration, making it useful for workflow timing control
|
1137
1164
|
and debugging scenarios.
|
1138
1165
|
|
1139
|
-
|
1140
|
-
|
1141
|
-
|
1142
|
-
|
1143
|
-
|
1144
|
-
|
1145
|
-
|
1146
|
-
- name: "Debug Parameters"
|
1147
|
-
echo: "Processing file: ${{ params.filename }}"
|
1148
|
-
```
|
1149
|
-
|
1150
|
-
>>> stage = EmptyStage(
|
1151
|
-
... name="Status Update",
|
1152
|
-
... echo="Processing completed successfully",
|
1153
|
-
... sleep=1.0
|
1154
|
-
... )
|
1166
|
+
Examples:
|
1167
|
+
>>> stage = EmptyStage.model_validate({
|
1168
|
+
... "id": "empty-stage",
|
1169
|
+
... "name": "Status Update",
|
1170
|
+
... "echo": "Processing completed successfully",
|
1171
|
+
... "sleep": 1.0,
|
1172
|
+
... })
|
1155
1173
|
"""
|
1156
1174
|
|
1157
1175
|
echo: StrOrNone = Field(
|
@@ -1195,6 +1213,9 @@ class EmptyStage(BaseAsyncStage):
|
|
1195
1213
|
event: An event manager that use to track parent process
|
1196
1214
|
was not force stopped.
|
1197
1215
|
|
1216
|
+
Raises:
|
1217
|
+
StageCancelError: If event was set before start process.
|
1218
|
+
|
1198
1219
|
Returns:
|
1199
1220
|
Result: The execution result with status and context data.
|
1200
1221
|
"""
|
@@ -1246,6 +1267,9 @@ class EmptyStage(BaseAsyncStage):
|
|
1246
1267
|
event: An event manager that use to track parent process
|
1247
1268
|
was not force stopped.
|
1248
1269
|
|
1270
|
+
Raises:
|
1271
|
+
StageCancelError: If event was set before start process.
|
1272
|
+
|
1249
1273
|
Returns:
|
1250
1274
|
Result: The execution result with status and context data.
|
1251
1275
|
"""
|
@@ -1287,14 +1311,15 @@ class BashStage(BaseRetryStage):
|
|
1287
1311
|
statement. Thus, it will write the `.sh` file before start running bash
|
1288
1312
|
command for fix this issue.
|
1289
1313
|
|
1290
|
-
|
1291
|
-
>>> stage = {
|
1314
|
+
Examples:
|
1315
|
+
>>> stage = BaseStage.model_validate({
|
1316
|
+
... "id": "bash-stage",
|
1292
1317
|
... "name": "The Shell stage execution",
|
1293
1318
|
... "bash": 'echo "Hello $FOO"',
|
1294
1319
|
... "env": {
|
1295
1320
|
... "FOO": "BAR",
|
1296
1321
|
... },
|
1297
|
-
... }
|
1322
|
+
... })
|
1298
1323
|
"""
|
1299
1324
|
|
1300
1325
|
action_stage: ClassVar[bool] = True
|
@@ -1312,17 +1337,20 @@ class BashStage(BaseRetryStage):
|
|
1312
1337
|
)
|
1313
1338
|
|
1314
1339
|
@contextlib.asynccontextmanager
|
1315
|
-
async def
|
1340
|
+
async def async_make_sh_file(
|
1316
1341
|
self, bash: str, env: DictStr, run_id: StrOrNone = None
|
1317
1342
|
) -> AsyncIterator[TupleStr]:
|
1318
1343
|
"""Async create and write `.sh` file with the `aiofiles` package.
|
1319
1344
|
|
1320
|
-
:
|
1321
|
-
|
1322
|
-
|
1323
|
-
|
1345
|
+
Args:
|
1346
|
+
bash (str): A bash statement.
|
1347
|
+
env (DictStr): An environment variable that set before run bash.
|
1348
|
+
run_id (StrOrNone, default None): A running stage ID that use for
|
1349
|
+
writing `.sh` file instead generate by UUID4.
|
1324
1350
|
|
1325
|
-
:
|
1351
|
+
Returns:
|
1352
|
+
AsyncIterator[TupleStr]: Return context of prepared bash statement
|
1353
|
+
that want to execute.
|
1326
1354
|
"""
|
1327
1355
|
import aiofiles
|
1328
1356
|
|
@@ -1348,19 +1376,21 @@ class BashStage(BaseRetryStage):
|
|
1348
1376
|
Path(f"./{f_name}").unlink()
|
1349
1377
|
|
1350
1378
|
@contextlib.contextmanager
|
1351
|
-
def
|
1379
|
+
def make_sh_file(
|
1352
1380
|
self, bash: str, env: DictStr, run_id: StrOrNone = None
|
1353
1381
|
) -> Iterator[TupleStr]:
|
1354
1382
|
"""Create and write the `.sh` file before giving this file name to
|
1355
1383
|
context. After that, it will auto delete this file automatic.
|
1356
1384
|
|
1357
|
-
:
|
1358
|
-
|
1359
|
-
|
1360
|
-
|
1385
|
+
Args:
|
1386
|
+
bash (str): A bash statement.
|
1387
|
+
env (DictStr): An environment variable that set before run bash.
|
1388
|
+
run_id (StrOrNone, default None): A running stage ID that use for
|
1389
|
+
writing `.sh` file instead generate by UUID4.
|
1361
1390
|
|
1362
|
-
:
|
1363
|
-
|
1391
|
+
Returns:
|
1392
|
+
Iterator[TupleStr]: Return context of prepared bash statement that
|
1393
|
+
want to execute.
|
1364
1394
|
"""
|
1365
1395
|
f_name: str = f"{run_id or uuid.uuid4()}.sh"
|
1366
1396
|
f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
|
@@ -1402,13 +1432,19 @@ class BashStage(BaseRetryStage):
|
|
1402
1432
|
`return_code`, `stdout`, and `stderr`.
|
1403
1433
|
|
1404
1434
|
Args:
|
1405
|
-
params: A parameter data that want to use in this
|
1435
|
+
params (DictData): A parameter data that want to use in this
|
1406
1436
|
execution.
|
1407
|
-
run_id: A running stage ID.
|
1408
|
-
context: A context data
|
1409
|
-
|
1410
|
-
|
1411
|
-
|
1437
|
+
run_id (str): A running stage ID.
|
1438
|
+
context (DictData): A context data that was passed from handler
|
1439
|
+
method.
|
1440
|
+
parent_run_id (str, default None): A parent running ID.
|
1441
|
+
event (Event, default None): An event manager that use to track
|
1442
|
+
parent process was not force stopped.
|
1443
|
+
|
1444
|
+
Raises:
|
1445
|
+
StageCancelError: If event was set before start process.
|
1446
|
+
StageError: If the return code form subprocess run function gather
|
1447
|
+
than 0.
|
1412
1448
|
|
1413
1449
|
Returns:
|
1414
1450
|
Result: The execution result with status and context data.
|
@@ -1419,12 +1455,16 @@ class BashStage(BaseRetryStage):
|
|
1419
1455
|
bash: str = param2template(
|
1420
1456
|
dedent(self.bash.strip("\n")), params, extras=self.extras
|
1421
1457
|
)
|
1422
|
-
with self.
|
1458
|
+
with self.make_sh_file(
|
1423
1459
|
bash=bash,
|
1424
1460
|
env=param2template(self.env, params, extras=self.extras),
|
1425
1461
|
run_id=run_id,
|
1426
1462
|
) as sh:
|
1427
|
-
|
1463
|
+
|
1464
|
+
if event and event.is_set():
|
1465
|
+
raise StageCancelError("Cancel before start bash process.")
|
1466
|
+
|
1467
|
+
trace.debug(f"[STAGE]: Create `{sh[1]}` file.", module="stage")
|
1428
1468
|
rs: CompletedProcess = subprocess.run(
|
1429
1469
|
sh,
|
1430
1470
|
shell=False,
|
@@ -1466,13 +1506,19 @@ class BashStage(BaseRetryStage):
|
|
1466
1506
|
stdout.
|
1467
1507
|
|
1468
1508
|
Args:
|
1469
|
-
params: A parameter data that want to use in this
|
1509
|
+
params (DictData): A parameter data that want to use in this
|
1470
1510
|
execution.
|
1471
|
-
run_id: A running stage ID.
|
1472
|
-
context: A context data
|
1473
|
-
|
1474
|
-
|
1475
|
-
|
1511
|
+
run_id (str): A running stage ID.
|
1512
|
+
context (DictData): A context data that was passed from handler
|
1513
|
+
method.
|
1514
|
+
parent_run_id (str, default None): A parent running ID.
|
1515
|
+
event (Event, default None): An event manager that use to track
|
1516
|
+
parent process was not force stopped.
|
1517
|
+
|
1518
|
+
Raises:
|
1519
|
+
StageCancelError: If event was set before start process.
|
1520
|
+
StageError: If the return code form subprocess run function gather
|
1521
|
+
than 0.
|
1476
1522
|
|
1477
1523
|
Returns:
|
1478
1524
|
Result: The execution result with status and context data.
|
@@ -1483,11 +1529,15 @@ class BashStage(BaseRetryStage):
|
|
1483
1529
|
bash: str = param2template(
|
1484
1530
|
dedent(self.bash.strip("\n")), params, extras=self.extras
|
1485
1531
|
)
|
1486
|
-
async with self.
|
1532
|
+
async with self.async_make_sh_file(
|
1487
1533
|
bash=bash,
|
1488
1534
|
env=param2template(self.env, params, extras=self.extras),
|
1489
1535
|
run_id=run_id,
|
1490
1536
|
) as sh:
|
1537
|
+
|
1538
|
+
if event and event.is_set():
|
1539
|
+
raise StageCancelError("Cancel before start bash process.")
|
1540
|
+
|
1491
1541
|
await trace.adebug(f"[STAGE]: Create `{sh[1]}` file.")
|
1492
1542
|
rs: CompletedProcess = subprocess.run(
|
1493
1543
|
sh,
|
@@ -1497,7 +1547,6 @@ class BashStage(BaseRetryStage):
|
|
1497
1547
|
text=True,
|
1498
1548
|
encoding="utf-8",
|
1499
1549
|
)
|
1500
|
-
|
1501
1550
|
if rs.returncode > 0:
|
1502
1551
|
e: str = rs.stderr.removesuffix("\n")
|
1503
1552
|
e_bash: str = bash.replace("\n", "\n\t")
|
@@ -1532,14 +1581,15 @@ class PyStage(BaseRetryStage):
|
|
1532
1581
|
module to validate exec-string before running or exclude the `os` package
|
1533
1582
|
from the current globals variable.
|
1534
1583
|
|
1535
|
-
|
1536
|
-
>>> stage = {
|
1584
|
+
Examples:
|
1585
|
+
>>> stage = PyStage.model_validate({
|
1586
|
+
... "id": "py-stage",
|
1537
1587
|
... "name": "Python stage execution",
|
1538
1588
|
... "run": 'print(f"Hello {VARIABLE}")',
|
1539
1589
|
... "vars": {
|
1540
1590
|
... "VARIABLE": "WORLD",
|
1541
1591
|
... },
|
1542
|
-
... }
|
1592
|
+
... })
|
1543
1593
|
"""
|
1544
1594
|
|
1545
1595
|
action_stage: ClassVar[bool] = True
|
@@ -1557,11 +1607,13 @@ class PyStage(BaseRetryStage):
|
|
1557
1607
|
@staticmethod
|
1558
1608
|
def filter_locals(values: DictData) -> Iterator[str]:
|
1559
1609
|
"""Filter a locals mapping values that be module, class, or
|
1560
|
-
__annotations__
|
1610
|
+
`__annotations__`.
|
1561
1611
|
|
1562
|
-
:
|
1612
|
+
Args:
|
1613
|
+
values: (DictData) A locals values that want to filter.
|
1563
1614
|
|
1564
|
-
:
|
1615
|
+
Returns:
|
1616
|
+
Iterator[str]: Iter string value.
|
1565
1617
|
"""
|
1566
1618
|
for value in values:
|
1567
1619
|
|
@@ -1581,12 +1633,14 @@ class PyStage(BaseRetryStage):
|
|
1581
1633
|
"""Override set an outputs method for the Python execution process that
|
1582
1634
|
extract output from all the locals values.
|
1583
1635
|
|
1584
|
-
:
|
1585
|
-
output
|
1586
|
-
|
1587
|
-
|
1636
|
+
Args:
|
1637
|
+
output (DictData): An output data that want to extract to an
|
1638
|
+
output key.
|
1639
|
+
to (DictData): A context data that want to add output result.
|
1640
|
+
info (DictData):
|
1588
1641
|
|
1589
|
-
:
|
1642
|
+
Returns:
|
1643
|
+
DictData: A context data that have merged with the output data.
|
1590
1644
|
"""
|
1591
1645
|
output: DictData = output.copy()
|
1592
1646
|
lc: DictData = output.pop("locals", {})
|
@@ -1638,18 +1692,13 @@ class PyStage(BaseRetryStage):
|
|
1638
1692
|
}
|
1639
1693
|
)
|
1640
1694
|
|
1695
|
+
if event and event.is_set():
|
1696
|
+
raise StageCancelError("Cancel before start exec process.")
|
1697
|
+
|
1641
1698
|
# WARNING: The exec build-in function is very dangerous. So, it
|
1642
1699
|
# should use the re module to validate exec-string before running.
|
1643
|
-
exec(
|
1644
|
-
|
1645
|
-
param2template(dedent(self.run), params, extras=self.extras)
|
1646
|
-
),
|
1647
|
-
gb,
|
1648
|
-
lc,
|
1649
|
-
)
|
1650
|
-
return Result(
|
1651
|
-
run_id=run_id,
|
1652
|
-
parent_run_id=parent_run_id,
|
1700
|
+
exec(self.pass_template(dedent(self.run), params), gb, lc)
|
1701
|
+
return Result.from_trace(trace).catch(
|
1653
1702
|
status=SUCCESS,
|
1654
1703
|
context=catch(
|
1655
1704
|
context=context,
|
@@ -1670,7 +1719,6 @@ class PyStage(BaseRetryStage):
|
|
1670
1719
|
},
|
1671
1720
|
},
|
1672
1721
|
),
|
1673
|
-
extras=self.extras,
|
1674
1722
|
)
|
1675
1723
|
|
1676
1724
|
async def async_process(
|
@@ -1689,13 +1737,17 @@ class PyStage(BaseRetryStage):
|
|
1689
1737
|
- https://stackoverflow.com/questions/44859165/async-exec-in-python
|
1690
1738
|
|
1691
1739
|
Args:
|
1692
|
-
params: A parameter data that want to use in this
|
1740
|
+
params (DictData): A parameter data that want to use in this
|
1693
1741
|
execution.
|
1694
|
-
run_id: A running stage ID.
|
1695
|
-
context: A context data
|
1696
|
-
|
1697
|
-
|
1698
|
-
|
1742
|
+
run_id (str): A running stage ID.
|
1743
|
+
context (DictData): A context data that was passed from handler
|
1744
|
+
method.
|
1745
|
+
parent_run_id (str, default None): A parent running ID.
|
1746
|
+
event (Event, default None): An event manager that use to track
|
1747
|
+
parent process was not force stopped.
|
1748
|
+
|
1749
|
+
Raises:
|
1750
|
+
StageCancelError: If event was set before start process.
|
1699
1751
|
|
1700
1752
|
Returns:
|
1701
1753
|
Result: The execution result with status and context data.
|
@@ -1718,16 +1770,14 @@ class PyStage(BaseRetryStage):
|
|
1718
1770
|
)
|
1719
1771
|
}
|
1720
1772
|
)
|
1773
|
+
|
1774
|
+
if event and event.is_set():
|
1775
|
+
raise StageCancelError("Cancel before start exec process.")
|
1776
|
+
|
1721
1777
|
# WARNING: The exec build-in function is very dangerous. So, it
|
1722
1778
|
# should use the re module to validate exec-string before running.
|
1723
|
-
exec(
|
1724
|
-
|
1725
|
-
gb,
|
1726
|
-
lc,
|
1727
|
-
)
|
1728
|
-
return Result(
|
1729
|
-
run_id=run_id,
|
1730
|
-
parent_run_id=parent_run_id,
|
1779
|
+
exec(self.pass_template(dedent(self.run), params), gb, lc)
|
1780
|
+
return Result.from_trace(trace).catch(
|
1731
1781
|
status=SUCCESS,
|
1732
1782
|
context=catch(
|
1733
1783
|
context=context,
|
@@ -1748,7 +1798,6 @@ class PyStage(BaseRetryStage):
|
|
1748
1798
|
},
|
1749
1799
|
},
|
1750
1800
|
),
|
1751
|
-
extras=self.extras,
|
1752
1801
|
)
|
1753
1802
|
|
1754
1803
|
|
@@ -1773,12 +1822,13 @@ class CallStage(BaseRetryStage):
|
|
1773
1822
|
The caller registry to get a caller function should importable by the
|
1774
1823
|
current Python execution pointer.
|
1775
1824
|
|
1776
|
-
|
1777
|
-
>>> stage = {
|
1825
|
+
Examples:
|
1826
|
+
>>> stage = CallStage.model_validate({
|
1827
|
+
... "id": "call-stage",
|
1778
1828
|
... "name": "Task stage execution",
|
1779
1829
|
... "uses": "tasks/function-name@tag-name",
|
1780
1830
|
... "args": {"arg01": "BAR", "kwarg01": 10},
|
1781
|
-
... }
|
1831
|
+
... })
|
1782
1832
|
"""
|
1783
1833
|
|
1784
1834
|
action_stage: ClassVar[bool] = True
|
@@ -1798,26 +1848,36 @@ class CallStage(BaseRetryStage):
|
|
1798
1848
|
)
|
1799
1849
|
|
1800
1850
|
@field_validator("args", mode="before")
|
1801
|
-
def __validate_args_key(cls,
|
1851
|
+
def __validate_args_key(cls, data: Any) -> Any:
|
1802
1852
|
"""Validate argument keys on the ``args`` field should not include the
|
1803
1853
|
special keys.
|
1804
1854
|
|
1805
|
-
:
|
1855
|
+
Args:
|
1856
|
+
data (Any): A data that want to check the special keys.
|
1806
1857
|
|
1807
|
-
:
|
1858
|
+
Returns:
|
1859
|
+
Any: An any data.
|
1808
1860
|
"""
|
1809
|
-
if isinstance(
|
1810
|
-
k in
|
1861
|
+
if isinstance(data, dict) and any(
|
1862
|
+
k in data for k in ("result", "extras")
|
1811
1863
|
):
|
1812
1864
|
raise ValueError(
|
1813
1865
|
"The argument on workflow template for the caller stage "
|
1814
1866
|
"should not pass `result` and `extras`. They are special "
|
1815
1867
|
"arguments."
|
1816
1868
|
)
|
1817
|
-
return
|
1869
|
+
return data
|
1818
1870
|
|
1819
1871
|
def get_caller(self, params: DictData) -> Callable[[], TagFunc]:
|
1820
|
-
"""Get the lazy TagFuc object from registry.
|
1872
|
+
"""Get the lazy TagFuc object from registry.
|
1873
|
+
|
1874
|
+
Args:
|
1875
|
+
params (DictData): A parameters.
|
1876
|
+
|
1877
|
+
Returns:
|
1878
|
+
Callable[[], TagFunc]: A lazy partial function that return the
|
1879
|
+
TagFunc object.
|
1880
|
+
"""
|
1821
1881
|
return extract_call(
|
1822
1882
|
param2template(self.uses, params, extras=self.extras),
|
1823
1883
|
registries=self.extras.get("registry_caller"),
|
@@ -1835,13 +1895,19 @@ class CallStage(BaseRetryStage):
|
|
1835
1895
|
"""Execute this caller function with its argument parameter.
|
1836
1896
|
|
1837
1897
|
Args:
|
1838
|
-
params: A parameter data that want to use in this
|
1898
|
+
params (DictData): A parameter data that want to use in this
|
1839
1899
|
execution.
|
1840
|
-
run_id: A running stage ID.
|
1841
|
-
context: A context data
|
1842
|
-
|
1843
|
-
|
1844
|
-
|
1900
|
+
run_id (str): A running stage ID.
|
1901
|
+
context (DictData): A context data that was passed from handler
|
1902
|
+
method.
|
1903
|
+
parent_run_id (str, default None): A parent running ID.
|
1904
|
+
event (Event, default None): An event manager that use to track
|
1905
|
+
parent process was not force stopped.
|
1906
|
+
|
1907
|
+
Raises:
|
1908
|
+
ValueError: If the necessary parameters do not exist in args field.
|
1909
|
+
TypeError: If the returning type of caller function does not match
|
1910
|
+
with dict type.
|
1845
1911
|
|
1846
1912
|
Returns:
|
1847
1913
|
Result: The execution result with status and context data.
|
@@ -1864,7 +1930,9 @@ class CallStage(BaseRetryStage):
|
|
1864
1930
|
),
|
1865
1931
|
"extras": self.extras,
|
1866
1932
|
} | self.pass_template(self.args, params)
|
1867
|
-
|
1933
|
+
|
1934
|
+
# NOTE: Catch the necessary parameters.
|
1935
|
+
sig: inspect.Signature = inspect.signature(call_func)
|
1868
1936
|
necessary_params: list[str] = []
|
1869
1937
|
has_keyword: bool = False
|
1870
1938
|
for k in sig.parameters:
|
@@ -1883,11 +1951,9 @@ class CallStage(BaseRetryStage):
|
|
1883
1951
|
(k.removeprefix("_") not in args and k not in args)
|
1884
1952
|
for k in necessary_params
|
1885
1953
|
):
|
1886
|
-
|
1887
|
-
necessary_params
|
1888
|
-
|
1889
|
-
if "extras" in necessary_params:
|
1890
|
-
necessary_params.remove("extras")
|
1954
|
+
for k in ("result", "extras"):
|
1955
|
+
if k in necessary_params:
|
1956
|
+
necessary_params.remove(k)
|
1891
1957
|
|
1892
1958
|
args.pop("result")
|
1893
1959
|
args.pop("extras")
|
@@ -1897,18 +1963,15 @@ class CallStage(BaseRetryStage):
|
|
1897
1963
|
)
|
1898
1964
|
|
1899
1965
|
if not has_keyword:
|
1900
|
-
|
1901
|
-
|
1966
|
+
for k in ("result", "extras"):
|
1967
|
+
if k not in sig.parameters:
|
1968
|
+
args.pop(k)
|
1902
1969
|
|
1903
|
-
|
1904
|
-
args.pop("extras")
|
1970
|
+
args: DictData = self.validate_model_args(call_func, args)
|
1905
1971
|
|
1906
1972
|
if event and event.is_set():
|
1907
1973
|
raise StageCancelError("Cancel before start call process.")
|
1908
1974
|
|
1909
|
-
args: DictData = self.validate_model_args(
|
1910
|
-
call_func, args, run_id, parent_run_id, extras=self.extras
|
1911
|
-
)
|
1912
1975
|
if inspect.iscoroutinefunction(call_func):
|
1913
1976
|
loop = asyncio.get_event_loop()
|
1914
1977
|
rs: DictData = loop.run_until_complete(
|
@@ -1962,6 +2025,11 @@ class CallStage(BaseRetryStage):
|
|
1962
2025
|
event: An event manager that use to track parent process
|
1963
2026
|
was not force stopped.
|
1964
2027
|
|
2028
|
+
Raises:
|
2029
|
+
ValueError: If the necessary parameters do not exist in args field.
|
2030
|
+
TypeError: If the returning type of caller function does not match
|
2031
|
+
with dict type.
|
2032
|
+
|
1965
2033
|
Returns:
|
1966
2034
|
Result: The execution result with status and context data.
|
1967
2035
|
"""
|
@@ -1985,7 +2053,9 @@ class CallStage(BaseRetryStage):
|
|
1985
2053
|
),
|
1986
2054
|
"extras": self.extras,
|
1987
2055
|
} | self.pass_template(self.args, params)
|
1988
|
-
|
2056
|
+
|
2057
|
+
# NOTE: Catch the necessary parameters.
|
2058
|
+
sig: inspect.Signature = inspect.signature(call_func)
|
1989
2059
|
necessary_params: list[str] = []
|
1990
2060
|
has_keyword: bool = False
|
1991
2061
|
for k in sig.parameters:
|
@@ -2003,11 +2073,9 @@ class CallStage(BaseRetryStage):
|
|
2003
2073
|
(k.removeprefix("_") not in args and k not in args)
|
2004
2074
|
for k in necessary_params
|
2005
2075
|
):
|
2006
|
-
|
2007
|
-
necessary_params
|
2008
|
-
|
2009
|
-
if "extras" in necessary_params:
|
2010
|
-
necessary_params.remove("extras")
|
2076
|
+
for k in ("result", "extras"):
|
2077
|
+
if k in necessary_params:
|
2078
|
+
necessary_params.remove(k)
|
2011
2079
|
|
2012
2080
|
args.pop("result")
|
2013
2081
|
args.pop("extras")
|
@@ -2017,18 +2085,15 @@ class CallStage(BaseRetryStage):
|
|
2017
2085
|
)
|
2018
2086
|
|
2019
2087
|
if not has_keyword:
|
2020
|
-
|
2021
|
-
|
2088
|
+
for k in ("result", "extras"):
|
2089
|
+
if k not in sig.parameters:
|
2090
|
+
args.pop(k)
|
2022
2091
|
|
2023
|
-
|
2024
|
-
args.pop("extras")
|
2092
|
+
args: DictData = self.validate_model_args(call_func, args)
|
2025
2093
|
|
2026
2094
|
if event and event.is_set():
|
2027
2095
|
raise StageCancelError("Cancel before start call process.")
|
2028
2096
|
|
2029
|
-
args: DictData = self.validate_model_args(
|
2030
|
-
call_func, args, run_id, parent_run_id, extras=self.extras
|
2031
|
-
)
|
2032
2097
|
if inspect.iscoroutinefunction(call_func):
|
2033
2098
|
rs: DictOrModel = await call_func(
|
2034
2099
|
**param2template(args, params, extras=self.extras)
|
@@ -2061,24 +2126,18 @@ class CallStage(BaseRetryStage):
|
|
2061
2126
|
)
|
2062
2127
|
|
2063
2128
|
@staticmethod
|
2064
|
-
def validate_model_args(
|
2065
|
-
func: TagFunc,
|
2066
|
-
args: DictData,
|
2067
|
-
run_id: str,
|
2068
|
-
parent_run_id: Optional[str] = None,
|
2069
|
-
extras: Optional[DictData] = None,
|
2070
|
-
) -> DictData:
|
2129
|
+
def validate_model_args(func: TagFunc, args: DictData) -> DictData:
|
2071
2130
|
"""Validate an input arguments before passing to the caller function.
|
2072
2131
|
|
2073
2132
|
Args:
|
2074
2133
|
func (TagFunc): A tag function object that want to get typing.
|
2075
2134
|
args (DictData): An arguments before passing to this tag func.
|
2076
|
-
|
2077
|
-
|
2078
|
-
|
2135
|
+
|
2136
|
+
Raises:
|
2137
|
+
StageError: If model validation was raised the ValidationError.
|
2079
2138
|
|
2080
2139
|
Returns:
|
2081
|
-
DictData: A prepared args
|
2140
|
+
DictData: A prepared args parameter that validate with model args.
|
2082
2141
|
"""
|
2083
2142
|
try:
|
2084
2143
|
override: DictData = dict(
|
@@ -2101,15 +2160,6 @@ class CallStage(BaseRetryStage):
|
|
2101
2160
|
raise StageError(
|
2102
2161
|
"Validate argument from the caller function raise invalid type."
|
2103
2162
|
) from e
|
2104
|
-
except TypeError as e:
|
2105
|
-
trace: Trace = get_trace(
|
2106
|
-
run_id, parent_run_id=parent_run_id, extras=extras
|
2107
|
-
)
|
2108
|
-
trace.warning(
|
2109
|
-
f"[STAGE]: Get type hint raise TypeError: {e}, so, it skip "
|
2110
|
-
f"parsing model args process."
|
2111
|
-
)
|
2112
|
-
return args
|
2113
2163
|
|
2114
2164
|
def dryrun(
|
2115
2165
|
self,
|
@@ -2119,12 +2169,23 @@ class CallStage(BaseRetryStage):
|
|
2119
2169
|
*,
|
2120
2170
|
parent_run_id: Optional[str] = None,
|
2121
2171
|
event: Optional[Event] = None,
|
2122
|
-
) ->
|
2172
|
+
) -> Result: # pragma: no cov
|
2123
2173
|
"""Override the dryrun method for this CallStage.
|
2124
2174
|
|
2125
2175
|
Steps:
|
2126
2176
|
- Pre-hook caller function that exist.
|
2127
2177
|
- Show function parameters
|
2178
|
+
|
2179
|
+
Args:
|
2180
|
+
params (DictData): A parameter data that want to use in this
|
2181
|
+
execution.
|
2182
|
+
run_id (str): A running stage ID.
|
2183
|
+
context (DictData): A context data that was passed from handler
|
2184
|
+
method.
|
2185
|
+
parent_run_id (str, default None): A parent running ID.
|
2186
|
+
event (Event, default None): An event manager that use to track
|
2187
|
+
parent process was not force stopped.
|
2188
|
+
|
2128
2189
|
"""
|
2129
2190
|
trace: Trace = get_trace(
|
2130
2191
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
@@ -2142,7 +2203,9 @@ class CallStage(BaseRetryStage):
|
|
2142
2203
|
),
|
2143
2204
|
"extras": self.extras,
|
2144
2205
|
} | self.pass_template(self.args, params)
|
2145
|
-
|
2206
|
+
|
2207
|
+
# NOTE: Catch the necessary parameters.
|
2208
|
+
sig: inspect.Signature = inspect.signature(call_func)
|
2146
2209
|
trace.debug(f"[STAGE]: {sig.parameters}")
|
2147
2210
|
necessary_params: list[str] = []
|
2148
2211
|
has_keyword: bool = False
|
@@ -2164,24 +2227,22 @@ class CallStage(BaseRetryStage):
|
|
2164
2227
|
if p in func_typed
|
2165
2228
|
)
|
2166
2229
|
map_type_args: str = "||".join(f"\t{a}: {type(a)}" for a in args)
|
2167
|
-
if not has_keyword:
|
2168
|
-
if "result" not in sig.parameters:
|
2169
|
-
args.pop("result")
|
2170
2230
|
|
2171
|
-
|
2172
|
-
|
2231
|
+
if not has_keyword:
|
2232
|
+
for k in ("result", "extras"):
|
2233
|
+
if k not in sig.parameters:
|
2234
|
+
args.pop(k)
|
2173
2235
|
|
2174
|
-
trace.
|
2236
|
+
trace.info(
|
2175
2237
|
f"[STAGE]: Details"
|
2176
2238
|
f"||Necessary Params:"
|
2177
2239
|
f"||{map_type}"
|
2240
|
+
f"||Supported Keyword Params: {has_keyword}"
|
2178
2241
|
f"||Return Type: {func_typed['return']}"
|
2179
2242
|
f"||Argument Params:"
|
2180
2243
|
f"||{map_type_args}"
|
2181
2244
|
f"||"
|
2182
2245
|
)
|
2183
|
-
if has_keyword:
|
2184
|
-
trace.debug("[STAGE]: This caller function support keyword param.")
|
2185
2246
|
return Result(
|
2186
2247
|
run_id=run_id,
|
2187
2248
|
parent_run_id=parent_run_id,
|
@@ -2191,71 +2252,6 @@ class CallStage(BaseRetryStage):
|
|
2191
2252
|
)
|
2192
2253
|
|
2193
2254
|
|
2194
|
-
class BaseNestedStage(BaseAsyncStage, ABC):
|
2195
|
-
"""Base Nested Stage model. This model is use for checking the child stage
|
2196
|
-
is the nested stage or not.
|
2197
|
-
"""
|
2198
|
-
|
2199
|
-
def set_outputs(
|
2200
|
-
self, output: DictData, to: DictData, info: Optional[DictData] = None
|
2201
|
-
) -> DictData:
|
2202
|
-
"""Override the set outputs method that support for nested-stage."""
|
2203
|
-
return super().set_outputs(output, to=to)
|
2204
|
-
|
2205
|
-
def get_outputs(self, output: DictData) -> DictData:
|
2206
|
-
"""Override the get outputs method that support for nested-stage"""
|
2207
|
-
return super().get_outputs(output)
|
2208
|
-
|
2209
|
-
@property
|
2210
|
-
def is_nested(self) -> bool:
|
2211
|
-
"""Check if this stage is a nested stage or not.
|
2212
|
-
|
2213
|
-
:rtype: bool
|
2214
|
-
"""
|
2215
|
-
return True
|
2216
|
-
|
2217
|
-
@staticmethod
|
2218
|
-
def mark_errors(context: DictData, error: StageError) -> None:
|
2219
|
-
"""Make the errors context result with the refs value depends on the nested
|
2220
|
-
execute func.
|
2221
|
-
|
2222
|
-
Args:
|
2223
|
-
context: (DictData) A context data.
|
2224
|
-
error: (StageError) A stage exception object.
|
2225
|
-
"""
|
2226
|
-
if "errors" in context:
|
2227
|
-
context["errors"][error.refs] = error.to_dict()
|
2228
|
-
else:
|
2229
|
-
context["errors"] = error.to_dict(with_refs=True)
|
2230
|
-
|
2231
|
-
async def async_process(
|
2232
|
-
self,
|
2233
|
-
params: DictData,
|
2234
|
-
run_id: str,
|
2235
|
-
context: DictData,
|
2236
|
-
*,
|
2237
|
-
parent_run_id: Optional[str] = None,
|
2238
|
-
event: Optional[Event] = None,
|
2239
|
-
) -> Result:
|
2240
|
-
"""Async process for nested-stage do not implement yet.
|
2241
|
-
|
2242
|
-
Args:
|
2243
|
-
params: A parameter data that want to use in this
|
2244
|
-
execution.
|
2245
|
-
run_id: A running stage ID.
|
2246
|
-
context: A context data.
|
2247
|
-
parent_run_id: A parent running ID. (Default is None)
|
2248
|
-
event: An event manager that use to track parent process
|
2249
|
-
was not force stopped.
|
2250
|
-
|
2251
|
-
Returns:
|
2252
|
-
Result: The execution result with status and context data.
|
2253
|
-
"""
|
2254
|
-
raise NotImplementedError(
|
2255
|
-
"The nested-stage does not implement the `axecute` method yet."
|
2256
|
-
)
|
2257
|
-
|
2258
|
-
|
2259
2255
|
class TriggerStage(BaseRetryStage):
|
2260
2256
|
"""Trigger workflow executor stage that run an input trigger Workflow
|
2261
2257
|
execute method. This is the stage that allow you to create the reusable
|
@@ -2264,12 +2260,13 @@ class TriggerStage(BaseRetryStage):
|
|
2264
2260
|
This stage does not allow to pass the workflow model directly to the
|
2265
2261
|
trigger field. A trigger workflow name should exist on the config path only.
|
2266
2262
|
|
2267
|
-
|
2268
|
-
>>> stage = {
|
2263
|
+
Examples:
|
2264
|
+
>>> stage = TriggerStage.model_validate({
|
2265
|
+
... "id": "trigger-stage",
|
2269
2266
|
... "name": "Trigger workflow stage execution",
|
2270
2267
|
... "trigger": 'workflow-name-for-loader',
|
2271
2268
|
... "params": {"run-date": "2024-08-01", "source": "src"},
|
2272
|
-
... }
|
2269
|
+
... })
|
2273
2270
|
"""
|
2274
2271
|
|
2275
2272
|
trigger: str = Field(
|
@@ -2316,51 +2313,38 @@ class TriggerStage(BaseRetryStage):
|
|
2316
2313
|
_trigger: str = param2template(self.trigger, params, extras=self.extras)
|
2317
2314
|
if _trigger == self.extras.get("__sys_exec_break_circle", "NOTSET"):
|
2318
2315
|
raise StageError("Circle execute via trigger itself workflow name.")
|
2316
|
+
|
2319
2317
|
trace.info(f"[NESTED]: Load Workflow Config: {_trigger!r}")
|
2320
|
-
|
2318
|
+
workflow: Workflow = Workflow.from_conf(
|
2321
2319
|
name=pass_env(_trigger),
|
2322
2320
|
extras=self.extras,
|
2323
|
-
)
|
2324
|
-
|
2321
|
+
)
|
2322
|
+
|
2323
|
+
if event and event.is_set():
|
2324
|
+
raise StageCancelError("Cancel before start trigger process.")
|
2325
|
+
|
2326
|
+
# IMPORTANT: Should not use the `pass_env` function on this `params`
|
2327
|
+
# parameter.
|
2328
|
+
result: Result = workflow.execute(
|
2325
2329
|
params=param2template(self.params, params, extras=self.extras),
|
2326
2330
|
run_id=parent_run_id,
|
2327
2331
|
event=event,
|
2328
2332
|
)
|
2333
|
+
catch(context, status=result.status, updated=result.context)
|
2329
2334
|
if result.status == FAILED:
|
2330
2335
|
err_msg: str = (
|
2331
2336
|
f" with:\n{msg}"
|
2332
2337
|
if (msg := result.context.get("errors", {}).get("message"))
|
2333
2338
|
else "."
|
2334
2339
|
)
|
2335
|
-
|
2336
|
-
|
2337
|
-
context={
|
2338
|
-
"status": FAILED,
|
2339
|
-
"errors": StageError(
|
2340
|
-
f"Trigger workflow was failed{err_msg}"
|
2341
|
-
).to_dict(),
|
2342
|
-
},
|
2340
|
+
err = StageError(
|
2341
|
+
f"Trigger workflow was failed{err_msg}", allow_traceback=False
|
2343
2342
|
)
|
2343
|
+
raise err
|
2344
2344
|
elif result.status == CANCEL:
|
2345
|
-
|
2346
|
-
status=CANCEL,
|
2347
|
-
context={
|
2348
|
-
"status": CANCEL,
|
2349
|
-
"errors": StageCancelError(
|
2350
|
-
"Trigger workflow was cancel."
|
2351
|
-
).to_dict(),
|
2352
|
-
},
|
2353
|
-
)
|
2345
|
+
raise StageCancelError("Trigger workflow was cancel.")
|
2354
2346
|
elif result.status == SKIP:
|
2355
|
-
|
2356
|
-
status=SKIP,
|
2357
|
-
context={
|
2358
|
-
"status": SKIP,
|
2359
|
-
"errors": StageSkipError(
|
2360
|
-
"Trigger workflow was skipped."
|
2361
|
-
).to_dict(),
|
2362
|
-
},
|
2363
|
-
)
|
2347
|
+
raise StageSkipError("Trigger workflow was skipped.")
|
2364
2348
|
return result
|
2365
2349
|
|
2366
2350
|
async def async_process(
|
@@ -2372,7 +2356,7 @@ class TriggerStage(BaseRetryStage):
|
|
2372
2356
|
parent_run_id: Optional[str] = None,
|
2373
2357
|
event: Optional[Event] = None,
|
2374
2358
|
) -> Result: # pragma: no cov
|
2375
|
-
"""Async process for
|
2359
|
+
"""Async process for trigger-stage do not implement yet.
|
2376
2360
|
|
2377
2361
|
Args:
|
2378
2362
|
params: A parameter data that want to use in this
|
@@ -2391,6 +2375,72 @@ class TriggerStage(BaseRetryStage):
|
|
2391
2375
|
)
|
2392
2376
|
|
2393
2377
|
|
2378
|
+
class BaseNestedStage(BaseAsyncStage, ABC):
|
2379
|
+
"""Base Nested Stage model. This model is use for checking the child stage
|
2380
|
+
is the nested stage or not.
|
2381
|
+
"""
|
2382
|
+
|
2383
|
+
def set_outputs(
|
2384
|
+
self, output: DictData, to: DictData, info: Optional[DictData] = None
|
2385
|
+
) -> DictData:
|
2386
|
+
"""Override the set outputs method that support for nested-stage."""
|
2387
|
+
return super().set_outputs(output, to=to)
|
2388
|
+
|
2389
|
+
def get_outputs(self, output: DictData) -> DictData:
|
2390
|
+
"""Override the get outputs method that support for nested-stage"""
|
2391
|
+
return super().get_outputs(output)
|
2392
|
+
|
2393
|
+
@property
|
2394
|
+
def is_nested(self) -> bool:
|
2395
|
+
"""Check if this stage is a nested stage or not.
|
2396
|
+
|
2397
|
+
Returns:
|
2398
|
+
bool: True only.
|
2399
|
+
"""
|
2400
|
+
return True
|
2401
|
+
|
2402
|
+
@staticmethod
|
2403
|
+
def mark_errors(context: DictData, error: StageError) -> None:
|
2404
|
+
"""Make the errors context result with the refs value depends on the nested
|
2405
|
+
execute func.
|
2406
|
+
|
2407
|
+
Args:
|
2408
|
+
context: (DictData) A context data.
|
2409
|
+
error: (StageError) A stage exception object.
|
2410
|
+
"""
|
2411
|
+
if "errors" in context:
|
2412
|
+
context["errors"][error.refs] = error.to_dict()
|
2413
|
+
else:
|
2414
|
+
context["errors"] = error.to_dict(with_refs=True)
|
2415
|
+
|
2416
|
+
async def async_process(
|
2417
|
+
self,
|
2418
|
+
params: DictData,
|
2419
|
+
run_id: str,
|
2420
|
+
context: DictData,
|
2421
|
+
*,
|
2422
|
+
parent_run_id: Optional[str] = None,
|
2423
|
+
event: Optional[Event] = None,
|
2424
|
+
) -> Result:
|
2425
|
+
"""Async process for nested-stage do not implement yet.
|
2426
|
+
|
2427
|
+
Args:
|
2428
|
+
params: A parameter data that want to use in this
|
2429
|
+
execution.
|
2430
|
+
run_id: A running stage ID.
|
2431
|
+
context: A context data.
|
2432
|
+
parent_run_id: A parent running ID. (Default is None)
|
2433
|
+
event: An event manager that use to track parent process
|
2434
|
+
was not force stopped.
|
2435
|
+
|
2436
|
+
Returns:
|
2437
|
+
Result: The execution result with status and context data.
|
2438
|
+
"""
|
2439
|
+
raise NotImplementedError(
|
2440
|
+
"The nested-stage does not implement the `axecute` method yet."
|
2441
|
+
)
|
2442
|
+
|
2443
|
+
|
2394
2444
|
class ParallelContext(TypedDict):
|
2395
2445
|
branch: str
|
2396
2446
|
stages: NotRequired[dict[str, Any]]
|
@@ -2404,8 +2454,9 @@ class ParallelStage(BaseNestedStage):
|
|
2404
2454
|
This stage is not the low-level stage model because it runs multi-stages
|
2405
2455
|
in this stage execution.
|
2406
2456
|
|
2407
|
-
|
2408
|
-
>>> stage = {
|
2457
|
+
Examples:
|
2458
|
+
>>> stage = ParallelStage.model_validate({
|
2459
|
+
... "id": "parallel-stage",
|
2409
2460
|
... "name": "Parallel stage execution.",
|
2410
2461
|
... "parallel": {
|
2411
2462
|
... "branch01": [
|
@@ -2427,7 +2478,7 @@ class ParallelStage(BaseNestedStage):
|
|
2427
2478
|
... },
|
2428
2479
|
... ],
|
2429
2480
|
... }
|
2430
|
-
... }
|
2481
|
+
... })
|
2431
2482
|
"""
|
2432
2483
|
|
2433
2484
|
parallel: dict[str, list[Stage]] = Field(
|
@@ -2677,8 +2728,9 @@ class ForEachStage(BaseNestedStage):
|
|
2677
2728
|
This stage is not the low-level stage model because it runs
|
2678
2729
|
multi-stages in this stage execution.
|
2679
2730
|
|
2680
|
-
|
2681
|
-
>>> stage = {
|
2731
|
+
Examples:
|
2732
|
+
>>> stage = ForEachStage.model_validate({
|
2733
|
+
... "id": "foreach-stage",
|
2682
2734
|
... "name": "For-each stage execution",
|
2683
2735
|
... "foreach": [1, 2, 3]
|
2684
2736
|
... "stages": [
|
@@ -2687,7 +2739,7 @@ class ForEachStage(BaseNestedStage):
|
|
2687
2739
|
... "echo": "Start run with item ${{ item }}"
|
2688
2740
|
... },
|
2689
2741
|
... ],
|
2690
|
-
... }
|
2742
|
+
... })
|
2691
2743
|
"""
|
2692
2744
|
|
2693
2745
|
foreach: EachType = Field(
|
@@ -2862,15 +2914,18 @@ class ForEachStage(BaseNestedStage):
|
|
2862
2914
|
"""Validate foreach value that already passed to this model.
|
2863
2915
|
|
2864
2916
|
Args:
|
2865
|
-
value:
|
2917
|
+
value (Any): An any foreach value.
|
2866
2918
|
|
2867
2919
|
Raises:
|
2868
2920
|
TypeError: If value can not try-convert to list type.
|
2869
|
-
ValueError:
|
2921
|
+
ValueError: If the foreach value is dict type.
|
2922
|
+
ValueError: If the foreach value contain duplication item without
|
2923
|
+
enable using index as key flag.
|
2870
2924
|
|
2871
2925
|
Returns:
|
2872
2926
|
list[Any]: list of item.
|
2873
2927
|
"""
|
2928
|
+
# NOTE: Try to cast a foreach with string type to list of items.
|
2874
2929
|
if isinstance(value, str):
|
2875
2930
|
try:
|
2876
2931
|
value: list[Any] = str2list(value)
|
@@ -2879,6 +2934,7 @@ class ForEachStage(BaseNestedStage):
|
|
2879
2934
|
f"Does not support string foreach: {value!r} that can "
|
2880
2935
|
f"not convert to list."
|
2881
2936
|
) from e
|
2937
|
+
|
2882
2938
|
# [VALIDATE]: Type of the foreach should be `list` type.
|
2883
2939
|
elif isinstance(value, dict):
|
2884
2940
|
raise TypeError(
|
@@ -3000,8 +3056,9 @@ class UntilStage(BaseNestedStage):
|
|
3000
3056
|
This stage is not the low-level stage model because it runs
|
3001
3057
|
multi-stages in this stage execution.
|
3002
3058
|
|
3003
|
-
|
3004
|
-
>>> stage = {
|
3059
|
+
Examples:
|
3060
|
+
>>> stage = UntilStage.model_validate({
|
3061
|
+
... "id": "until-stage",
|
3005
3062
|
... "name": "Until stage execution",
|
3006
3063
|
... "item": 1,
|
3007
3064
|
... "until": "${{ item }} > 3"
|
@@ -3014,7 +3071,7 @@ class UntilStage(BaseNestedStage):
|
|
3014
3071
|
... )
|
3015
3072
|
... },
|
3016
3073
|
... ],
|
3017
|
-
... }
|
3074
|
+
... })
|
3018
3075
|
"""
|
3019
3076
|
|
3020
3077
|
item: Union[str, int, bool] = Field(
|
@@ -3285,6 +3342,8 @@ class Match(BaseModel):
|
|
3285
3342
|
|
3286
3343
|
|
3287
3344
|
class Else(BaseModel):
|
3345
|
+
"""Else model for the Case Stage."""
|
3346
|
+
|
3288
3347
|
other: list[Stage] = Field(
|
3289
3348
|
description="A list of stage that does not match any case.",
|
3290
3349
|
alias="else",
|
@@ -3294,8 +3353,9 @@ class Else(BaseModel):
|
|
3294
3353
|
class CaseStage(BaseNestedStage):
|
3295
3354
|
"""Case stage executor that execute all stages if the condition was matched.
|
3296
3355
|
|
3297
|
-
|
3298
|
-
>>> stage = {
|
3356
|
+
Examples:
|
3357
|
+
>>> stage = CaseStage.model_validate({
|
3358
|
+
... "id": "case-stage",
|
3299
3359
|
... "name": "If stage execution.",
|
3300
3360
|
... "case": "${{ param.test }}",
|
3301
3361
|
... "match": [
|
@@ -3318,9 +3378,10 @@ class CaseStage(BaseNestedStage):
|
|
3318
3378
|
... ],
|
3319
3379
|
... },
|
3320
3380
|
... ],
|
3321
|
-
... }
|
3381
|
+
... })
|
3322
3382
|
|
3323
|
-
>>> stage = {
|
3383
|
+
>>> stage = CaseStage.model_validate({
|
3384
|
+
... "id": "case-stage",
|
3324
3385
|
... "name": "If stage execution.",
|
3325
3386
|
... "case": "${{ param.test }}",
|
3326
3387
|
... "match": [
|
@@ -3342,7 +3403,7 @@ class CaseStage(BaseNestedStage):
|
|
3342
3403
|
... ],
|
3343
3404
|
... },
|
3344
3405
|
... ],
|
3345
|
-
... }
|
3406
|
+
... })
|
3346
3407
|
|
3347
3408
|
"""
|
3348
3409
|
|
@@ -3361,9 +3422,16 @@ class CaseStage(BaseNestedStage):
|
|
3361
3422
|
|
3362
3423
|
@field_validator("match", mode="after")
|
3363
3424
|
def __validate_match(
|
3364
|
-
cls,
|
3425
|
+
cls,
|
3426
|
+
match: list[Union[Match, Else]],
|
3365
3427
|
) -> list[Union[Match, Else]]:
|
3366
|
-
"""Validate the match field should contain only one Else model.
|
3428
|
+
"""Validate the match field should contain only one Else model.
|
3429
|
+
|
3430
|
+
Raises:
|
3431
|
+
ValueError: If match field contain Else more than 1 model.
|
3432
|
+
ValueError: If match field contain Match with '_' case (it represent
|
3433
|
+
the else case) more than 1 model.
|
3434
|
+
"""
|
3367
3435
|
c_else_case: int = 0
|
3368
3436
|
c_else_model: int = 0
|
3369
3437
|
for m in match:
|
@@ -3562,8 +3630,10 @@ class CaseStage(BaseNestedStage):
|
|
3562
3630
|
case: StrOrNone = param2template(self.case, params, extras=self.extras)
|
3563
3631
|
trace.info(f"[NESTED]: Get Case: {case!r}.")
|
3564
3632
|
case, stages = self.extract_stages_from_case(case, params=params)
|
3633
|
+
|
3565
3634
|
if event and event.is_set():
|
3566
3635
|
raise StageCancelError("Cancel before start case process.")
|
3636
|
+
|
3567
3637
|
status, context = self._process_nested(
|
3568
3638
|
case=case,
|
3569
3639
|
stages=stages,
|
@@ -3585,11 +3655,12 @@ class RaiseStage(BaseAsyncStage):
|
|
3585
3655
|
"""Raise error stage executor that raise `StageError` that use a message
|
3586
3656
|
field for making error message before raise.
|
3587
3657
|
|
3588
|
-
|
3589
|
-
>>> stage = {
|
3658
|
+
Examples:
|
3659
|
+
>>> stage = RaiseStage.model_validate({
|
3660
|
+
... "id": "raise-stage",
|
3590
3661
|
... "name": "Raise stage",
|
3591
3662
|
... "raise": "raise this stage",
|
3592
|
-
... }
|
3663
|
+
... })
|
3593
3664
|
|
3594
3665
|
"""
|
3595
3666
|
|
@@ -3890,7 +3961,7 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
3890
3961
|
)
|
3891
3962
|
|
3892
3963
|
@contextlib.contextmanager
|
3893
|
-
def
|
3964
|
+
def make_py_file(
|
3894
3965
|
self,
|
3895
3966
|
py: str,
|
3896
3967
|
values: DictData,
|
@@ -3966,13 +4037,14 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
3966
4037
|
- Execution python file with `uv run` via Python subprocess module.
|
3967
4038
|
|
3968
4039
|
Args:
|
3969
|
-
params: A parameter data that want to use in this
|
4040
|
+
params (DictData): A parameter data that want to use in this
|
3970
4041
|
execution.
|
3971
|
-
run_id: A running stage ID.
|
3972
|
-
context: A context data
|
3973
|
-
|
3974
|
-
|
3975
|
-
|
4042
|
+
run_id (str): A running stage ID.
|
4043
|
+
context (DictData): A context data that was passed from handler
|
4044
|
+
method.
|
4045
|
+
parent_run_id (str, default None): A parent running ID.
|
4046
|
+
event (Event, default None): An event manager that use to track
|
4047
|
+
parent process was not force stopped.
|
3976
4048
|
|
3977
4049
|
Returns:
|
3978
4050
|
Result: The execution result with status and context data.
|
@@ -3981,12 +4053,18 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
3981
4053
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
3982
4054
|
)
|
3983
4055
|
run: str = param2template(dedent(self.run), params, extras=self.extras)
|
3984
|
-
with self.
|
4056
|
+
with self.make_py_file(
|
3985
4057
|
py=run,
|
3986
4058
|
values=param2template(self.vars, params, extras=self.extras),
|
3987
4059
|
deps=param2template(self.deps, params, extras=self.extras),
|
3988
4060
|
run_id=run_id,
|
3989
4061
|
) as py:
|
4062
|
+
|
4063
|
+
if event and event.is_set():
|
4064
|
+
raise StageCancelError(
|
4065
|
+
"Cancel before start virtual python process."
|
4066
|
+
)
|
4067
|
+
|
3990
4068
|
trace.debug(f"[STAGE]: Create `{py}` file.")
|
3991
4069
|
rs: CompletedProcess = subprocess.run(
|
3992
4070
|
["python", "-m", "uv", "run", py, "--no-cache"],
|
@@ -4032,6 +4110,18 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
4032
4110
|
parent_run_id: Optional[str] = None,
|
4033
4111
|
event: Optional[Event] = None,
|
4034
4112
|
) -> Result:
|
4113
|
+
"""Async execution method for this Virtual Python stage.
|
4114
|
+
|
4115
|
+
Args:
|
4116
|
+
params (DictData): A parameter data that want to use in this
|
4117
|
+
execution.
|
4118
|
+
run_id (str): A running stage ID.
|
4119
|
+
context (DictData): A context data that was passed from handler
|
4120
|
+
method.
|
4121
|
+
parent_run_id (str, default None): A parent running ID.
|
4122
|
+
event (Event, default None): An event manager that use to track
|
4123
|
+
parent process was not force stopped.
|
4124
|
+
"""
|
4035
4125
|
raise NotImplementedError(
|
4036
4126
|
"Async process of Virtual Python stage does not implement yet."
|
4037
4127
|
)
|