ddeutil-workflow 0.0.53__py3-none-any.whl → 0.0.54__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/api/api.py +1 -1
- ddeutil/workflow/api/routes/job.py +2 -2
- ddeutil/workflow/job.py +112 -161
- ddeutil/workflow/stages.py +491 -302
- ddeutil/workflow/utils.py +5 -4
- ddeutil/workflow/workflow.py +105 -230
- {ddeutil_workflow-0.0.53.dist-info → ddeutil_workflow-0.0.54.dist-info}/METADATA +1 -7
- {ddeutil_workflow-0.0.53.dist-info → ddeutil_workflow-0.0.54.dist-info}/RECORD +14 -14
- /ddeutil/workflow/api/{log.py → logs.py} +0 -0
- /ddeutil/workflow/api/{repeat.py → utils.py} +0 -0
- {ddeutil_workflow-0.0.53.dist-info → ddeutil_workflow-0.0.54.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.53.dist-info → ddeutil_workflow-0.0.54.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.53.dist-info → ddeutil_workflow-0.0.54.dist-info}/top_level.txt +0 -0
ddeutil/workflow/stages.py
CHANGED
@@ -4,23 +4,27 @@
|
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
6
|
# [x] Use dynamic config
|
7
|
-
"""
|
8
|
-
|
9
|
-
(same thread at its job owner) that mean it is the lowest executor
|
10
|
-
|
7
|
+
"""Stages module include all stage model that use be the minimum execution layer
|
8
|
+
of this workflow engine. The stage handle the minimize task that run in some
|
9
|
+
thread (same thread at its job owner) that mean it is the lowest executor that
|
10
|
+
you can track logs.
|
11
11
|
|
12
|
-
The output of stage execution only return
|
13
|
-
handle stage error on this stage
|
14
|
-
use-case, and it does not worry
|
12
|
+
The output of stage execution only return SUCCESS or CANCEL status because
|
13
|
+
I do not want to handle stage error on this stage execution. I think stage model
|
14
|
+
have a lot of use-case, and it should does not worry about it error output.
|
15
15
|
|
16
|
-
|
16
|
+
So, I will create `handler_execute` for any exception class that raise from
|
17
|
+
the stage execution method.
|
17
18
|
|
18
|
-
|
19
|
-
|
19
|
+
Execution --> Ok ---( handler )--> Result with `SUCCESS` or `CANCEL`
|
20
|
+
|
21
|
+
--> Error ┬--( handler )-> Result with `FAILED` (Set `raise_error` flag)
|
22
|
+
|
|
23
|
+
╰--( handler )-> Raise StageException(...)
|
20
24
|
|
21
25
|
On the context I/O that pass to a stage object at execute process. The
|
22
|
-
execute method receives a `params={"params": {...}}` value for
|
23
|
-
|
26
|
+
execute method receives a `params={"params": {...}}` value for passing template
|
27
|
+
searching.
|
24
28
|
"""
|
25
29
|
from __future__ import annotations
|
26
30
|
|
@@ -28,12 +32,13 @@ import asyncio
|
|
28
32
|
import contextlib
|
29
33
|
import copy
|
30
34
|
import inspect
|
35
|
+
import json
|
31
36
|
import subprocess
|
32
37
|
import sys
|
33
38
|
import time
|
34
39
|
import uuid
|
35
40
|
from abc import ABC, abstractmethod
|
36
|
-
from collections.abc import Iterator
|
41
|
+
from collections.abc import AsyncIterator, Iterator
|
37
42
|
from concurrent.futures import (
|
38
43
|
FIRST_EXCEPTION,
|
39
44
|
Future,
|
@@ -59,6 +64,7 @@ from .exceptions import StageException, UtilException, to_dict
|
|
59
64
|
from .result import CANCEL, FAILED, SUCCESS, WAIT, Result, Status
|
60
65
|
from .reusables import TagFunc, extract_call, not_in_template, param2template
|
61
66
|
from .utils import (
|
67
|
+
NEWLINE,
|
62
68
|
delay,
|
63
69
|
filter_func,
|
64
70
|
gen_id,
|
@@ -66,20 +72,22 @@ from .utils import (
|
|
66
72
|
)
|
67
73
|
|
68
74
|
T = TypeVar("T")
|
75
|
+
StrOrInt = Union[str, int]
|
69
76
|
|
70
77
|
|
71
78
|
class BaseStage(BaseModel, ABC):
|
72
|
-
"""Base Stage Model that keep only
|
73
|
-
metadata. If you want to implement any custom
|
74
|
-
|
79
|
+
"""Base Stage Model that keep only necessary fields like `id`, `name` or
|
80
|
+
`condition` for the stage metadata. If you want to implement any custom
|
81
|
+
stage, you can inherit this class and implement `self.execute()` method
|
82
|
+
only.
|
75
83
|
|
76
|
-
This class is the abstraction class for any stage model that
|
77
|
-
|
84
|
+
This class is the abstraction class for any inherit stage model that
|
85
|
+
want to implement on this workflow package.
|
78
86
|
"""
|
79
87
|
|
80
88
|
extras: DictData = Field(
|
81
89
|
default_factory=dict,
|
82
|
-
description="An extra override config values.",
|
90
|
+
description="An extra parameter that override core config values.",
|
83
91
|
)
|
84
92
|
id: Optional[str] = Field(
|
85
93
|
default=None,
|
@@ -93,14 +101,17 @@ class BaseStage(BaseModel, ABC):
|
|
93
101
|
)
|
94
102
|
condition: Optional[str] = Field(
|
95
103
|
default=None,
|
96
|
-
description=
|
104
|
+
description=(
|
105
|
+
"A stage condition statement to allow stage executable. This field "
|
106
|
+
"alise with `if` field."
|
107
|
+
),
|
97
108
|
alias="if",
|
98
109
|
)
|
99
110
|
|
100
111
|
@property
|
101
112
|
def iden(self) -> str:
|
102
|
-
"""Return
|
103
|
-
|
113
|
+
"""Return this stage identity that return the `id` field first and if
|
114
|
+
this `id` field does not set, it will use the `name` field instead.
|
104
115
|
|
105
116
|
:rtype: str
|
106
117
|
"""
|
@@ -156,8 +167,8 @@ class BaseStage(BaseModel, ABC):
|
|
156
167
|
run_id: str | None = None,
|
157
168
|
parent_run_id: str | None = None,
|
158
169
|
result: Result | None = None,
|
159
|
-
raise_error: bool | None = None,
|
160
170
|
event: Event | None = None,
|
171
|
+
raise_error: bool | None = None,
|
161
172
|
) -> Result | DictData:
|
162
173
|
"""Handler stage execution result from the stage `execute` method.
|
163
174
|
|
@@ -170,7 +181,7 @@ class BaseStage(BaseModel, ABC):
|
|
170
181
|
╰-context:
|
171
182
|
╰-outputs: ...
|
172
183
|
|
173
|
-
--> Error --> Result (if
|
184
|
+
--> Error --> Result (if `raise_error` was set)
|
174
185
|
|-status: FAILED
|
175
186
|
╰-errors:
|
176
187
|
|-class: ...
|
@@ -179,18 +190,17 @@ class BaseStage(BaseModel, ABC):
|
|
179
190
|
|
180
191
|
--> Error --> Raise StageException(...)
|
181
192
|
|
182
|
-
On the last step, it will set the running ID on a return result
|
183
|
-
from current stage ID before release the final result.
|
193
|
+
On the last step, it will set the running ID on a return result
|
194
|
+
object from the current stage ID before release the final result.
|
184
195
|
|
185
|
-
:param params: (DictData) A
|
186
|
-
stage execution.
|
196
|
+
:param params: (DictData) A parameter data.
|
187
197
|
:param run_id: (str) A running stage ID for this execution.
|
188
198
|
:param parent_run_id: (str) A parent workflow running ID for this
|
189
199
|
execution.
|
190
200
|
:param result: (Result) A result object for keeping context and status
|
191
201
|
data before execution.
|
192
|
-
:param raise_error: (bool) A flag that all this method raise error
|
193
202
|
:param event: (Event) An event manager that pass to the stage execution.
|
203
|
+
:param raise_error: (bool) A flag that all this method raise error
|
194
204
|
|
195
205
|
:rtype: Result
|
196
206
|
"""
|
@@ -206,8 +216,9 @@ class BaseStage(BaseModel, ABC):
|
|
206
216
|
rs: Result = self.execute(params, result=result, event=event)
|
207
217
|
return rs
|
208
218
|
except Exception as e:
|
209
|
-
result.trace.error(
|
210
|
-
|
219
|
+
result.trace.error(
|
220
|
+
f"[STAGE]: Handler:{NEWLINE}{e.__class__.__name__}: {e}"
|
221
|
+
)
|
211
222
|
if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
|
212
223
|
if isinstance(e, StageException):
|
213
224
|
raise
|
@@ -217,37 +228,39 @@ class BaseStage(BaseModel, ABC):
|
|
217
228
|
f"{e.__class__.__name__}: {e}"
|
218
229
|
) from e
|
219
230
|
|
220
|
-
|
221
|
-
return result.catch(status=FAILED, context=errors)
|
231
|
+
return result.catch(status=FAILED, context={"errors": to_dict(e)})
|
222
232
|
|
223
233
|
def set_outputs(self, output: DictData, to: DictData) -> DictData:
|
224
|
-
"""Set an outputs from execution
|
225
|
-
|
234
|
+
"""Set an outputs from execution result context to the received context
|
235
|
+
with a `to` input parameter. The result context from stage execution
|
236
|
+
will be set with `outputs` key in this stage ID key.
|
226
237
|
|
227
238
|
For example of setting output method, If you receive execute output
|
228
239
|
and want to set on the `to` like;
|
229
240
|
|
230
|
-
... (i) output: {'foo': bar}
|
241
|
+
... (i) output: {'foo': 'bar', 'skipped': True}
|
231
242
|
... (ii) to: {'stages': {}}
|
232
243
|
|
233
|
-
The
|
244
|
+
The received context in the `to` argument will be;
|
234
245
|
|
235
246
|
... (iii) to: {
|
236
247
|
'stages': {
|
237
248
|
'<stage-id>': {
|
238
249
|
'outputs': {'foo': 'bar'},
|
239
|
-
'skipped':
|
250
|
+
'skipped': True,
|
240
251
|
}
|
241
252
|
}
|
242
253
|
}
|
243
254
|
|
244
255
|
Important:
|
256
|
+
|
245
257
|
This method is use for reconstruct the result context and transfer
|
246
|
-
to the `to` argument.
|
258
|
+
to the `to` argument. The result context was soft copied before set
|
259
|
+
output step.
|
247
260
|
|
248
|
-
:param output: (DictData)
|
249
|
-
|
250
|
-
:param to: (DictData) A context data
|
261
|
+
:param output: (DictData) A result data context that want to extract
|
262
|
+
and transfer to the `outputs` key in receive context.
|
263
|
+
:param to: (DictData) A received context data.
|
251
264
|
|
252
265
|
:rtype: DictData
|
253
266
|
"""
|
@@ -282,11 +295,12 @@ class BaseStage(BaseModel, ABC):
|
|
282
295
|
}
|
283
296
|
return to
|
284
297
|
|
285
|
-
def get_outputs(self,
|
298
|
+
def get_outputs(self, output: DictData) -> DictData:
|
286
299
|
"""Get the outputs from stages data. It will get this stage ID from
|
287
300
|
the stage outputs mapping.
|
288
301
|
|
289
|
-
:param
|
302
|
+
:param output: (DictData) A stage output context that want to get this
|
303
|
+
stage ID `outputs` key.
|
290
304
|
|
291
305
|
:rtype: DictData
|
292
306
|
"""
|
@@ -296,33 +310,31 @@ class BaseStage(BaseModel, ABC):
|
|
296
310
|
return {}
|
297
311
|
|
298
312
|
_id: str = (
|
299
|
-
param2template(self.id, params=
|
313
|
+
param2template(self.id, params=output, extras=self.extras)
|
300
314
|
if self.id
|
301
315
|
else gen_id(
|
302
|
-
param2template(self.name, params=
|
316
|
+
param2template(self.name, params=output, extras=self.extras)
|
303
317
|
)
|
304
318
|
)
|
305
|
-
return
|
319
|
+
return output.get("stages", {}).get(_id, {}).get("outputs", {})
|
306
320
|
|
307
|
-
def is_skipped(self, params: DictData
|
321
|
+
def is_skipped(self, params: DictData) -> bool:
|
308
322
|
"""Return true if condition of this stage do not correct. This process
|
309
323
|
use build-in eval function to execute the if-condition.
|
310
324
|
|
325
|
+
:param params: (DictData) A parameters that want to pass to condition
|
326
|
+
template.
|
327
|
+
|
311
328
|
:raise StageException: When it has any error raise from the eval
|
312
329
|
condition statement.
|
313
330
|
:raise StageException: When return type of the eval condition statement
|
314
331
|
does not return with boolean type.
|
315
332
|
|
316
|
-
:param params: (DictData) A parameters that want to pass to condition
|
317
|
-
template.
|
318
|
-
|
319
333
|
:rtype: bool
|
320
334
|
"""
|
321
335
|
if self.condition is None:
|
322
336
|
return False
|
323
337
|
|
324
|
-
params: DictData = {} if params is None else params
|
325
|
-
|
326
338
|
try:
|
327
339
|
# WARNING: The eval build-in function is very dangerous. So, it
|
328
340
|
# should use the `re` module to validate eval-string before
|
@@ -340,7 +352,14 @@ class BaseStage(BaseModel, ABC):
|
|
340
352
|
|
341
353
|
|
342
354
|
class BaseAsyncStage(BaseStage):
|
343
|
-
"""Base Async Stage model
|
355
|
+
"""Base Async Stage model to make any stage model allow async execution for
|
356
|
+
optimize CPU and Memory on the current node. If you want to implement any
|
357
|
+
custom async stage, you can inherit this class and implement
|
358
|
+
`self.axecute()` (async + execute = axecute) method only.
|
359
|
+
|
360
|
+
This class is the abstraction class for any inherit asyncable stage
|
361
|
+
model.
|
362
|
+
"""
|
344
363
|
|
345
364
|
@abstractmethod
|
346
365
|
def execute(
|
@@ -385,20 +404,18 @@ class BaseAsyncStage(BaseStage):
|
|
385
404
|
run_id: str | None = None,
|
386
405
|
parent_run_id: str | None = None,
|
387
406
|
result: Result | None = None,
|
388
|
-
raise_error: bool | None = None,
|
389
407
|
event: Event | None = None,
|
408
|
+
raise_error: bool | None = None,
|
390
409
|
) -> Result:
|
391
410
|
"""Async Handler stage execution result from the stage `execute` method.
|
392
411
|
|
393
|
-
:param params: (DictData) A
|
394
|
-
|
395
|
-
:param
|
396
|
-
:param
|
397
|
-
|
398
|
-
|
399
|
-
data before execution.
|
412
|
+
:param params: (DictData) A parameter data.
|
413
|
+
:param run_id: (str) A stage running ID.
|
414
|
+
:param parent_run_id: (str) A parent job running ID.
|
415
|
+
:param result: (Result) A Result instance for return context and status.
|
416
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
417
|
+
execution if it forces stopped by parent execution.
|
400
418
|
:param raise_error: (bool) A flag that all this method raise error
|
401
|
-
:param event: (Event) An event manager that pass to the stage execution.
|
402
419
|
|
403
420
|
:rtype: Result
|
404
421
|
"""
|
@@ -414,7 +431,9 @@ class BaseAsyncStage(BaseStage):
|
|
414
431
|
rs: Result = await self.axecute(params, result=result, event=event)
|
415
432
|
return rs
|
416
433
|
except Exception as e: # pragma: no cov
|
417
|
-
await result.trace.aerror(
|
434
|
+
await result.trace.aerror(
|
435
|
+
f"[STAGE]: Handler {e.__class__.__name__}: {e}"
|
436
|
+
)
|
418
437
|
|
419
438
|
if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
|
420
439
|
if isinstance(e, StageException):
|
@@ -425,13 +444,16 @@ class BaseAsyncStage(BaseStage):
|
|
425
444
|
f"{e.__class__.__name__}: {e}"
|
426
445
|
) from None
|
427
446
|
|
428
|
-
|
429
|
-
return result.catch(status=FAILED, context=errors)
|
447
|
+
return result.catch(status=FAILED, context={"errors": to_dict(e)})
|
430
448
|
|
431
449
|
|
432
450
|
class EmptyStage(BaseAsyncStage):
|
433
|
-
"""Empty stage that do nothing
|
434
|
-
|
451
|
+
"""Empty stage executor that do nothing and log the `message` field to
|
452
|
+
stdout only. It can use for tracking a template parameter on the workflow or
|
453
|
+
debug step.
|
454
|
+
|
455
|
+
You can pass a sleep value in second unit to this stage for waiting
|
456
|
+
after log message.
|
435
457
|
|
436
458
|
Data Validate:
|
437
459
|
>>> stage = {
|
@@ -443,11 +465,14 @@ class EmptyStage(BaseAsyncStage):
|
|
443
465
|
|
444
466
|
echo: Optional[str] = Field(
|
445
467
|
default=None,
|
446
|
-
description="A
|
468
|
+
description="A message that want to show on the stdout.",
|
447
469
|
)
|
448
470
|
sleep: float = Field(
|
449
471
|
default=0,
|
450
|
-
description=
|
472
|
+
description=(
|
473
|
+
"A second value to sleep before start execution. This value should "
|
474
|
+
"gather or equal 0, and less than 1800 seconds."
|
475
|
+
),
|
451
476
|
ge=0,
|
452
477
|
lt=1800,
|
453
478
|
)
|
@@ -460,18 +485,15 @@ class EmptyStage(BaseAsyncStage):
|
|
460
485
|
event: Event | None = None,
|
461
486
|
) -> Result:
|
462
487
|
"""Execution method for the Empty stage that do only logging out to
|
463
|
-
stdout.
|
464
|
-
it does not get any error from logging function.
|
488
|
+
stdout.
|
465
489
|
|
466
490
|
The result context should be empty and do not process anything
|
467
491
|
without calling logging function.
|
468
492
|
|
469
|
-
:param params: (DictData) A
|
470
|
-
|
471
|
-
:param
|
472
|
-
|
473
|
-
:param event: (Event) An event manager that use to track parent execute
|
474
|
-
was not force stopped.
|
493
|
+
:param params: (DictData) A parameter data.
|
494
|
+
:param result: (Result) A Result instance for return context and status.
|
495
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
496
|
+
execution if it forces stopped by parent execution.
|
475
497
|
|
476
498
|
:rtype: Result
|
477
499
|
"""
|
@@ -484,21 +506,18 @@ class EmptyStage(BaseAsyncStage):
|
|
484
506
|
message: str = "..."
|
485
507
|
else:
|
486
508
|
message: str = param2template(
|
487
|
-
dedent(self.echo), params, extras=self.extras
|
509
|
+
dedent(self.echo.strip("\n")), params, extras=self.extras
|
488
510
|
)
|
489
|
-
if "\n" in
|
490
|
-
message: str =
|
491
|
-
"\n"
|
492
|
-
)
|
511
|
+
if "\n" in message:
|
512
|
+
message: str = NEWLINE + message.replace("\n", NEWLINE)
|
493
513
|
|
494
514
|
result.trace.info(
|
495
515
|
f"[STAGE]: Empty-Execute: {self.name!r}: ( {message} )"
|
496
516
|
)
|
497
517
|
if self.sleep > 0:
|
498
518
|
if self.sleep > 5:
|
499
|
-
result.trace.info(f"[STAGE]: ... sleep ({self.sleep}
|
519
|
+
result.trace.info(f"[STAGE]: ... sleep ({self.sleep} sec)")
|
500
520
|
time.sleep(self.sleep)
|
501
|
-
|
502
521
|
return result.catch(status=SUCCESS)
|
503
522
|
|
504
523
|
async def axecute(
|
@@ -511,12 +530,10 @@ class EmptyStage(BaseAsyncStage):
|
|
511
530
|
"""Async execution method for this Empty stage that only logging out to
|
512
531
|
stdout.
|
513
532
|
|
514
|
-
:param params: (DictData) A
|
515
|
-
|
516
|
-
:param
|
517
|
-
|
518
|
-
:param event: (Event) An event manager that use to track parent execute
|
519
|
-
was not force stopped.
|
533
|
+
:param params: (DictData) A parameter data.
|
534
|
+
:param result: (Result) A Result instance for return context and status.
|
535
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
536
|
+
execution if it forces stopped by parent execution.
|
520
537
|
|
521
538
|
:rtype: Result
|
522
539
|
"""
|
@@ -526,29 +543,36 @@ class EmptyStage(BaseAsyncStage):
|
|
526
543
|
extras=self.extras,
|
527
544
|
)
|
528
545
|
|
529
|
-
|
530
|
-
|
531
|
-
|
532
|
-
|
546
|
+
if not self.echo:
|
547
|
+
message: str = "..."
|
548
|
+
else:
|
549
|
+
message: str = param2template(
|
550
|
+
dedent(self.echo.strip("\n")), params, extras=self.extras
|
551
|
+
)
|
552
|
+
if "\n" in message:
|
553
|
+
message: str = NEWLINE + message.replace("\n", NEWLINE)
|
533
554
|
|
555
|
+
result.trace.info(
|
556
|
+
f"[STAGE]: Empty-Execute: {self.name!r}: ( {message} )"
|
557
|
+
)
|
534
558
|
if self.sleep > 0:
|
535
559
|
if self.sleep > 5:
|
536
560
|
await result.trace.ainfo(
|
537
|
-
f"[STAGE]: ... sleep ({self.sleep}
|
561
|
+
f"[STAGE]: ... sleep ({self.sleep} sec)"
|
538
562
|
)
|
539
563
|
await asyncio.sleep(self.sleep)
|
540
|
-
|
541
564
|
return result.catch(status=SUCCESS)
|
542
565
|
|
543
566
|
|
544
567
|
class BashStage(BaseStage):
|
545
|
-
"""Bash
|
546
|
-
If your current OS is Windows, it will run on the bash
|
568
|
+
"""Bash stage executor that execute bash script on the current OS.
|
569
|
+
If your current OS is Windows, it will run on the bash from the current WSL.
|
570
|
+
It will use `bash` for Windows OS and use `sh` for Linux OS.
|
547
571
|
|
548
|
-
|
549
|
-
subprocess package. It does not good enough to use multiline
|
550
|
-
Thus,
|
551
|
-
issue.
|
572
|
+
This stage has some limitation when it runs shell statement with the
|
573
|
+
built-in subprocess package. It does not good enough to use multiline
|
574
|
+
statement. Thus, it will write the `.sh` file before start running bash
|
575
|
+
command for fix this issue.
|
552
576
|
|
553
577
|
Data Validate:
|
554
578
|
>>> stage = {
|
@@ -560,15 +584,46 @@ class BashStage(BaseStage):
|
|
560
584
|
... }
|
561
585
|
"""
|
562
586
|
|
563
|
-
bash: str = Field(
|
587
|
+
bash: str = Field(
|
588
|
+
description=(
|
589
|
+
"A bash statement that want to execute via Python subprocess."
|
590
|
+
)
|
591
|
+
)
|
564
592
|
env: DictStr = Field(
|
565
593
|
default_factory=dict,
|
566
594
|
description=(
|
567
|
-
"An environment variables that set before
|
568
|
-
"on the header of the `.sh` file."
|
595
|
+
"An environment variables that set before run bash command. It "
|
596
|
+
"will add on the header of the `.sh` file."
|
569
597
|
),
|
570
598
|
)
|
571
599
|
|
600
|
+
@contextlib.asynccontextmanager
|
601
|
+
async def acreate_sh_file(
|
602
|
+
self, bash: str, env: DictStr, run_id: str | None = None
|
603
|
+
) -> AsyncIterator: # pragma no cov
|
604
|
+
import aiofiles
|
605
|
+
|
606
|
+
f_name: str = f"{run_id or uuid.uuid4()}.sh"
|
607
|
+
f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
|
608
|
+
|
609
|
+
async with aiofiles.open(f"./{f_name}", mode="w", newline="\n") as f:
|
610
|
+
# NOTE: write header of `.sh` file
|
611
|
+
await f.write(f"#!/bin/{f_shebang}\n\n")
|
612
|
+
|
613
|
+
# NOTE: add setting environment variable before bash skip statement.
|
614
|
+
await f.writelines([f"{k}='{env[k]}';\n" for k in env])
|
615
|
+
|
616
|
+
# NOTE: make sure that shell script file does not have `\r` char.
|
617
|
+
await f.write("\n" + bash.replace("\r\n", "\n"))
|
618
|
+
|
619
|
+
# NOTE: Make this .sh file able to executable.
|
620
|
+
make_exec(f"./{f_name}")
|
621
|
+
|
622
|
+
yield [f_shebang, f_name]
|
623
|
+
|
624
|
+
# Note: Remove .sh file that use to run bash.
|
625
|
+
Path(f"./{f_name}").unlink()
|
626
|
+
|
572
627
|
@contextlib.contextmanager
|
573
628
|
def create_sh_file(
|
574
629
|
self, bash: str, env: DictStr, run_id: str | None = None
|
@@ -577,16 +632,14 @@ class BashStage(BaseStage):
|
|
577
632
|
step will write the `.sh` file before giving this file name to context.
|
578
633
|
After that, it will auto delete this file automatic.
|
579
634
|
|
580
|
-
:param bash: (str) A bash statement
|
581
|
-
:param env: (DictStr) An environment variable that
|
582
|
-
statement.
|
635
|
+
:param bash: (str) A bash statement.
|
636
|
+
:param env: (DictStr) An environment variable that set before run bash.
|
583
637
|
:param run_id: (str | None) A running stage ID that use for writing sh
|
584
638
|
file instead generate by UUID4.
|
585
639
|
|
586
640
|
:rtype: Iterator[TupleStr]
|
587
641
|
"""
|
588
|
-
|
589
|
-
f_name: str = f"{run_id}.sh"
|
642
|
+
f_name: str = f"{run_id or uuid.uuid4()}.sh"
|
590
643
|
f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
|
591
644
|
|
592
645
|
with open(f"./{f_name}", mode="w", newline="\n") as f:
|
@@ -614,14 +667,14 @@ class BashStage(BaseStage):
|
|
614
667
|
result: Result | None = None,
|
615
668
|
event: Event | None = None,
|
616
669
|
) -> Result:
|
617
|
-
"""Execute
|
618
|
-
|
670
|
+
"""Execute bash statement with the Python build-in `subprocess` package.
|
671
|
+
It will catch result from the `subprocess.run` returning output like
|
672
|
+
`return_code`, `stdout`, and `stderr`.
|
619
673
|
|
620
|
-
:param params: A parameter data
|
621
|
-
:param result: (Result) A
|
622
|
-
|
623
|
-
|
624
|
-
was not force stopped.
|
674
|
+
:param params: (DictData) A parameter data.
|
675
|
+
:param result: (Result) A Result instance for return context and status.
|
676
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
677
|
+
execution if it forces stopped by parent execution.
|
625
678
|
|
626
679
|
:rtype: Result
|
627
680
|
"""
|
@@ -631,11 +684,12 @@ class BashStage(BaseStage):
|
|
631
684
|
extras=self.extras,
|
632
685
|
)
|
633
686
|
|
687
|
+
result.trace.info(f"[STAGE]: Shell-Execute: {self.name}")
|
688
|
+
|
634
689
|
bash: str = param2template(
|
635
|
-
dedent(self.bash), params, extras=self.extras
|
690
|
+
dedent(self.bash.strip("\n")), params, extras=self.extras
|
636
691
|
)
|
637
692
|
|
638
|
-
result.trace.info(f"[STAGE]: Shell-Execute: {self.name}")
|
639
693
|
with self.create_sh_file(
|
640
694
|
bash=bash,
|
641
695
|
env=param2template(self.env, params, extras=self.extras),
|
@@ -654,7 +708,7 @@ class BashStage(BaseStage):
|
|
654
708
|
else rs.stderr
|
655
709
|
).removesuffix("\n")
|
656
710
|
raise StageException(
|
657
|
-
f"Subprocess: {e}\
|
711
|
+
f"Subprocess: {e}\n---( statement )---\n"
|
658
712
|
f"```bash\n{bash}\n```"
|
659
713
|
)
|
660
714
|
return result.catch(
|
@@ -668,18 +722,24 @@ class BashStage(BaseStage):
|
|
668
722
|
|
669
723
|
|
670
724
|
class PyStage(BaseStage):
|
671
|
-
"""Python
|
672
|
-
|
725
|
+
"""Python stage that running the Python statement with the current globals
|
726
|
+
and passing an input additional variables via `exec` built-in function.
|
673
727
|
|
674
728
|
This stage allow you to use any Python object that exists on the globals
|
675
729
|
such as import your installed package.
|
676
730
|
|
731
|
+
Warning:
|
732
|
+
|
733
|
+
The exec build-in function is very dangerous. So, it should use the `re`
|
734
|
+
module to validate exec-string before running or exclude the `os` package
|
735
|
+
from the current globals variable.
|
736
|
+
|
677
737
|
Data Validate:
|
678
738
|
>>> stage = {
|
679
739
|
... "name": "Python stage execution",
|
680
|
-
... "run": 'print("Hello {
|
740
|
+
... "run": 'print(f"Hello {VARIABLE}")',
|
681
741
|
... "vars": {
|
682
|
-
... "
|
742
|
+
... "VARIABLE": "WORLD",
|
683
743
|
... },
|
684
744
|
... }
|
685
745
|
"""
|
@@ -741,9 +801,9 @@ class PyStage(BaseStage):
|
|
741
801
|
event: Event | None = None,
|
742
802
|
) -> Result:
|
743
803
|
"""Execute the Python statement that pass all globals and input params
|
744
|
-
to globals argument on
|
804
|
+
to globals argument on `exec` build-in function.
|
745
805
|
|
746
|
-
:param params: A parameter
|
806
|
+
:param params: (DictData) A parameter data.
|
747
807
|
:param result: (Result) A result object for keeping context and status
|
748
808
|
data.
|
749
809
|
:param event: (Event) An event manager that use to track parent execute
|
@@ -792,20 +852,36 @@ class PyStage(BaseStage):
|
|
792
852
|
},
|
793
853
|
)
|
794
854
|
|
855
|
+
async def axecute(
|
856
|
+
self,
|
857
|
+
):
|
858
|
+
"""Async execution method.
|
859
|
+
|
860
|
+
References:
|
861
|
+
- https://stackoverflow.com/questions/44859165/async-exec-in-python
|
862
|
+
"""
|
863
|
+
|
795
864
|
|
796
865
|
class CallStage(BaseStage):
|
797
|
-
"""Call executor that call the Python function from registry with tag
|
798
|
-
decorator function in
|
866
|
+
"""Call stage executor that call the Python function from registry with tag
|
867
|
+
decorator function in `reusables` module and run it with input arguments.
|
868
|
+
|
869
|
+
This stage is different with PyStage because the PyStage is just run
|
870
|
+
a Python statement with the `exec` function and pass the current locals and
|
871
|
+
globals before exec that statement. This stage will import the caller
|
872
|
+
function can call it with an input arguments. So, you can create your
|
873
|
+
function complexly that you can for your objective to invoked by this stage
|
874
|
+
object.
|
799
875
|
|
800
|
-
This stage is
|
801
|
-
|
802
|
-
|
803
|
-
|
876
|
+
This stage is the most powerfull stage of this package for run every
|
877
|
+
use-case by a custom requirement that you want by creating the Python
|
878
|
+
function and adding it to the caller registry value by importer syntax like
|
879
|
+
`module.caller.registry` not path style like `module/caller/registry`.
|
804
880
|
|
805
|
-
|
806
|
-
|
807
|
-
|
808
|
-
|
881
|
+
Warning:
|
882
|
+
|
883
|
+
The caller registry to get a caller function should importable by the
|
884
|
+
current Python execution pointer.
|
809
885
|
|
810
886
|
Data Validate:
|
811
887
|
>>> stage = {
|
@@ -817,12 +893,16 @@ class CallStage(BaseStage):
|
|
817
893
|
|
818
894
|
uses: str = Field(
|
819
895
|
description=(
|
820
|
-
"A
|
896
|
+
"A caller function with registry importer syntax that use to load "
|
897
|
+
"function before execute step. The caller registry syntax should "
|
898
|
+
"be `<import.part>/<func-name>@<tag-name>`."
|
821
899
|
),
|
822
900
|
)
|
823
901
|
args: DictData = Field(
|
824
902
|
default_factory=dict,
|
825
|
-
description=
|
903
|
+
description=(
|
904
|
+
"An argument parameter that will pass to this caller function."
|
905
|
+
),
|
826
906
|
alias="with",
|
827
907
|
)
|
828
908
|
|
@@ -833,19 +913,12 @@ class CallStage(BaseStage):
|
|
833
913
|
result: Result | None = None,
|
834
914
|
event: Event | None = None,
|
835
915
|
) -> Result:
|
836
|
-
"""Execute
|
916
|
+
"""Execute this caller function with its argument parameter.
|
837
917
|
|
838
|
-
:
|
839
|
-
|
840
|
-
:
|
841
|
-
|
842
|
-
|
843
|
-
:param params: (DictData) A parameter that want to pass before run any
|
844
|
-
statement.
|
845
|
-
:param result: (Result) A result object for keeping context and status
|
846
|
-
data.
|
847
|
-
:param event: (Event) An event manager that use to track parent execute
|
848
|
-
was not force stopped.
|
918
|
+
:param params: (DictData) A parameter data.
|
919
|
+
:param result: (Result) A Result instance for return context and status.
|
920
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
921
|
+
execution if it forces stopped by parent execution.
|
849
922
|
|
850
923
|
:raise ValueError: If necessary arguments does not pass from the `args`
|
851
924
|
field.
|
@@ -860,12 +933,15 @@ class CallStage(BaseStage):
|
|
860
933
|
extras=self.extras,
|
861
934
|
)
|
862
935
|
|
863
|
-
has_keyword: bool = False
|
864
936
|
call_func: TagFunc = extract_call(
|
865
937
|
param2template(self.uses, params, extras=self.extras),
|
866
938
|
registries=self.extras.get("registry_caller"),
|
867
939
|
)()
|
868
940
|
|
941
|
+
result.trace.info(
|
942
|
+
f"[STAGE]: Call-Execute: {call_func.name}@{call_func.tag}"
|
943
|
+
)
|
944
|
+
|
869
945
|
# VALIDATE: check input task caller parameters that exists before
|
870
946
|
# calling.
|
871
947
|
args: DictData = {"result": result} | param2template(
|
@@ -873,6 +949,7 @@ class CallStage(BaseStage):
|
|
873
949
|
)
|
874
950
|
ips = inspect.signature(call_func)
|
875
951
|
necessary_params: list[str] = []
|
952
|
+
has_keyword: bool = False
|
876
953
|
for k in ips.parameters:
|
877
954
|
if (
|
878
955
|
v := ips.parameters[k]
|
@@ -896,10 +973,6 @@ class CallStage(BaseStage):
|
|
896
973
|
if "result" not in ips.parameters and not has_keyword:
|
897
974
|
args.pop("result")
|
898
975
|
|
899
|
-
result.trace.info(
|
900
|
-
f"[STAGE]: Call-Execute: {call_func.name}@{call_func.tag}"
|
901
|
-
)
|
902
|
-
|
903
976
|
args = self.parse_model_args(call_func, args, result)
|
904
977
|
|
905
978
|
if inspect.iscoroutinefunction(call_func):
|
@@ -970,9 +1043,9 @@ class CallStage(BaseStage):
|
|
970
1043
|
|
971
1044
|
|
972
1045
|
class TriggerStage(BaseStage):
|
973
|
-
"""Trigger
|
974
|
-
the
|
975
|
-
|
1046
|
+
"""Trigger workflow executor stage that run an input trigger Workflow
|
1047
|
+
execute method. This is the stage that allow you to create the reusable
|
1048
|
+
Workflow template with dynamic parameters.
|
976
1049
|
|
977
1050
|
Data Validate:
|
978
1051
|
>>> stage = {
|
@@ -984,12 +1057,13 @@ class TriggerStage(BaseStage):
|
|
984
1057
|
|
985
1058
|
trigger: str = Field(
|
986
1059
|
description=(
|
987
|
-
"A trigger workflow name
|
1060
|
+
"A trigger workflow name. This workflow name should exist on the "
|
1061
|
+
"config path because it will load by the `load_conf` method."
|
988
1062
|
),
|
989
1063
|
)
|
990
1064
|
params: DictData = Field(
|
991
1065
|
default_factory=dict,
|
992
|
-
description="A parameter that
|
1066
|
+
description="A parameter that will pass to workflow execution method.",
|
993
1067
|
)
|
994
1068
|
|
995
1069
|
def execute(
|
@@ -1002,7 +1076,7 @@ class TriggerStage(BaseStage):
|
|
1002
1076
|
"""Trigger another workflow execution. It will wait the trigger
|
1003
1077
|
workflow running complete before catching its result.
|
1004
1078
|
|
1005
|
-
:param params: A parameter data
|
1079
|
+
:param params: (DictData) A parameter data.
|
1006
1080
|
:param result: (Result) A result object for keeping context and status
|
1007
1081
|
data.
|
1008
1082
|
:param event: (Event) An event manager that use to track parent execute
|
@@ -1037,14 +1111,18 @@ class TriggerStage(BaseStage):
|
|
1037
1111
|
err_msg: str | None = (
|
1038
1112
|
f" with:\n{msg}"
|
1039
1113
|
if (msg := rs.context.get("errors", {}).get("message"))
|
1040
|
-
else ""
|
1114
|
+
else "."
|
1115
|
+
)
|
1116
|
+
raise StageException(
|
1117
|
+
f"Trigger workflow return failed status{err_msg}"
|
1041
1118
|
)
|
1042
|
-
raise StageException(f"Trigger workflow was failed{err_msg}.")
|
1043
1119
|
return rs
|
1044
1120
|
|
1045
1121
|
|
1046
1122
|
class ParallelStage(BaseStage): # pragma: no cov
|
1047
|
-
"""Parallel
|
1123
|
+
"""Parallel stage executor that execute branch stages with multithreading.
|
1124
|
+
This stage let you set the fix branches for running substage inside it on
|
1125
|
+
multithread pool.
|
1048
1126
|
|
1049
1127
|
This stage is not the low-level stage model because it runs muti-stages
|
1050
1128
|
in this stage execution.
|
@@ -1072,48 +1150,46 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1072
1150
|
"""
|
1073
1151
|
|
1074
1152
|
parallel: dict[str, list[Stage]] = Field(
|
1075
|
-
description="A mapping of
|
1153
|
+
description="A mapping of branch name and its stages.",
|
1076
1154
|
)
|
1077
1155
|
max_workers: int = Field(
|
1078
1156
|
default=2,
|
1079
1157
|
ge=1,
|
1080
1158
|
lt=20,
|
1081
1159
|
description=(
|
1082
|
-
"The maximum thread pool worker size for execution parallel."
|
1160
|
+
"The maximum multi-thread pool worker size for execution parallel. "
|
1161
|
+
"This value should be gather or equal than 1, and less than 20."
|
1083
1162
|
),
|
1084
1163
|
alias="max-workers",
|
1085
1164
|
)
|
1086
1165
|
|
1087
|
-
def
|
1166
|
+
def execute_branch(
|
1088
1167
|
self,
|
1089
1168
|
branch: str,
|
1090
1169
|
params: DictData,
|
1091
1170
|
result: Result,
|
1092
1171
|
*,
|
1093
1172
|
event: Event | None = None,
|
1094
|
-
extras: DictData | None = None,
|
1095
1173
|
) -> DictData:
|
1096
|
-
"""
|
1174
|
+
"""Branch execution method for execute all stages of a specific branch
|
1175
|
+
ID.
|
1097
1176
|
|
1098
|
-
:param branch: A branch ID.
|
1099
|
-
:param params: A parameter data
|
1100
|
-
:param result: (Result) A
|
1101
|
-
|
1102
|
-
|
1103
|
-
was not force stopped.
|
1104
|
-
:param extras: (DictData) An extra parameters that want to override
|
1105
|
-
config values.
|
1177
|
+
:param branch: (str) A branch ID.
|
1178
|
+
:param params: (DictData) A parameter data.
|
1179
|
+
:param result: (Result) A Result instance for return context and status.
|
1180
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1181
|
+
execution if it forces stopped by parent execution.
|
1106
1182
|
|
1107
1183
|
:rtype: DictData
|
1108
1184
|
"""
|
1109
|
-
result.trace.debug(f"
|
1185
|
+
result.trace.debug(f"[STAGE]: Execute Branch: {branch!r}")
|
1110
1186
|
context: DictData = copy.deepcopy(params)
|
1111
1187
|
context.update({"branch": branch})
|
1112
1188
|
output: DictData = {"branch": branch, "stages": {}}
|
1113
1189
|
for stage in self.parallel[branch]:
|
1114
1190
|
|
1115
|
-
if extras:
|
1116
|
-
stage.extras = extras
|
1191
|
+
if self.extras:
|
1192
|
+
stage.extras = self.extras
|
1117
1193
|
|
1118
1194
|
if stage.is_skipped(params=context):
|
1119
1195
|
result.trace.info(f"... Skip stage: {stage.iden!r}")
|
@@ -1154,7 +1230,7 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1154
1230
|
|
1155
1231
|
if rs.status == FAILED:
|
1156
1232
|
error_msg: str = (
|
1157
|
-
f"
|
1233
|
+
f"Branch-Stage was break because it has a sub stage, "
|
1158
1234
|
f"{stage.iden}, failed without raise error."
|
1159
1235
|
)
|
1160
1236
|
return result.catch(
|
@@ -1185,14 +1261,12 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1185
1261
|
result: Result | None = None,
|
1186
1262
|
event: Event | None = None,
|
1187
1263
|
) -> Result:
|
1188
|
-
"""Execute
|
1189
|
-
or async mode by changing `async_mode` flag.
|
1264
|
+
"""Execute parallel each branch via multi-threading pool.
|
1190
1265
|
|
1191
|
-
:param params: A parameter
|
1192
|
-
:param result: (Result) A
|
1193
|
-
|
1194
|
-
|
1195
|
-
was not force stopped.
|
1266
|
+
:param params: (DictData) A parameter data.
|
1267
|
+
:param result: (Result) A Result instance for return context and status.
|
1268
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1269
|
+
execution if it forces stopped by parent execution.
|
1196
1270
|
|
1197
1271
|
:rtype: Result
|
1198
1272
|
"""
|
@@ -1216,12 +1290,11 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1216
1290
|
|
1217
1291
|
futures: list[Future] = (
|
1218
1292
|
executor.submit(
|
1219
|
-
self.
|
1293
|
+
self.execute_branch,
|
1220
1294
|
branch=branch,
|
1221
1295
|
params=params,
|
1222
1296
|
result=result,
|
1223
1297
|
event=event,
|
1224
|
-
extras=self.extras,
|
1225
1298
|
)
|
1226
1299
|
for branch in self.parallel
|
1227
1300
|
)
|
@@ -1241,11 +1314,11 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1241
1314
|
|
1242
1315
|
|
1243
1316
|
class ForEachStage(BaseStage):
|
1244
|
-
"""For-Each
|
1245
|
-
|
1246
|
-
muti-stages in this stage execution.
|
1317
|
+
"""For-Each stage executor that execute all stages with each item in the
|
1318
|
+
foreach list.
|
1247
1319
|
|
1248
|
-
|
1320
|
+
This stage is not the low-level stage model because it runs
|
1321
|
+
muti-stages in this stage execution.
|
1249
1322
|
|
1250
1323
|
Data Validate:
|
1251
1324
|
>>> stage = {
|
@@ -1254,7 +1327,7 @@ class ForEachStage(BaseStage):
|
|
1254
1327
|
... "stages": [
|
1255
1328
|
... {
|
1256
1329
|
... "name": "Echo stage",
|
1257
|
-
... "echo": "Start run with item {{ item }}"
|
1330
|
+
... "echo": "Start run with item ${{ item }}"
|
1258
1331
|
... },
|
1259
1332
|
... ],
|
1260
1333
|
... }
|
@@ -1262,13 +1335,14 @@ class ForEachStage(BaseStage):
|
|
1262
1335
|
|
1263
1336
|
foreach: Union[list[str], list[int], str] = Field(
|
1264
1337
|
description=(
|
1265
|
-
"A items for passing to
|
1338
|
+
"A items for passing to stages via ${{ item }} template parameter."
|
1266
1339
|
),
|
1267
1340
|
)
|
1268
1341
|
stages: list[Stage] = Field(
|
1269
1342
|
default_factory=list,
|
1270
1343
|
description=(
|
1271
|
-
"A list of stage that will run with each item in the foreach
|
1344
|
+
"A list of stage that will run with each item in the `foreach` "
|
1345
|
+
"field."
|
1272
1346
|
),
|
1273
1347
|
)
|
1274
1348
|
concurrent: int = Field(
|
@@ -1283,7 +1357,7 @@ class ForEachStage(BaseStage):
|
|
1283
1357
|
|
1284
1358
|
def execute_item(
|
1285
1359
|
self,
|
1286
|
-
item:
|
1360
|
+
item: StrOrInt,
|
1287
1361
|
params: DictData,
|
1288
1362
|
result: Result,
|
1289
1363
|
*,
|
@@ -1292,18 +1366,16 @@ class ForEachStage(BaseStage):
|
|
1292
1366
|
"""Execute foreach item from list of item.
|
1293
1367
|
|
1294
1368
|
:param item: (str | int) An item that want to execution.
|
1295
|
-
:param params: (DictData) A parameter
|
1296
|
-
|
1297
|
-
:param
|
1298
|
-
|
1299
|
-
:param event: (Event) An event manager that use to track parent execute
|
1300
|
-
was not force stopped.
|
1369
|
+
:param params: (DictData) A parameter data.
|
1370
|
+
:param result: (Result) A Result instance for return context and status.
|
1371
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1372
|
+
execution if it forces stopped by parent execution.
|
1301
1373
|
|
1302
1374
|
:raise StageException: If the stage execution raise errors.
|
1303
1375
|
|
1304
1376
|
:rtype: Result
|
1305
1377
|
"""
|
1306
|
-
result.trace.debug(f"[STAGE]: Execute
|
1378
|
+
result.trace.debug(f"[STAGE]: Execute Item: {item!r}")
|
1307
1379
|
context: DictData = copy.deepcopy(params)
|
1308
1380
|
context.update({"item": item})
|
1309
1381
|
output: DictData = {"item": item, "stages": {}}
|
@@ -1345,8 +1417,18 @@ class ForEachStage(BaseStage):
|
|
1345
1417
|
stage.set_outputs(stage.get_outputs(output), to=context)
|
1346
1418
|
except (StageException, UtilException) as e:
|
1347
1419
|
result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
|
1420
|
+
result.catch(
|
1421
|
+
status=FAILED,
|
1422
|
+
foreach={
|
1423
|
+
item: {
|
1424
|
+
"item": item,
|
1425
|
+
"stages": filter_func(output.pop("stages", {})),
|
1426
|
+
"errors": e.to_dict(),
|
1427
|
+
},
|
1428
|
+
},
|
1429
|
+
)
|
1348
1430
|
raise StageException(
|
1349
|
-
f"Sub-Stage
|
1431
|
+
f"Sub-Stage raise: {e.__class__.__name__}: {e}"
|
1350
1432
|
) from None
|
1351
1433
|
|
1352
1434
|
if rs.status == FAILED:
|
@@ -1383,11 +1465,10 @@ class ForEachStage(BaseStage):
|
|
1383
1465
|
) -> Result:
|
1384
1466
|
"""Execute the stages that pass each item form the foreach field.
|
1385
1467
|
|
1386
|
-
:param params: A parameter
|
1387
|
-
:param result: (Result) A
|
1388
|
-
|
1389
|
-
|
1390
|
-
was not force stopped.
|
1468
|
+
:param params: (DictData) A parameter data.
|
1469
|
+
:param result: (Result) A Result instance for return context and status.
|
1470
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1471
|
+
execution if it forces stopped by parent execution.
|
1391
1472
|
|
1392
1473
|
:rtype: Result
|
1393
1474
|
"""
|
@@ -1403,9 +1484,7 @@ class ForEachStage(BaseStage):
|
|
1403
1484
|
else self.foreach
|
1404
1485
|
)
|
1405
1486
|
if not isinstance(foreach, list):
|
1406
|
-
raise StageException(
|
1407
|
-
f"Foreach does not support foreach value: {foreach!r}"
|
1408
|
-
)
|
1487
|
+
raise StageException(f"Does not support foreach: {foreach!r}")
|
1409
1488
|
|
1410
1489
|
result.trace.info(f"[STAGE]: Foreach-Execute: {foreach!r}.")
|
1411
1490
|
result.catch(status=WAIT, context={"items": foreach, "foreach": {}})
|
@@ -1437,33 +1516,33 @@ class ForEachStage(BaseStage):
|
|
1437
1516
|
context: DictData = {}
|
1438
1517
|
status: Status = SUCCESS
|
1439
1518
|
|
1440
|
-
done, not_done = wait(
|
1441
|
-
futures, timeout=1800, return_when=FIRST_EXCEPTION
|
1442
|
-
)
|
1443
|
-
|
1519
|
+
done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
|
1444
1520
|
if len(done) != len(futures):
|
1445
1521
|
result.trace.warning(
|
1446
|
-
"[STAGE]: Set
|
1522
|
+
"[STAGE]: Set event for stop pending stage future."
|
1447
1523
|
)
|
1448
1524
|
event.set()
|
1449
1525
|
for future in not_done:
|
1450
1526
|
future.cancel()
|
1451
1527
|
|
1528
|
+
nd: str = f", item not run: {not_done}" if not_done else ""
|
1529
|
+
result.trace.debug(f"... Foreach set Fail-Fast{nd}")
|
1530
|
+
|
1452
1531
|
for future in done:
|
1453
1532
|
try:
|
1454
1533
|
future.result()
|
1455
|
-
except StageException as e:
|
1534
|
+
except (StageException, UtilException) as e:
|
1456
1535
|
status = FAILED
|
1457
1536
|
result.trace.error(
|
1458
|
-
f"[STAGE]: {e.__class__.__name__}
|
1537
|
+
f"[STAGE]: {e.__class__.__name__}:{NEWLINE}{e}"
|
1459
1538
|
)
|
1460
1539
|
context.update({"errors": e.to_dict()})
|
1461
|
-
|
1462
1540
|
return result.catch(status=status, context=context)
|
1463
1541
|
|
1464
1542
|
|
1465
|
-
class UntilStage(BaseStage):
|
1466
|
-
"""Until
|
1543
|
+
class UntilStage(BaseStage):
|
1544
|
+
"""Until stage executor that will run stages in each loop until it valid
|
1545
|
+
with stop loop condition.
|
1467
1546
|
|
1468
1547
|
Data Validate:
|
1469
1548
|
>>> stage = {
|
@@ -1485,19 +1564,21 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1485
1564
|
"An initial value that can be any value in str, int, or bool type."
|
1486
1565
|
),
|
1487
1566
|
)
|
1488
|
-
until: str = Field(description="A until condition.")
|
1567
|
+
until: str = Field(description="A until condition for stop the while loop.")
|
1489
1568
|
stages: list[Stage] = Field(
|
1490
1569
|
default_factory=list,
|
1491
1570
|
description=(
|
1492
|
-
"A list of stage that will run with each item until
|
1493
|
-
"correct."
|
1571
|
+
"A list of stage that will run with each item in until loop."
|
1494
1572
|
),
|
1495
1573
|
)
|
1496
1574
|
max_loop: int = Field(
|
1497
1575
|
default=10,
|
1498
1576
|
ge=1,
|
1499
1577
|
lt=100,
|
1500
|
-
description=
|
1578
|
+
description=(
|
1579
|
+
"The maximum value of loop for this until stage. This value should "
|
1580
|
+
"be gather or equal than 1, and less than 100."
|
1581
|
+
),
|
1501
1582
|
alias="max-loop",
|
1502
1583
|
)
|
1503
1584
|
|
@@ -1509,17 +1590,15 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1509
1590
|
result: Result,
|
1510
1591
|
event: Event | None = None,
|
1511
1592
|
) -> tuple[Result, T]:
|
1512
|
-
"""Execute
|
1593
|
+
"""Execute loop item that was set from some stage or set by default loop
|
1513
1594
|
variable.
|
1514
1595
|
|
1515
1596
|
:param item: (T) An item that want to execution.
|
1516
1597
|
:param loop: (int) A number of loop.
|
1517
|
-
:param params: (DictData) A parameter
|
1518
|
-
|
1519
|
-
:param
|
1520
|
-
|
1521
|
-
:param event: (Event) An event manager that use to track parent execute
|
1522
|
-
was not force stopped.
|
1598
|
+
:param params: (DictData) A parameter data.
|
1599
|
+
:param result: (Result) A Result instance for return context and status.
|
1600
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1601
|
+
execution if it forces stopped by parent execution.
|
1523
1602
|
|
1524
1603
|
:rtype: tuple[Result, T]
|
1525
1604
|
"""
|
@@ -1602,11 +1681,10 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1602
1681
|
"""Execute the stages that pass item from until condition field and
|
1603
1682
|
setter step.
|
1604
1683
|
|
1605
|
-
:param params: A parameter
|
1606
|
-
:param result: (Result) A
|
1607
|
-
|
1608
|
-
|
1609
|
-
was not force stopped.
|
1684
|
+
:param params: (DictData) A parameter data.
|
1685
|
+
:param result: (Result) A Result instance for return context and status.
|
1686
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1687
|
+
execution if it forces stopped by parent execution.
|
1610
1688
|
|
1611
1689
|
:rtype: Result
|
1612
1690
|
"""
|
@@ -1679,14 +1757,14 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1679
1757
|
class Match(BaseModel):
|
1680
1758
|
"""Match model for the Case Stage."""
|
1681
1759
|
|
1682
|
-
case:
|
1760
|
+
case: StrOrInt = Field(description="A match case.")
|
1683
1761
|
stages: list[Stage] = Field(
|
1684
1762
|
description="A list of stage to execution for this case."
|
1685
1763
|
)
|
1686
1764
|
|
1687
1765
|
|
1688
1766
|
class CaseStage(BaseStage):
|
1689
|
-
"""Case
|
1767
|
+
"""Case stage executor that execute all stages if the condition was matched.
|
1690
1768
|
|
1691
1769
|
Data Validate:
|
1692
1770
|
>>> stage = {
|
@@ -1742,12 +1820,10 @@ class CaseStage(BaseStage):
|
|
1742
1820
|
|
1743
1821
|
:param case: (str) A case that want to execution.
|
1744
1822
|
:param stages: (list[Stage]) A list of stage.
|
1745
|
-
:param params: (DictData) A parameter
|
1746
|
-
|
1747
|
-
:param
|
1748
|
-
|
1749
|
-
:param event: (Event) An event manager that use to track parent execute
|
1750
|
-
was not force stopped.
|
1823
|
+
:param params: (DictData) A parameter data.
|
1824
|
+
:param result: (Result) A Result instance for return context and status.
|
1825
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1826
|
+
execution if it forces stopped by parent execution.
|
1751
1827
|
|
1752
1828
|
:rtype: Result
|
1753
1829
|
"""
|
@@ -1830,11 +1906,10 @@ class CaseStage(BaseStage):
|
|
1830
1906
|
) -> Result:
|
1831
1907
|
"""Execute case-match condition that pass to the case field.
|
1832
1908
|
|
1833
|
-
:param params: A parameter
|
1834
|
-
:param result: (Result) A
|
1835
|
-
|
1836
|
-
|
1837
|
-
was not force stopped.
|
1909
|
+
:param params: (DictData) A parameter data.
|
1910
|
+
:param result: (Result) A Result instance for return context and status.
|
1911
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1912
|
+
execution if it forces stopped by parent execution.
|
1838
1913
|
|
1839
1914
|
:rtype: Result
|
1840
1915
|
"""
|
@@ -1911,7 +1986,7 @@ class RaiseStage(BaseStage): # pragma: no cov
|
|
1911
1986
|
|
1912
1987
|
message: str = Field(
|
1913
1988
|
description=(
|
1914
|
-
"An error message that want to raise with StageException class"
|
1989
|
+
"An error message that want to raise with `StageException` class"
|
1915
1990
|
),
|
1916
1991
|
alias="raise",
|
1917
1992
|
)
|
@@ -1925,11 +2000,10 @@ class RaiseStage(BaseStage): # pragma: no cov
|
|
1925
2000
|
) -> Result:
|
1926
2001
|
"""Raise the StageException object with the message field execution.
|
1927
2002
|
|
1928
|
-
:param params: A parameter
|
1929
|
-
:param result: (Result) A
|
1930
|
-
|
1931
|
-
|
1932
|
-
was not force stopped.
|
2003
|
+
:param params: (DictData) A parameter data.
|
2004
|
+
:param result: (Result) A Result instance for return context and status.
|
2005
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
2006
|
+
execution if it forces stopped by parent execution.
|
1933
2007
|
"""
|
1934
2008
|
if result is None: # pragma: no cov
|
1935
2009
|
result: Result = Result(
|
@@ -1941,27 +2015,13 @@ class RaiseStage(BaseStage): # pragma: no cov
|
|
1941
2015
|
raise StageException(message)
|
1942
2016
|
|
1943
2017
|
|
1944
|
-
# TODO: Not implement this stages yet
|
1945
|
-
class HookStage(BaseStage): # pragma: no cov
|
1946
|
-
"""Hook stage execution."""
|
1947
|
-
|
1948
|
-
hook: str
|
1949
|
-
args: DictData = Field(default_factory=dict)
|
1950
|
-
callback: str
|
1951
|
-
|
1952
|
-
def execute(
|
1953
|
-
self,
|
1954
|
-
params: DictData,
|
1955
|
-
*,
|
1956
|
-
result: Result | None = None,
|
1957
|
-
event: Event | None = None,
|
1958
|
-
) -> Result:
|
1959
|
-
raise NotImplementedError("Hook Stage does not implement yet.")
|
1960
|
-
|
1961
|
-
|
1962
|
-
# TODO: Not implement this stages yet
|
1963
2018
|
class DockerStage(BaseStage): # pragma: no cov
|
1964
|
-
"""Docker container stage execution
|
2019
|
+
"""Docker container stage execution that will pull the specific Docker image
|
2020
|
+
with custom authentication and run this image by passing environment
|
2021
|
+
variables and mounting local volume to this Docker container.
|
2022
|
+
|
2023
|
+
The volume path that mount to this Docker container will limit. That is
|
2024
|
+
this stage does not allow you to mount any path to this container.
|
1965
2025
|
|
1966
2026
|
Data Validate:
|
1967
2027
|
>>> stage = {
|
@@ -1969,10 +2029,7 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
1969
2029
|
... "image": "image-name.pkg.com",
|
1970
2030
|
... "env": {
|
1971
2031
|
... "ENV": "dev",
|
1972
|
-
... "
|
1973
|
-
... },
|
1974
|
-
... "volume": {
|
1975
|
-
... "secrets": "/secrets",
|
2032
|
+
... "SECRET": "${SPECIFIC_SECRET}",
|
1976
2033
|
... },
|
1977
2034
|
... "auth": {
|
1978
2035
|
... "username": "__json_key",
|
@@ -1985,8 +2042,16 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
1985
2042
|
description="A Docker image url with tag that want to run.",
|
1986
2043
|
)
|
1987
2044
|
tag: str = Field(default="latest", description="An Docker image tag.")
|
1988
|
-
env: DictData = Field(
|
1989
|
-
|
2045
|
+
env: DictData = Field(
|
2046
|
+
default_factory=dict,
|
2047
|
+
description=(
|
2048
|
+
"An environment variable that want pass to Docker container.",
|
2049
|
+
),
|
2050
|
+
)
|
2051
|
+
volume: DictData = Field(
|
2052
|
+
default_factory=dict,
|
2053
|
+
description="A mapping of local and target mounting path.",
|
2054
|
+
)
|
1990
2055
|
auth: DictData = Field(
|
1991
2056
|
default_factory=dict,
|
1992
2057
|
description=(
|
@@ -1998,7 +2063,17 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
1998
2063
|
self,
|
1999
2064
|
params: DictData,
|
2000
2065
|
result: Result,
|
2001
|
-
|
2066
|
+
event: Event | None = None,
|
2067
|
+
) -> Result:
|
2068
|
+
"""Execute Docker container task.
|
2069
|
+
|
2070
|
+
:param params: (DictData) A parameter data.
|
2071
|
+
:param result: (Result) A Result instance for return context and status.
|
2072
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
2073
|
+
execution if it forces stopped by parent execution.
|
2074
|
+
|
2075
|
+
:rtype: Result
|
2076
|
+
"""
|
2002
2077
|
from docker import DockerClient
|
2003
2078
|
from docker.errors import ContainerError
|
2004
2079
|
|
@@ -2016,6 +2091,16 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2016
2091
|
for line in resp:
|
2017
2092
|
result.trace.info(f"... {line}")
|
2018
2093
|
|
2094
|
+
if event and event.is_set():
|
2095
|
+
error_msg: str = (
|
2096
|
+
"Docker-Stage was canceled from event that had set before "
|
2097
|
+
"run the Docker container."
|
2098
|
+
)
|
2099
|
+
return result.catch(
|
2100
|
+
status=CANCEL,
|
2101
|
+
context={"errors": StageException(error_msg).to_dict()},
|
2102
|
+
)
|
2103
|
+
|
2019
2104
|
unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S%f}"
|
2020
2105
|
container = client.containers.run(
|
2021
2106
|
image=f"{self.image}:{self.tag}",
|
@@ -2054,6 +2139,13 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2054
2139
|
f"{self.image}:{self.tag}",
|
2055
2140
|
out,
|
2056
2141
|
)
|
2142
|
+
output_file: Path = Path(f".docker.{result.run_id}.logs/outputs.json")
|
2143
|
+
if not output_file.exists():
|
2144
|
+
return result.catch(status=SUCCESS)
|
2145
|
+
|
2146
|
+
with output_file.open(mode="rt") as f:
|
2147
|
+
data = json.load(f)
|
2148
|
+
return result.catch(status=SUCCESS, context=data)
|
2057
2149
|
|
2058
2150
|
def execute(
|
2059
2151
|
self,
|
@@ -2062,13 +2154,34 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2062
2154
|
result: Result | None = None,
|
2063
2155
|
event: Event | None = None,
|
2064
2156
|
) -> Result:
|
2157
|
+
"""Execute the Docker image via Python API.
|
2158
|
+
|
2159
|
+
:param params: (DictData) A parameter data.
|
2160
|
+
:param result: (Result) A Result instance for return context and status.
|
2161
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
2162
|
+
execution if it forces stopped by parent execution.
|
2163
|
+
|
2164
|
+
:rtype: Result
|
2165
|
+
"""
|
2166
|
+
if result is None: # pragma: no cov
|
2167
|
+
result: Result = Result(
|
2168
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
2169
|
+
)
|
2170
|
+
|
2171
|
+
result.trace.info(f"[STAGE]: Docker-Execute: {self.image}:{self.tag}")
|
2172
|
+
|
2065
2173
|
raise NotImplementedError("Docker Stage does not implement yet.")
|
2066
2174
|
|
2067
2175
|
|
2068
|
-
# TODO: Not implement this stages yet
|
2069
2176
|
class VirtualPyStage(PyStage): # pragma: no cov
|
2070
|
-
"""
|
2177
|
+
"""Virtual Python stage executor that run Python statement on the dependent
|
2178
|
+
Python virtual environment via the `uv` package.
|
2179
|
+
"""
|
2071
2180
|
|
2181
|
+
version: str = Field(
|
2182
|
+
default="3.9",
|
2183
|
+
description="A Python version that want to run.",
|
2184
|
+
)
|
2072
2185
|
deps: list[str] = Field(
|
2073
2186
|
description=(
|
2074
2187
|
"list of Python dependency that want to install before execution "
|
@@ -2076,7 +2189,54 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
2076
2189
|
),
|
2077
2190
|
)
|
2078
2191
|
|
2079
|
-
|
2192
|
+
@contextlib.contextmanager
|
2193
|
+
def create_py_file(
|
2194
|
+
self,
|
2195
|
+
py: str,
|
2196
|
+
values: DictData,
|
2197
|
+
deps: list[str],
|
2198
|
+
run_id: str | None = None,
|
2199
|
+
) -> Iterator[str]:
|
2200
|
+
"""Create the .py file with an input Python string statement.
|
2201
|
+
|
2202
|
+
:param py: A Python string statement.
|
2203
|
+
:param values: A variable that want to set before running this
|
2204
|
+
:param deps: An additional Python dependencies that want install before
|
2205
|
+
run this python stage.
|
2206
|
+
:param run_id: (str | None) A running ID of this stage execution.
|
2207
|
+
"""
|
2208
|
+
run_id: str = run_id or uuid.uuid4()
|
2209
|
+
f_name: str = f"{run_id}.py"
|
2210
|
+
with open(f"./{f_name}", mode="w", newline="\n") as f:
|
2211
|
+
# NOTE: Create variable mapping that write before running statement.
|
2212
|
+
vars_str: str = "\n ".join(
|
2213
|
+
f"{var} = {value!r}" for var, value in values.items()
|
2214
|
+
)
|
2215
|
+
|
2216
|
+
# NOTE: uv supports PEP 723 — inline TOML metadata.
|
2217
|
+
f.write(
|
2218
|
+
dedent(
|
2219
|
+
f"""
|
2220
|
+
# /// script
|
2221
|
+
# dependencies = [{', '.join(f'"{dep}"' for dep in deps)}]
|
2222
|
+
# ///
|
2223
|
+
{vars_str}
|
2224
|
+
""".strip(
|
2225
|
+
"\n"
|
2226
|
+
)
|
2227
|
+
)
|
2228
|
+
)
|
2229
|
+
|
2230
|
+
# NOTE: make sure that py script file does not have `\r` char.
|
2231
|
+
f.write("\n" + py.replace("\r\n", "\n"))
|
2232
|
+
|
2233
|
+
# NOTE: Make this .py file able to executable.
|
2234
|
+
make_exec(f"./{f_name}")
|
2235
|
+
|
2236
|
+
yield f_name
|
2237
|
+
|
2238
|
+
# Note: Remove .py file that use to run Python.
|
2239
|
+
Path(f"./{f_name}").unlink()
|
2080
2240
|
|
2081
2241
|
def execute(
|
2082
2242
|
self,
|
@@ -2088,15 +2248,13 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
2088
2248
|
"""Execute the Python statement via Python virtual environment.
|
2089
2249
|
|
2090
2250
|
Steps:
|
2091
|
-
- Create python file.
|
2092
|
-
-
|
2093
|
-
- Execution python file with uv and specific `.venv`.
|
2251
|
+
- Create python file with the `uv` syntax.
|
2252
|
+
- Execution python file with `uv run` via Python subprocess module.
|
2094
2253
|
|
2095
|
-
:param params: A parameter
|
2096
|
-
:param result: (Result) A
|
2097
|
-
|
2098
|
-
|
2099
|
-
was not force stopped.
|
2254
|
+
:param params: (DictData) A parameter data.
|
2255
|
+
:param result: (Result) A Result instance for return context and status.
|
2256
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
2257
|
+
execution if it forces stopped by parent execution.
|
2100
2258
|
|
2101
2259
|
:rtype: Result
|
2102
2260
|
"""
|
@@ -2106,13 +2264,45 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
2106
2264
|
)
|
2107
2265
|
|
2108
2266
|
result.trace.info(f"[STAGE]: Py-Virtual-Execute: {self.name}")
|
2109
|
-
|
2110
|
-
|
2267
|
+
run: str = param2template(dedent(self.run), params, extras=self.extras)
|
2268
|
+
with self.create_py_file(
|
2269
|
+
py=run,
|
2270
|
+
values=param2template(self.vars, params, extras=self.extras),
|
2271
|
+
deps=param2template(self.deps, params, extras=self.extras),
|
2272
|
+
run_id=result.run_id,
|
2273
|
+
) as py:
|
2274
|
+
result.trace.debug(f"... Create `{py}` file.")
|
2275
|
+
rs: CompletedProcess = subprocess.run(
|
2276
|
+
["uv", "run", py, "--no-cache"],
|
2277
|
+
# ["uv", "run", "--python", "3.9", py],
|
2278
|
+
shell=False,
|
2279
|
+
capture_output=True,
|
2280
|
+
text=True,
|
2281
|
+
)
|
2282
|
+
|
2283
|
+
if rs.returncode > 0:
|
2284
|
+
# NOTE: Prepare stderr message that returning from subprocess.
|
2285
|
+
e: str = (
|
2286
|
+
rs.stderr.encode("utf-8").decode("utf-16")
|
2287
|
+
if "\\x00" in rs.stderr
|
2288
|
+
else rs.stderr
|
2289
|
+
).removesuffix("\n")
|
2290
|
+
raise StageException(
|
2291
|
+
f"Subprocess: {e}\nRunning Statement:\n---\n"
|
2292
|
+
f"```python\n{run}\n```"
|
2293
|
+
)
|
2294
|
+
return result.catch(
|
2295
|
+
status=SUCCESS,
|
2296
|
+
context={
|
2297
|
+
"return_code": rs.returncode,
|
2298
|
+
"stdout": None if (out := rs.stdout.strip("\n")) == "" else out,
|
2299
|
+
"stderr": None if (out := rs.stderr.strip("\n")) == "" else out,
|
2300
|
+
},
|
2111
2301
|
)
|
2112
2302
|
|
2113
2303
|
|
2114
2304
|
# NOTE:
|
2115
|
-
# An order of parsing stage model on the Job model with
|
2305
|
+
# An order of parsing stage model on the Job model with `stages` field.
|
2116
2306
|
# From the current build-in stages, they do not have stage that have the same
|
2117
2307
|
# fields that because of parsing on the Job's stages key.
|
2118
2308
|
#
|
@@ -2121,7 +2311,6 @@ Stage = Annotated[
|
|
2121
2311
|
DockerStage,
|
2122
2312
|
BashStage,
|
2123
2313
|
CallStage,
|
2124
|
-
HookStage,
|
2125
2314
|
TriggerStage,
|
2126
2315
|
ForEachStage,
|
2127
2316
|
UntilStage,
|