ddeutil-workflow 0.0.72__py3-none-any.whl → 0.0.74__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__cron.py +14 -8
- ddeutil/workflow/__init__.py +119 -10
- ddeutil/workflow/__types.py +53 -41
- ddeutil/workflow/api/__init__.py +74 -3
- ddeutil/workflow/api/routes/job.py +15 -29
- ddeutil/workflow/api/routes/logs.py +9 -9
- ddeutil/workflow/api/routes/workflows.py +3 -3
- ddeutil/workflow/audits.py +70 -55
- ddeutil/workflow/cli.py +1 -15
- ddeutil/workflow/conf.py +78 -26
- ddeutil/workflow/errors.py +86 -19
- ddeutil/workflow/event.py +268 -169
- ddeutil/workflow/job.py +331 -192
- ddeutil/workflow/params.py +37 -7
- ddeutil/workflow/result.py +96 -70
- ddeutil/workflow/reusables.py +56 -6
- ddeutil/workflow/stages.py +1088 -575
- ddeutil/workflow/traces.py +218 -128
- ddeutil/workflow/utils.py +60 -8
- ddeutil/workflow/workflow.py +424 -290
- {ddeutil_workflow-0.0.72.dist-info → ddeutil_workflow-0.0.74.dist-info}/METADATA +27 -17
- ddeutil_workflow-0.0.74.dist-info/RECORD +30 -0
- ddeutil_workflow-0.0.72.dist-info/RECORD +0 -30
- {ddeutil_workflow-0.0.72.dist-info → ddeutil_workflow-0.0.74.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.72.dist-info → ddeutil_workflow-0.0.74.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.72.dist-info → ddeutil_workflow-0.0.74.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.72.dist-info → ddeutil_workflow-0.0.74.dist-info}/top_level.txt +0 -0
ddeutil/workflow/stages.py
CHANGED
@@ -12,7 +12,7 @@ you can track logs.
|
|
12
12
|
I do not want to handle stage error on this stage execution. I think stage model
|
13
13
|
have a lot of use-case, and it should does not worry about it error output.
|
14
14
|
|
15
|
-
So, I will create `
|
15
|
+
So, I will create `execute` for any exception class that raise from
|
16
16
|
the stage execution method.
|
17
17
|
|
18
18
|
Handler --> Ok --> Result
|
@@ -32,7 +32,7 @@ the stage execution method.
|
|
32
32
|
|-name: ...
|
33
33
|
╰-message: ...
|
34
34
|
|
35
|
-
On the context I/O that pass to a stage object at execute
|
35
|
+
On the context I/O that pass to a stage object at execute step. The
|
36
36
|
execute method receives a `params={"params": {...}}` value for passing template
|
37
37
|
searching.
|
38
38
|
|
@@ -95,6 +95,7 @@ from .result import (
|
|
95
95
|
WAIT,
|
96
96
|
Result,
|
97
97
|
Status,
|
98
|
+
catch,
|
98
99
|
get_status_from_error,
|
99
100
|
validate_statuses,
|
100
101
|
)
|
@@ -105,6 +106,7 @@ from .reusables import (
|
|
105
106
|
not_in_template,
|
106
107
|
param2template,
|
107
108
|
)
|
109
|
+
from .traces import Trace, get_trace
|
108
110
|
from .utils import (
|
109
111
|
delay,
|
110
112
|
dump_all,
|
@@ -119,13 +121,40 @@ DictOrModel = Union[DictData, BaseModel]
|
|
119
121
|
|
120
122
|
|
121
123
|
class BaseStage(BaseModel, ABC):
|
122
|
-
"""
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
124
|
+
"""Abstract base class for all stage implementations.
|
125
|
+
|
126
|
+
BaseStage provides the foundation for all stage types in the workflow system.
|
127
|
+
It defines the common interface and metadata fields that all stages must
|
128
|
+
implement, ensuring consistent behavior across different stage types.
|
129
|
+
|
130
|
+
This abstract class handles core stage functionality including:
|
131
|
+
- Stage identification and naming
|
132
|
+
- Conditional execution logic
|
133
|
+
- Output management and templating
|
134
|
+
- Execution lifecycle management
|
135
|
+
|
136
|
+
Custom stages should inherit from this class and implement the abstract
|
137
|
+
`process()` method to define their specific execution behavior.
|
138
|
+
|
139
|
+
Attributes:
|
140
|
+
extras (dict): Additional configuration parameters
|
141
|
+
id (str, optional): Unique stage identifier for output reference
|
142
|
+
name (str): Human-readable stage name for logging
|
143
|
+
desc (str, optional): Stage description for documentation
|
144
|
+
condition (str, optional): Conditional expression for execution
|
145
|
+
|
146
|
+
Abstract Methods:
|
147
|
+
process: Main execution logic that must be implemented by subclasses
|
148
|
+
|
149
|
+
Example:
|
150
|
+
```python
|
151
|
+
class CustomStage(BaseStage):
|
152
|
+
custom_param: str = Field(description="Custom parameter")
|
153
|
+
|
154
|
+
def process(self, params: dict, **kwargs) -> Result:
|
155
|
+
# Custom execution logic
|
156
|
+
return Result(status=SUCCESS)
|
157
|
+
```
|
129
158
|
"""
|
130
159
|
|
131
160
|
extras: DictData = Field(
|
@@ -162,7 +191,8 @@ class BaseStage(BaseModel, ABC):
|
|
162
191
|
"""Return this stage identity that return the `id` field first and if
|
163
192
|
this `id` field does not set, it will use the `name` field instead.
|
164
193
|
|
165
|
-
:
|
194
|
+
Returns:
|
195
|
+
str: Return an identity of this stage for making output.
|
166
196
|
"""
|
167
197
|
return self.id or self.name
|
168
198
|
|
@@ -194,37 +224,40 @@ class BaseStage(BaseModel, ABC):
|
|
194
224
|
return self
|
195
225
|
|
196
226
|
@abstractmethod
|
197
|
-
def
|
227
|
+
def process(
|
198
228
|
self,
|
199
229
|
params: DictData,
|
230
|
+
run_id: str,
|
231
|
+
context: DictData,
|
200
232
|
*,
|
201
|
-
|
233
|
+
parent_run_id: Optional[str] = None,
|
202
234
|
event: Optional[Event] = None,
|
203
235
|
) -> Result:
|
204
|
-
"""
|
236
|
+
"""Process abstraction method that action something by sub-model class.
|
205
237
|
This is important method that make this class is able to be the stage.
|
206
238
|
|
207
|
-
:
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
239
|
+
Args:
|
240
|
+
params: A parameter data that want to use in this
|
241
|
+
execution.
|
242
|
+
run_id: A running stage ID.
|
243
|
+
context: A context data.
|
244
|
+
parent_run_id: A parent running ID. (Default is None)
|
245
|
+
event: An event manager that use to track parent process
|
246
|
+
was not force stopped.
|
247
|
+
|
248
|
+
Returns:
|
249
|
+
Result: The execution result with status and context data.
|
215
250
|
"""
|
216
|
-
raise NotImplementedError("Stage should implement `
|
251
|
+
raise NotImplementedError("Stage should implement `process` method.")
|
217
252
|
|
218
|
-
def
|
253
|
+
def execute(
|
219
254
|
self,
|
220
255
|
params: DictData,
|
221
256
|
*,
|
222
257
|
run_id: StrOrNone = None,
|
223
|
-
parent_run_id: StrOrNone = None,
|
224
|
-
result: Optional[Result] = None,
|
225
258
|
event: Optional[Event] = None,
|
226
259
|
) -> Union[Result, DictData]:
|
227
|
-
"""Handler stage execution result from the stage `
|
260
|
+
"""Handler stage execution result from the stage `process` method.
|
228
261
|
|
229
262
|
This handler strategy will catch and mapping message to the result
|
230
263
|
context data before returning. All possible status that will return from
|
@@ -250,33 +283,36 @@ class BaseStage(BaseModel, ABC):
|
|
250
283
|
On the last step, it will set the running ID on a return result
|
251
284
|
object from the current stage ID before release the final result.
|
252
285
|
|
253
|
-
:
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
(Default is None)
|
259
|
-
:param event: (Event) An event manager that pass to the stage execution.
|
260
|
-
(Default is None)
|
286
|
+
Args:
|
287
|
+
params: A parameter data.
|
288
|
+
run_id: A running stage ID. (Default is None)
|
289
|
+
event: An event manager that pass to the stage execution.
|
290
|
+
(Default is None)
|
261
291
|
|
262
|
-
:
|
292
|
+
Returns:
|
293
|
+
Result: The execution result with updated status and context.
|
263
294
|
"""
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
extras=self.extras
|
295
|
+
ts: float = time.monotonic()
|
296
|
+
parent_run_id: str = run_id
|
297
|
+
run_id: str = run_id or gen_id(self.iden, unique=True)
|
298
|
+
context: DictData = {}
|
299
|
+
trace: Trace = get_trace(
|
300
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
270
301
|
)
|
271
302
|
try:
|
272
|
-
|
303
|
+
_id: str = (
|
304
|
+
f" with ID: {param2template(self.id, params=params)!r}"
|
305
|
+
if self.id
|
306
|
+
else ""
|
307
|
+
)
|
308
|
+
trace.info(
|
273
309
|
f"[STAGE]: Handler {to_train(self.__class__.__name__)}: "
|
274
|
-
f"{self.name!r}."
|
310
|
+
f"{self.name!r}{_id}."
|
275
311
|
)
|
276
312
|
|
277
313
|
# NOTE: Show the description of this stage before execution.
|
278
314
|
if self.desc:
|
279
|
-
|
315
|
+
trace.debug(f"[STAGE]: Description:||{self.desc}||")
|
280
316
|
|
281
317
|
# VALIDATE: Checking stage condition before execution.
|
282
318
|
if self.is_skipped(params):
|
@@ -287,13 +323,19 @@ class BaseStage(BaseModel, ABC):
|
|
287
323
|
# NOTE: Start call wrapped execution method that will use custom
|
288
324
|
# execution before the real execution from inherit stage model.
|
289
325
|
result_caught: Result = self._execute(
|
290
|
-
params,
|
326
|
+
params,
|
327
|
+
run_id=run_id,
|
328
|
+
context=context,
|
329
|
+
parent_run_id=parent_run_id,
|
330
|
+
event=event,
|
291
331
|
)
|
292
332
|
if result_caught.status == WAIT:
|
293
333
|
raise StageError(
|
294
334
|
"Status from execution should not return waiting status."
|
295
335
|
)
|
296
|
-
return result_caught
|
336
|
+
return result_caught.make_info(
|
337
|
+
{"execution_time": time.monotonic() - ts}
|
338
|
+
)
|
297
339
|
|
298
340
|
# NOTE: Catch this error in this line because the execution can raise
|
299
341
|
# this exception class at other location.
|
@@ -302,48 +344,82 @@ class BaseStage(BaseModel, ABC):
|
|
302
344
|
StageCancelError,
|
303
345
|
StageError,
|
304
346
|
) as e: # pragma: no cov
|
305
|
-
|
347
|
+
trace.info(
|
306
348
|
f"[STAGE]: Handler:||{e.__class__.__name__}: {e}||"
|
307
349
|
f"{traceback.format_exc()}"
|
308
350
|
)
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
351
|
+
st: Status = get_status_from_error(e)
|
352
|
+
return Result(
|
353
|
+
run_id=run_id,
|
354
|
+
parent_run_id=parent_run_id,
|
355
|
+
status=st,
|
356
|
+
context=catch(
|
357
|
+
context,
|
358
|
+
status=st,
|
359
|
+
updated=(
|
360
|
+
None
|
361
|
+
if isinstance(e, StageSkipError)
|
362
|
+
else {"errors": e.to_dict()}
|
363
|
+
),
|
315
364
|
),
|
365
|
+
info={"execution_time": time.monotonic() - ts},
|
366
|
+
extras=self.extras,
|
316
367
|
)
|
317
368
|
except Exception as e:
|
318
|
-
|
369
|
+
trace.error(
|
319
370
|
f"[STAGE]: Error Handler:||{e.__class__.__name__}: {e}||"
|
320
371
|
f"{traceback.format_exc()}"
|
321
372
|
)
|
322
|
-
return
|
373
|
+
return Result(
|
374
|
+
run_id=run_id,
|
375
|
+
parent_run_id=parent_run_id,
|
376
|
+
status=FAILED,
|
377
|
+
context=catch(
|
378
|
+
context, status=FAILED, updated={"errors": to_dict(e)}
|
379
|
+
),
|
380
|
+
info={"execution_time": time.monotonic() - ts},
|
381
|
+
extras=self.extras,
|
382
|
+
)
|
323
383
|
|
324
384
|
def _execute(
|
325
|
-
self,
|
385
|
+
self,
|
386
|
+
params: DictData,
|
387
|
+
run_id: str,
|
388
|
+
context: DictData,
|
389
|
+
parent_run_id: Optional[str] = None,
|
390
|
+
event: Optional[Event] = None,
|
326
391
|
) -> Result:
|
327
|
-
"""Wrapped the
|
392
|
+
"""Wrapped the process method before returning to handler execution.
|
328
393
|
|
329
|
-
:
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
was not force stopped.
|
394
|
+
Args:
|
395
|
+
params: A parameter data that want to use in this
|
396
|
+
execution.
|
397
|
+
event: An event manager that use to track parent process
|
398
|
+
was not force stopped.
|
335
399
|
|
336
|
-
:
|
400
|
+
Returns:
|
401
|
+
Result: The wrapped execution result.
|
337
402
|
"""
|
338
|
-
|
339
|
-
return self.
|
403
|
+
catch(context, status=WAIT)
|
404
|
+
return self.process(
|
405
|
+
params,
|
406
|
+
run_id=run_id,
|
407
|
+
context=context,
|
408
|
+
parent_run_id=parent_run_id,
|
409
|
+
event=event,
|
410
|
+
)
|
340
411
|
|
341
|
-
def set_outputs(
|
412
|
+
def set_outputs(
|
413
|
+
self,
|
414
|
+
output: DictData,
|
415
|
+
to: DictData,
|
416
|
+
info: Optional[DictData] = None,
|
417
|
+
) -> DictData:
|
342
418
|
"""Set an outputs from execution result context to the received context
|
343
419
|
with a `to` input parameter. The result context from stage execution
|
344
420
|
will be set with `outputs` key in this stage ID key.
|
345
421
|
|
346
|
-
For example of setting output method, If you receive
|
422
|
+
For example of setting output method, If you receive process output
|
347
423
|
and want to set on the `to` like;
|
348
424
|
|
349
425
|
... (i) output: {'foo': 'bar', 'skipped': True}
|
@@ -374,6 +450,7 @@ class BaseStage(BaseModel, ABC):
|
|
374
450
|
:param output: (DictData) A result data context that want to extract
|
375
451
|
and transfer to the `outputs` key in receive context.
|
376
452
|
:param to: (DictData) A received context data.
|
453
|
+
:param info: (DictData)
|
377
454
|
|
378
455
|
:rtype: DictData
|
379
456
|
"""
|
@@ -393,7 +470,8 @@ class BaseStage(BaseModel, ABC):
|
|
393
470
|
status: dict[str, Status] = (
|
394
471
|
{"status": output.pop("status")} if "status" in output else {}
|
395
472
|
)
|
396
|
-
|
473
|
+
info: DictData = {"info": info} if info else {}
|
474
|
+
to["stages"][_id] = {"outputs": output} | errors | status | info
|
397
475
|
return to
|
398
476
|
|
399
477
|
def get_outputs(self, output: DictData) -> DictData:
|
@@ -460,6 +538,7 @@ class BaseStage(BaseModel, ABC):
|
|
460
538
|
param2template(self.id, params=params, extras=self.extras)
|
461
539
|
if self.id
|
462
540
|
else gen_id(
|
541
|
+
# NOTE: The name should be non-sensitive case for uniqueness.
|
463
542
|
param2template(self.name, params=params, extras=self.extras)
|
464
543
|
)
|
465
544
|
)
|
@@ -491,67 +570,74 @@ class BaseAsyncStage(BaseStage, ABC):
|
|
491
570
|
"""
|
492
571
|
|
493
572
|
@abstractmethod
|
494
|
-
async def
|
573
|
+
async def async_process(
|
495
574
|
self,
|
496
575
|
params: DictData,
|
576
|
+
run_id: str,
|
577
|
+
context: DictData,
|
497
578
|
*,
|
498
|
-
|
579
|
+
parent_run_id: Optional[str] = None,
|
499
580
|
event: Optional[Event] = None,
|
500
581
|
) -> Result:
|
501
582
|
"""Async execution method for this Empty stage that only logging out to
|
502
583
|
stdout.
|
503
584
|
|
504
|
-
:
|
505
|
-
|
506
|
-
|
507
|
-
|
508
|
-
|
509
|
-
|
510
|
-
|
511
|
-
|
585
|
+
Args:
|
586
|
+
params: A parameter data that want to use in this
|
587
|
+
execution.
|
588
|
+
run_id: A running stage ID.
|
589
|
+
context: A context data.
|
590
|
+
parent_run_id: A parent running ID. (Default is None)
|
591
|
+
event: An event manager that use to track parent process
|
592
|
+
was not force stopped.
|
593
|
+
|
594
|
+
Returns:
|
595
|
+
Result: The execution result with status and context data.
|
512
596
|
"""
|
513
597
|
raise NotImplementedError(
|
514
598
|
"Async Stage should implement `axecute` method."
|
515
599
|
)
|
516
600
|
|
517
|
-
async def
|
601
|
+
async def axecute(
|
518
602
|
self,
|
519
603
|
params: DictData,
|
520
604
|
*,
|
521
605
|
run_id: StrOrNone = None,
|
522
|
-
parent_run_id: StrOrNone = None,
|
523
|
-
result: Optional[Result] = None,
|
524
606
|
event: Optional[Event] = None,
|
525
607
|
) -> Result:
|
526
608
|
"""Async Handler stage execution result from the stage `execute` method.
|
527
609
|
|
528
|
-
:
|
529
|
-
|
530
|
-
|
531
|
-
|
532
|
-
|
533
|
-
|
610
|
+
Args:
|
611
|
+
params: A parameter data that want to use in this
|
612
|
+
execution.
|
613
|
+
run_id: A running stage ID. (Default is None)
|
614
|
+
event: An event manager that use to track parent process
|
615
|
+
was not force stopped.
|
534
616
|
|
535
|
-
:
|
617
|
+
Returns:
|
618
|
+
Result: The execution result with status and context data.
|
536
619
|
"""
|
537
|
-
|
538
|
-
|
539
|
-
|
540
|
-
|
541
|
-
|
542
|
-
extras=self.extras
|
620
|
+
ts: float = time.monotonic()
|
621
|
+
parent_run_id: StrOrNone = run_id
|
622
|
+
run_id: str = run_id or gen_id(self.iden, unique=True)
|
623
|
+
context: DictData = {}
|
624
|
+
trace: Trace = get_trace(
|
625
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
543
626
|
)
|
544
627
|
try:
|
545
|
-
|
628
|
+
_id: str = (
|
629
|
+
f" with ID: {param2template(self.id, params=params)!r}"
|
630
|
+
if self.id
|
631
|
+
else ""
|
632
|
+
)
|
633
|
+
await trace.ainfo(
|
546
634
|
f"[STAGE]: Handler {to_train(self.__class__.__name__)}: "
|
547
|
-
f"{self.name!r}."
|
635
|
+
f"{self.name!r}{_id}."
|
548
636
|
)
|
549
637
|
|
550
638
|
# NOTE: Show the description of this stage before execution.
|
551
639
|
if self.desc:
|
552
|
-
await
|
553
|
-
f"[STAGE]: Description:||{self.desc}||"
|
554
|
-
)
|
640
|
+
await trace.adebug(f"[STAGE]: Description:||{self.desc}||")
|
555
641
|
|
556
642
|
# VALIDATE: Checking stage condition before execution.
|
557
643
|
if self.is_skipped(params=params):
|
@@ -562,7 +648,11 @@ class BaseAsyncStage(BaseStage, ABC):
|
|
562
648
|
# NOTE: Start call wrapped execution method that will use custom
|
563
649
|
# execution before the real execution from inherit stage model.
|
564
650
|
result_caught: Result = await self._axecute(
|
565
|
-
params,
|
651
|
+
params,
|
652
|
+
run_id=run_id,
|
653
|
+
context=context,
|
654
|
+
parent_run_id=parent_run_id,
|
655
|
+
event=event,
|
566
656
|
)
|
567
657
|
if result_caught.status == WAIT:
|
568
658
|
raise StageError(
|
@@ -577,41 +667,68 @@ class BaseAsyncStage(BaseStage, ABC):
|
|
577
667
|
StageCancelError,
|
578
668
|
StageError,
|
579
669
|
) as e: # pragma: no cov
|
580
|
-
await
|
670
|
+
await trace.ainfo(
|
581
671
|
f"[STAGE]: Skip Handler:||{e.__class__.__name__}: {e}||"
|
582
672
|
f"{traceback.format_exc()}"
|
583
673
|
)
|
584
|
-
|
585
|
-
|
586
|
-
|
587
|
-
|
588
|
-
|
589
|
-
|
674
|
+
st: Status = get_status_from_error(e)
|
675
|
+
return Result(
|
676
|
+
run_id=run_id,
|
677
|
+
parent_run_id=parent_run_id,
|
678
|
+
status=st,
|
679
|
+
context=catch(
|
680
|
+
context,
|
681
|
+
status=st,
|
682
|
+
updated=(
|
683
|
+
None
|
684
|
+
if isinstance(e, StageSkipError)
|
685
|
+
else {"status": st, "errors": e.to_dict()}
|
686
|
+
),
|
590
687
|
),
|
688
|
+
info={"execution_time": time.monotonic() - ts},
|
689
|
+
extras=self.extras,
|
591
690
|
)
|
592
691
|
except Exception as e:
|
593
|
-
await
|
692
|
+
await trace.aerror(
|
594
693
|
f"[STAGE]: Error Handler:||{e.__class__.__name__}: {e}||"
|
595
694
|
f"{traceback.format_exc()}"
|
596
695
|
)
|
597
|
-
return
|
696
|
+
return Result(
|
697
|
+
run_id=run_id,
|
698
|
+
parent_run_id=parent_run_id,
|
699
|
+
status=FAILED,
|
700
|
+
context=catch(
|
701
|
+
context, status=FAILED, updated={"errors": to_dict(e)}
|
702
|
+
),
|
703
|
+
info={"execution_time": time.monotonic() - ts},
|
704
|
+
extras=self.extras,
|
705
|
+
)
|
598
706
|
|
599
707
|
async def _axecute(
|
600
|
-
self,
|
708
|
+
self,
|
709
|
+
params: DictData,
|
710
|
+
run_id: str,
|
711
|
+
context: DictData,
|
712
|
+
parent_run_id: Optional[str] = None,
|
713
|
+
event: Optional[Event] = None,
|
601
714
|
) -> Result:
|
602
715
|
"""Wrapped the axecute method before returning to handler axecute.
|
603
716
|
|
604
717
|
:param params: (DictData) A parameter data that want to use in this
|
605
718
|
execution.
|
606
|
-
:param result: (Result) A result object for keeping context and status
|
607
|
-
data.
|
608
719
|
:param event: (Event) An event manager that use to track parent execute
|
609
720
|
was not force stopped.
|
610
721
|
|
611
722
|
:rtype: Result
|
612
723
|
"""
|
613
|
-
|
614
|
-
return await self.
|
724
|
+
catch(context, status=WAIT)
|
725
|
+
return await self.async_process(
|
726
|
+
params,
|
727
|
+
run_id=run_id,
|
728
|
+
context=context,
|
729
|
+
parent_run_id=parent_run_id,
|
730
|
+
event=event,
|
731
|
+
)
|
615
732
|
|
616
733
|
|
617
734
|
class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
@@ -629,16 +746,16 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
629
746
|
def _execute(
|
630
747
|
self,
|
631
748
|
params: DictData,
|
632
|
-
|
633
|
-
|
749
|
+
run_id: str,
|
750
|
+
context: DictData,
|
751
|
+
parent_run_id: Optional[str] = None,
|
752
|
+
event: Optional[Event] = None,
|
634
753
|
) -> Result:
|
635
754
|
"""Wrapped the execute method with retry strategy before returning to
|
636
755
|
handler execute.
|
637
756
|
|
638
757
|
:param params: (DictData) A parameter data that want to use in this
|
639
758
|
execution.
|
640
|
-
:param result: (Result) A result object for keeping context and status
|
641
|
-
data.
|
642
759
|
:param event: (Event) An event manager that use to track parent execute
|
643
760
|
was not force stopped.
|
644
761
|
|
@@ -646,13 +763,18 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
646
763
|
"""
|
647
764
|
current_retry: int = 0
|
648
765
|
exception: Exception
|
766
|
+
catch(context, status=WAIT)
|
767
|
+
trace: Trace = get_trace(
|
768
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
769
|
+
)
|
649
770
|
|
650
771
|
# NOTE: First execution for not pass to retry step if it passes.
|
651
772
|
try:
|
652
|
-
|
653
|
-
return self.execute(
|
773
|
+
return self.process(
|
654
774
|
params | {"retry": current_retry},
|
655
|
-
|
775
|
+
run_id=run_id,
|
776
|
+
context=context,
|
777
|
+
parent_run_id=parent_run_id,
|
656
778
|
event=event,
|
657
779
|
)
|
658
780
|
except Exception as e:
|
@@ -662,28 +784,34 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
662
784
|
if self.retry == 0:
|
663
785
|
raise exception
|
664
786
|
|
665
|
-
|
787
|
+
trace.warning(
|
666
788
|
f"[STAGE]: Retry count: {current_retry} ... "
|
667
789
|
f"( {exception.__class__.__name__} )"
|
668
790
|
)
|
669
791
|
|
670
792
|
while current_retry < (self.retry + 1):
|
671
793
|
try:
|
672
|
-
|
673
|
-
|
794
|
+
catch(
|
795
|
+
context=context,
|
796
|
+
status=WAIT,
|
797
|
+
updated={"retry": current_retry},
|
798
|
+
)
|
799
|
+
return self.process(
|
674
800
|
params | {"retry": current_retry},
|
675
|
-
|
801
|
+
run_id=run_id,
|
802
|
+
context=context,
|
803
|
+
parent_run_id=parent_run_id,
|
676
804
|
event=event,
|
677
805
|
)
|
678
806
|
except Exception as e:
|
679
807
|
current_retry += 1
|
680
|
-
|
808
|
+
trace.warning(
|
681
809
|
f"[STAGE]: Retry count: {current_retry} ... "
|
682
810
|
f"( {e.__class__.__name__} )"
|
683
811
|
)
|
684
812
|
exception = e
|
685
813
|
|
686
|
-
|
814
|
+
trace.error(
|
687
815
|
f"[STAGE]: Reach the maximum of retry number: {self.retry}."
|
688
816
|
)
|
689
817
|
raise exception
|
@@ -691,16 +819,16 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
691
819
|
async def _axecute(
|
692
820
|
self,
|
693
821
|
params: DictData,
|
694
|
-
|
695
|
-
|
822
|
+
run_id: str,
|
823
|
+
context: DictData,
|
824
|
+
parent_run_id: Optional[str] = None,
|
825
|
+
event: Optional[Event] = None,
|
696
826
|
) -> Result:
|
697
827
|
"""Wrapped the axecute method with retry strategy before returning to
|
698
828
|
handler axecute.
|
699
829
|
|
700
830
|
:param params: (DictData) A parameter data that want to use in this
|
701
831
|
execution.
|
702
|
-
:param result: (Result) A result object for keeping context and status
|
703
|
-
data.
|
704
832
|
:param event: (Event) An event manager that use to track parent execute
|
705
833
|
was not force stopped.
|
706
834
|
|
@@ -708,13 +836,18 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
708
836
|
"""
|
709
837
|
current_retry: int = 0
|
710
838
|
exception: Exception
|
839
|
+
catch(context, status=WAIT)
|
840
|
+
trace: Trace = get_trace(
|
841
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
842
|
+
)
|
711
843
|
|
712
844
|
# NOTE: First execution for not pass to retry step if it passes.
|
713
845
|
try:
|
714
|
-
|
715
|
-
return await self.axecute(
|
846
|
+
return await self.async_process(
|
716
847
|
params | {"retry": current_retry},
|
717
|
-
|
848
|
+
run_id=run_id,
|
849
|
+
context=context,
|
850
|
+
parent_run_id=parent_run_id,
|
718
851
|
event=event,
|
719
852
|
)
|
720
853
|
except Exception as e:
|
@@ -724,47 +857,75 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
724
857
|
if self.retry == 0:
|
725
858
|
raise exception
|
726
859
|
|
727
|
-
await
|
860
|
+
await trace.awarning(
|
728
861
|
f"[STAGE]: Retry count: {current_retry} ... "
|
729
862
|
f"( {exception.__class__.__name__} )"
|
730
863
|
)
|
731
864
|
|
732
865
|
while current_retry < (self.retry + 1):
|
733
866
|
try:
|
734
|
-
|
735
|
-
|
867
|
+
catch(
|
868
|
+
context=context,
|
869
|
+
status=WAIT,
|
870
|
+
updated={"retry": current_retry},
|
871
|
+
)
|
872
|
+
return await self.async_process(
|
736
873
|
params | {"retry": current_retry},
|
737
|
-
|
874
|
+
run_id=run_id,
|
875
|
+
context=context,
|
876
|
+
parent_run_id=parent_run_id,
|
738
877
|
event=event,
|
739
878
|
)
|
740
879
|
except Exception as e:
|
741
880
|
current_retry += 1
|
742
|
-
await
|
881
|
+
await trace.awarning(
|
743
882
|
f"[STAGE]: Retry count: {current_retry} ... "
|
744
883
|
f"( {e.__class__.__name__} )"
|
745
884
|
)
|
746
885
|
exception = e
|
747
886
|
|
748
|
-
await
|
887
|
+
await trace.aerror(
|
749
888
|
f"[STAGE]: Reach the maximum of retry number: {self.retry}."
|
750
889
|
)
|
751
890
|
raise exception
|
752
891
|
|
753
892
|
|
754
893
|
class EmptyStage(BaseAsyncStage):
|
755
|
-
"""Empty stage
|
756
|
-
|
757
|
-
|
758
|
-
|
759
|
-
|
760
|
-
|
761
|
-
|
762
|
-
|
763
|
-
|
764
|
-
|
765
|
-
|
766
|
-
|
767
|
-
|
894
|
+
"""Empty stage for logging and debugging workflows.
|
895
|
+
|
896
|
+
EmptyStage is a utility stage that performs no actual work but provides
|
897
|
+
logging output and optional delays. It's commonly used for:
|
898
|
+
- Debugging workflow execution flow
|
899
|
+
- Adding informational messages to workflows
|
900
|
+
- Creating delays between stages
|
901
|
+
- Testing template parameter resolution
|
902
|
+
|
903
|
+
The stage outputs the echo message to stdout and can optionally sleep
|
904
|
+
for a specified duration, making it useful for workflow timing control
|
905
|
+
and debugging scenarios.
|
906
|
+
|
907
|
+
Attributes:
|
908
|
+
echo (str, optional): Message to display during execution
|
909
|
+
sleep (float): Duration to sleep after logging (0-1800 seconds)
|
910
|
+
|
911
|
+
Example:
|
912
|
+
```yaml
|
913
|
+
stages:
|
914
|
+
- name: "Workflow Started"
|
915
|
+
echo: "Beginning data processing workflow"
|
916
|
+
sleep: 2
|
917
|
+
|
918
|
+
- name: "Debug Parameters"
|
919
|
+
echo: "Processing file: ${{ params.filename }}"
|
920
|
+
```
|
921
|
+
|
922
|
+
```python
|
923
|
+
stage = EmptyStage(
|
924
|
+
name="Status Update",
|
925
|
+
echo="Processing completed successfully",
|
926
|
+
sleep=1.0
|
927
|
+
)
|
928
|
+
```
|
768
929
|
"""
|
769
930
|
|
770
931
|
echo: StrOrNone = Field(
|
@@ -781,11 +942,13 @@ class EmptyStage(BaseAsyncStage):
|
|
781
942
|
lt=1800,
|
782
943
|
)
|
783
944
|
|
784
|
-
def
|
945
|
+
def process(
|
785
946
|
self,
|
786
947
|
params: DictData,
|
948
|
+
run_id: str,
|
949
|
+
context: DictData,
|
787
950
|
*,
|
788
|
-
|
951
|
+
parent_run_id: Optional[str] = None,
|
789
952
|
event: Optional[Event] = None,
|
790
953
|
) -> Result:
|
791
954
|
"""Execution method for the Empty stage that do only logging out to
|
@@ -794,16 +957,20 @@ class EmptyStage(BaseAsyncStage):
|
|
794
957
|
The result context should be empty and do not process anything
|
795
958
|
without calling logging function.
|
796
959
|
|
797
|
-
:
|
798
|
-
|
799
|
-
|
800
|
-
|
801
|
-
|
802
|
-
|
960
|
+
Args:
|
961
|
+
params: A parameter data that want to use in this
|
962
|
+
execution.
|
963
|
+
run_id: A running stage ID.
|
964
|
+
context: A context data.
|
965
|
+
parent_run_id: A parent running ID. (Default is None)
|
966
|
+
event: An event manager that use to track parent process
|
967
|
+
was not force stopped.
|
968
|
+
|
969
|
+
Returns:
|
970
|
+
Result: The execution result with status and context data.
|
803
971
|
"""
|
804
|
-
|
805
|
-
run_id=
|
806
|
-
extras=self.extras,
|
972
|
+
trace: Trace = get_trace(
|
973
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
807
974
|
)
|
808
975
|
message: str = (
|
809
976
|
param2template(
|
@@ -818,35 +985,46 @@ class EmptyStage(BaseAsyncStage):
|
|
818
985
|
"Execution was canceled from the event before start parallel."
|
819
986
|
)
|
820
987
|
|
821
|
-
|
988
|
+
trace.info(f"[STAGE]: Message: ( {message} )")
|
822
989
|
if self.sleep > 0:
|
823
990
|
if self.sleep > 5:
|
824
|
-
|
991
|
+
trace.info(f"[STAGE]: Sleep ... ({self.sleep} sec)")
|
825
992
|
time.sleep(self.sleep)
|
826
|
-
return
|
993
|
+
return Result(
|
994
|
+
run_id=run_id,
|
995
|
+
parent_run_id=parent_run_id,
|
996
|
+
status=SUCCESS,
|
997
|
+
context=catch(context=context, status=SUCCESS),
|
998
|
+
extras=self.extras,
|
999
|
+
)
|
827
1000
|
|
828
|
-
async def
|
1001
|
+
async def async_process(
|
829
1002
|
self,
|
830
1003
|
params: DictData,
|
1004
|
+
run_id: str,
|
1005
|
+
context: DictData,
|
831
1006
|
*,
|
832
|
-
|
1007
|
+
parent_run_id: Optional[str] = None,
|
833
1008
|
event: Optional[Event] = None,
|
834
1009
|
) -> Result:
|
835
1010
|
"""Async execution method for this Empty stage that only logging out to
|
836
1011
|
stdout.
|
837
1012
|
|
838
|
-
:
|
839
|
-
|
840
|
-
|
841
|
-
|
842
|
-
|
843
|
-
|
1013
|
+
Args:
|
1014
|
+
params: A parameter data that want to use in this
|
1015
|
+
execution.
|
1016
|
+
run_id: A running stage ID.
|
1017
|
+
context: A context data.
|
1018
|
+
parent_run_id: A parent running ID. (Default is None)
|
1019
|
+
event: An event manager that use to track parent process
|
1020
|
+
was not force stopped.
|
1021
|
+
|
1022
|
+
Returns:
|
1023
|
+
Result: The execution result with status and context data.
|
844
1024
|
"""
|
845
|
-
|
846
|
-
run_id=
|
847
|
-
extras=self.extras,
|
1025
|
+
trace: Trace = get_trace(
|
1026
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
848
1027
|
)
|
849
|
-
|
850
1028
|
message: str = (
|
851
1029
|
param2template(
|
852
1030
|
dedent(self.echo.strip("\n")), params, extras=self.extras
|
@@ -860,14 +1038,18 @@ class EmptyStage(BaseAsyncStage):
|
|
860
1038
|
"Execution was canceled from the event before start parallel."
|
861
1039
|
)
|
862
1040
|
|
863
|
-
|
1041
|
+
trace.info(f"[STAGE]: Message: ( {message} )")
|
864
1042
|
if self.sleep > 0:
|
865
1043
|
if self.sleep > 5:
|
866
|
-
await
|
867
|
-
f"[STAGE]: Sleep ... ({self.sleep} sec)"
|
868
|
-
)
|
1044
|
+
await trace.ainfo(f"[STAGE]: Sleep ... ({self.sleep} sec)")
|
869
1045
|
await asyncio.sleep(self.sleep)
|
870
|
-
return
|
1046
|
+
return Result(
|
1047
|
+
run_id=run_id,
|
1048
|
+
parent_run_id=parent_run_id,
|
1049
|
+
status=SUCCESS,
|
1050
|
+
context=catch(context=context, status=SUCCESS),
|
1051
|
+
extras=self.extras,
|
1052
|
+
)
|
871
1053
|
|
872
1054
|
|
873
1055
|
class BashStage(BaseRetryStage):
|
@@ -975,27 +1157,38 @@ class BashStage(BaseRetryStage):
|
|
975
1157
|
# Note: Remove .sh file that use to run bash.
|
976
1158
|
Path(f"./{f_name}").unlink()
|
977
1159
|
|
978
|
-
|
1160
|
+
@staticmethod
|
1161
|
+
def prepare_std(value: str) -> Optional[str]:
|
1162
|
+
"""Prepare returned standard string from subprocess."""
|
1163
|
+
return None if (out := value.strip("\n")) == "" else out
|
1164
|
+
|
1165
|
+
def process(
|
979
1166
|
self,
|
980
1167
|
params: DictData,
|
1168
|
+
run_id: str,
|
1169
|
+
context: DictData,
|
981
1170
|
*,
|
982
|
-
|
1171
|
+
parent_run_id: Optional[str] = None,
|
983
1172
|
event: Optional[Event] = None,
|
984
1173
|
) -> Result:
|
985
1174
|
"""Execute bash statement with the Python build-in `subprocess` package.
|
986
1175
|
It will catch result from the `subprocess.run` returning output like
|
987
1176
|
`return_code`, `stdout`, and `stderr`.
|
988
1177
|
|
989
|
-
:
|
990
|
-
|
991
|
-
|
992
|
-
|
993
|
-
|
994
|
-
|
1178
|
+
Args:
|
1179
|
+
params: A parameter data that want to use in this
|
1180
|
+
execution.
|
1181
|
+
run_id: A running stage ID.
|
1182
|
+
context: A context data.
|
1183
|
+
parent_run_id: A parent running ID. (Default is None)
|
1184
|
+
event: An event manager that use to track parent process
|
1185
|
+
was not force stopped.
|
1186
|
+
|
1187
|
+
Returns:
|
1188
|
+
Result: The execution result with status and context data.
|
995
1189
|
"""
|
996
|
-
|
997
|
-
run_id=
|
998
|
-
extras=self.extras,
|
1190
|
+
trace: Trace = get_trace(
|
1191
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
999
1192
|
)
|
1000
1193
|
bash: str = param2template(
|
1001
1194
|
dedent(self.bash.strip("\n")), params, extras=self.extras
|
@@ -1003,9 +1196,9 @@ class BashStage(BaseRetryStage):
|
|
1003
1196
|
with self.create_sh_file(
|
1004
1197
|
bash=bash,
|
1005
1198
|
env=param2template(self.env, params, extras=self.extras),
|
1006
|
-
run_id=
|
1199
|
+
run_id=run_id,
|
1007
1200
|
) as sh:
|
1008
|
-
|
1201
|
+
trace.debug(f"[STAGE]: Create `{sh[1]}` file.")
|
1009
1202
|
rs: CompletedProcess = subprocess.run(
|
1010
1203
|
sh,
|
1011
1204
|
shell=False,
|
@@ -1018,35 +1211,48 @@ class BashStage(BaseRetryStage):
|
|
1018
1211
|
e: str = rs.stderr.removesuffix("\n")
|
1019
1212
|
e_bash: str = bash.replace("\n", "\n\t")
|
1020
1213
|
raise StageError(f"Subprocess: {e}\n\t```bash\n\t{e_bash}\n\t```")
|
1021
|
-
return
|
1214
|
+
return Result(
|
1215
|
+
run_id=run_id,
|
1216
|
+
parent_run_id=parent_run_id,
|
1022
1217
|
status=SUCCESS,
|
1023
|
-
context=
|
1024
|
-
|
1025
|
-
|
1026
|
-
|
1027
|
-
|
1218
|
+
context=catch(
|
1219
|
+
context=context,
|
1220
|
+
status=SUCCESS,
|
1221
|
+
updated={
|
1222
|
+
"return_code": rs.returncode,
|
1223
|
+
"stdout": self.prepare_std(rs.stdout),
|
1224
|
+
"stderr": self.prepare_std(rs.stderr),
|
1225
|
+
},
|
1226
|
+
),
|
1227
|
+
extras=self.extras,
|
1028
1228
|
)
|
1029
1229
|
|
1030
|
-
async def
|
1230
|
+
async def async_process(
|
1031
1231
|
self,
|
1032
1232
|
params: DictData,
|
1233
|
+
run_id: str,
|
1234
|
+
context: DictData,
|
1033
1235
|
*,
|
1034
|
-
|
1236
|
+
parent_run_id: Optional[str] = None,
|
1035
1237
|
event: Optional[Event] = None,
|
1036
1238
|
) -> Result:
|
1037
1239
|
"""Async execution method for this Bash stage that only logging out to
|
1038
1240
|
stdout.
|
1039
1241
|
|
1040
|
-
:
|
1041
|
-
|
1042
|
-
|
1043
|
-
|
1044
|
-
|
1045
|
-
|
1242
|
+
Args:
|
1243
|
+
params: A parameter data that want to use in this
|
1244
|
+
execution.
|
1245
|
+
run_id: A running stage ID.
|
1246
|
+
context: A context data.
|
1247
|
+
parent_run_id: A parent running ID. (Default is None)
|
1248
|
+
event: An event manager that use to track parent process
|
1249
|
+
was not force stopped.
|
1250
|
+
|
1251
|
+
Returns:
|
1252
|
+
Result: The execution result with status and context data.
|
1046
1253
|
"""
|
1047
|
-
|
1048
|
-
run_id=
|
1049
|
-
extras=self.extras,
|
1254
|
+
trace: Trace = get_trace(
|
1255
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1050
1256
|
)
|
1051
1257
|
bash: str = param2template(
|
1052
1258
|
dedent(self.bash.strip("\n")), params, extras=self.extras
|
@@ -1054,9 +1260,9 @@ class BashStage(BaseRetryStage):
|
|
1054
1260
|
async with self.async_create_sh_file(
|
1055
1261
|
bash=bash,
|
1056
1262
|
env=param2template(self.env, params, extras=self.extras),
|
1057
|
-
run_id=
|
1263
|
+
run_id=run_id,
|
1058
1264
|
) as sh:
|
1059
|
-
await
|
1265
|
+
await trace.adebug(f"[STAGE]: Create `{sh[1]}` file.")
|
1060
1266
|
rs: CompletedProcess = subprocess.run(
|
1061
1267
|
sh,
|
1062
1268
|
shell=False,
|
@@ -1070,13 +1276,20 @@ class BashStage(BaseRetryStage):
|
|
1070
1276
|
e: str = rs.stderr.removesuffix("\n")
|
1071
1277
|
e_bash: str = bash.replace("\n", "\n\t")
|
1072
1278
|
raise StageError(f"Subprocess: {e}\n\t```bash\n\t{e_bash}\n\t```")
|
1073
|
-
return
|
1279
|
+
return Result(
|
1280
|
+
run_id=run_id,
|
1281
|
+
parent_run_id=parent_run_id,
|
1074
1282
|
status=SUCCESS,
|
1075
|
-
context=
|
1076
|
-
|
1077
|
-
|
1078
|
-
|
1079
|
-
|
1283
|
+
context=catch(
|
1284
|
+
context=context,
|
1285
|
+
status=SUCCESS,
|
1286
|
+
updated={
|
1287
|
+
"return_code": rs.returncode,
|
1288
|
+
"stdout": self.prepare_std(rs.stdout),
|
1289
|
+
"stderr": self.prepare_std(rs.stderr),
|
1290
|
+
},
|
1291
|
+
),
|
1292
|
+
extras=self.extras,
|
1080
1293
|
)
|
1081
1294
|
|
1082
1295
|
|
@@ -1135,13 +1348,16 @@ class PyStage(BaseRetryStage):
|
|
1135
1348
|
|
1136
1349
|
yield value
|
1137
1350
|
|
1138
|
-
def set_outputs(
|
1351
|
+
def set_outputs(
|
1352
|
+
self, output: DictData, to: DictData, info: Optional[DictData] = None
|
1353
|
+
) -> DictData:
|
1139
1354
|
"""Override set an outputs method for the Python execution process that
|
1140
1355
|
extract output from all the locals values.
|
1141
1356
|
|
1142
1357
|
:param output: (DictData) An output data that want to extract to an
|
1143
1358
|
output key.
|
1144
1359
|
:param to: (DictData) A context data that want to add output result.
|
1360
|
+
:param info: (DictData)
|
1145
1361
|
|
1146
1362
|
:rtype: DictData
|
1147
1363
|
"""
|
@@ -1152,33 +1368,47 @@ class PyStage(BaseRetryStage):
|
|
1152
1368
|
to.update({k: gb[k] for k in to if k in gb})
|
1153
1369
|
return to
|
1154
1370
|
|
1155
|
-
def
|
1371
|
+
def process(
|
1156
1372
|
self,
|
1157
1373
|
params: DictData,
|
1374
|
+
run_id: str,
|
1375
|
+
context: DictData,
|
1158
1376
|
*,
|
1159
|
-
|
1377
|
+
parent_run_id: Optional[str] = None,
|
1160
1378
|
event: Optional[Event] = None,
|
1161
1379
|
) -> Result:
|
1162
1380
|
"""Execute the Python statement that pass all globals and input params
|
1163
1381
|
to globals argument on `exec` build-in function.
|
1164
1382
|
|
1165
|
-
:
|
1166
|
-
|
1167
|
-
|
1168
|
-
|
1169
|
-
|
1170
|
-
|
1171
|
-
|
1383
|
+
Args:
|
1384
|
+
params: A parameter data that want to use in this
|
1385
|
+
execution.
|
1386
|
+
run_id: A running stage ID.
|
1387
|
+
context: A context data.
|
1388
|
+
parent_run_id: A parent running ID. (Default is None)
|
1389
|
+
event: An event manager that use to track parent process
|
1390
|
+
was not force stopped.
|
1391
|
+
|
1392
|
+
Returns:
|
1393
|
+
Result: The execution result with status and context data.
|
1172
1394
|
"""
|
1173
|
-
|
1174
|
-
run_id=
|
1175
|
-
extras=self.extras,
|
1395
|
+
trace: Trace = get_trace(
|
1396
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1176
1397
|
)
|
1398
|
+
trace.info("[STAGE]: Prepare `globals` and `locals` variables.")
|
1177
1399
|
lc: DictData = {}
|
1178
1400
|
gb: DictData = (
|
1179
1401
|
globals()
|
1180
1402
|
| param2template(self.vars, params, extras=self.extras)
|
1181
|
-
| {
|
1403
|
+
| {
|
1404
|
+
"result": Result(
|
1405
|
+
run_id=run_id,
|
1406
|
+
parent_run_id=parent_run_id,
|
1407
|
+
status=WAIT,
|
1408
|
+
context=context,
|
1409
|
+
extras=self.extras,
|
1410
|
+
)
|
1411
|
+
}
|
1182
1412
|
)
|
1183
1413
|
|
1184
1414
|
# WARNING: The exec build-in function is very dangerous. So, it
|
@@ -1190,55 +1420,76 @@ class PyStage(BaseRetryStage):
|
|
1190
1420
|
gb,
|
1191
1421
|
lc,
|
1192
1422
|
)
|
1193
|
-
|
1194
|
-
|
1423
|
+
return Result(
|
1424
|
+
run_id=run_id,
|
1425
|
+
parent_run_id=parent_run_id,
|
1195
1426
|
status=SUCCESS,
|
1196
|
-
context=
|
1197
|
-
|
1198
|
-
|
1199
|
-
|
1200
|
-
for k in
|
1201
|
-
|
1202
|
-
|
1203
|
-
|
1204
|
-
|
1205
|
-
|
1206
|
-
|
1207
|
-
|
1208
|
-
|
1427
|
+
context=catch(
|
1428
|
+
context=context,
|
1429
|
+
status=SUCCESS,
|
1430
|
+
updated={
|
1431
|
+
"locals": {k: lc[k] for k in self.filter_locals(lc)},
|
1432
|
+
"globals": {
|
1433
|
+
k: gb[k]
|
1434
|
+
for k in gb
|
1435
|
+
if (
|
1436
|
+
not k.startswith("__")
|
1437
|
+
and k != "annotations"
|
1438
|
+
and not ismodule(gb[k])
|
1439
|
+
and not isclass(gb[k])
|
1440
|
+
and not isfunction(gb[k])
|
1441
|
+
and k in params
|
1442
|
+
)
|
1443
|
+
},
|
1209
1444
|
},
|
1210
|
-
|
1445
|
+
),
|
1446
|
+
extras=self.extras,
|
1211
1447
|
)
|
1212
1448
|
|
1213
|
-
async def
|
1449
|
+
async def async_process(
|
1214
1450
|
self,
|
1215
1451
|
params: DictData,
|
1452
|
+
run_id: str,
|
1453
|
+
context: DictData,
|
1216
1454
|
*,
|
1217
|
-
|
1455
|
+
parent_run_id: Optional[str] = None,
|
1218
1456
|
event: Optional[Event] = None,
|
1219
1457
|
) -> Result:
|
1220
1458
|
"""Async execution method for this Bash stage that only logging out to
|
1221
1459
|
stdout.
|
1222
1460
|
|
1223
|
-
:param params: (DictData) A parameter data.
|
1224
|
-
:param result: (Result) A Result instance for return context and status.
|
1225
|
-
:param event: (Event) An Event manager instance that use to cancel this
|
1226
|
-
execution if it forces stopped by parent execution.
|
1227
|
-
|
1228
1461
|
References:
|
1229
1462
|
- https://stackoverflow.com/questions/44859165/async-exec-in-python
|
1230
1463
|
|
1231
|
-
:
|
1464
|
+
Args:
|
1465
|
+
params: A parameter data that want to use in this
|
1466
|
+
execution.
|
1467
|
+
run_id: A running stage ID.
|
1468
|
+
context: A context data.
|
1469
|
+
parent_run_id: A parent running ID. (Default is None)
|
1470
|
+
event: An event manager that use to track parent process
|
1471
|
+
was not force stopped.
|
1472
|
+
|
1473
|
+
Returns:
|
1474
|
+
Result: The execution result with status and context data.
|
1232
1475
|
"""
|
1233
|
-
|
1234
|
-
run_id=
|
1235
|
-
extras=self.extras,
|
1476
|
+
trace: Trace = get_trace(
|
1477
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1236
1478
|
)
|
1479
|
+
await trace.ainfo("[STAGE]: Prepare `globals` and `locals` variables.")
|
1237
1480
|
lc: DictData = {}
|
1238
1481
|
gb: DictData = (
|
1239
1482
|
globals()
|
1240
1483
|
| param2template(self.vars, params, extras=self.extras)
|
1241
|
-
| {
|
1484
|
+
| {
|
1485
|
+
"result": Result(
|
1486
|
+
run_id=run_id,
|
1487
|
+
parent_run_id=parent_run_id,
|
1488
|
+
status=WAIT,
|
1489
|
+
context=context,
|
1490
|
+
extras=self.extras,
|
1491
|
+
)
|
1492
|
+
}
|
1242
1493
|
)
|
1243
1494
|
# WARNING: The exec build-in function is very dangerous. So, it
|
1244
1495
|
# should use the re module to validate exec-string before running.
|
@@ -1247,23 +1498,30 @@ class PyStage(BaseRetryStage):
|
|
1247
1498
|
gb,
|
1248
1499
|
lc,
|
1249
1500
|
)
|
1250
|
-
return
|
1501
|
+
return Result(
|
1502
|
+
run_id=run_id,
|
1503
|
+
parent_run_id=parent_run_id,
|
1251
1504
|
status=SUCCESS,
|
1252
|
-
context=
|
1253
|
-
|
1254
|
-
|
1255
|
-
|
1256
|
-
for k in
|
1257
|
-
|
1258
|
-
|
1259
|
-
|
1260
|
-
|
1261
|
-
|
1262
|
-
|
1263
|
-
|
1264
|
-
|
1505
|
+
context=catch(
|
1506
|
+
context=context,
|
1507
|
+
status=SUCCESS,
|
1508
|
+
updated={
|
1509
|
+
"locals": {k: lc[k] for k in self.filter_locals(lc)},
|
1510
|
+
"globals": {
|
1511
|
+
k: gb[k]
|
1512
|
+
for k in gb
|
1513
|
+
if (
|
1514
|
+
not k.startswith("__")
|
1515
|
+
and k != "annotations"
|
1516
|
+
and not ismodule(gb[k])
|
1517
|
+
and not isclass(gb[k])
|
1518
|
+
and not isfunction(gb[k])
|
1519
|
+
and k in params
|
1520
|
+
)
|
1521
|
+
},
|
1265
1522
|
},
|
1266
|
-
|
1523
|
+
),
|
1524
|
+
extras=self.extras,
|
1267
1525
|
)
|
1268
1526
|
|
1269
1527
|
|
@@ -1311,46 +1569,69 @@ class CallStage(BaseRetryStage):
|
|
1311
1569
|
alias="with",
|
1312
1570
|
)
|
1313
1571
|
|
1314
|
-
|
1572
|
+
@field_validator("args", mode="before")
|
1573
|
+
def __validate_args_key(cls, value: Any) -> Any:
|
1574
|
+
"""Validate argument keys on the ``args`` field should not include the
|
1575
|
+
special keys.
|
1576
|
+
|
1577
|
+
:param value: (Any) A value that want to check the special keys.
|
1578
|
+
|
1579
|
+
:rtype: Any
|
1580
|
+
"""
|
1581
|
+
if isinstance(value, dict):
|
1582
|
+
if any(k in value for k in ("result", "extras")):
|
1583
|
+
raise ValueError(
|
1584
|
+
"The argument on workflow template for the caller stage "
|
1585
|
+
"should not pass `result` and `extras`. They are special "
|
1586
|
+
"arguments."
|
1587
|
+
)
|
1588
|
+
return value
|
1589
|
+
|
1590
|
+
def process(
|
1315
1591
|
self,
|
1316
1592
|
params: DictData,
|
1593
|
+
run_id: str,
|
1594
|
+
context: DictData,
|
1317
1595
|
*,
|
1318
|
-
|
1596
|
+
parent_run_id: Optional[str] = None,
|
1319
1597
|
event: Optional[Event] = None,
|
1320
1598
|
) -> Result:
|
1321
1599
|
"""Execute this caller function with its argument parameter.
|
1322
1600
|
|
1323
|
-
:
|
1324
|
-
|
1325
|
-
|
1326
|
-
|
1327
|
-
|
1328
|
-
|
1329
|
-
|
1330
|
-
|
1331
|
-
|
1332
|
-
|
1333
|
-
|
1601
|
+
Args:
|
1602
|
+
params: A parameter data that want to use in this
|
1603
|
+
execution.
|
1604
|
+
run_id: A running stage ID.
|
1605
|
+
context: A context data.
|
1606
|
+
parent_run_id: A parent running ID. (Default is None)
|
1607
|
+
event: An event manager that use to track parent process
|
1608
|
+
was not force stopped.
|
1609
|
+
|
1610
|
+
Returns:
|
1611
|
+
Result: The execution result with status and context data.
|
1334
1612
|
"""
|
1335
|
-
|
1336
|
-
run_id=
|
1337
|
-
extras=self.extras,
|
1613
|
+
trace: Trace = get_trace(
|
1614
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1338
1615
|
)
|
1339
|
-
|
1340
1616
|
call_func: TagFunc = extract_call(
|
1341
1617
|
param2template(self.uses, params, extras=self.extras),
|
1342
1618
|
registries=self.extras.get("registry_caller"),
|
1343
1619
|
)()
|
1344
1620
|
|
1345
|
-
|
1346
|
-
f"[STAGE]: Caller Func: '{call_func.name}@{call_func.tag}'"
|
1347
|
-
)
|
1621
|
+
trace.info(f"[STAGE]: Caller Func: '{call_func.name}@{call_func.tag}'")
|
1348
1622
|
|
1349
1623
|
# VALIDATE: check input task caller parameters that exists before
|
1350
1624
|
# calling.
|
1351
|
-
args: DictData = {
|
1352
|
-
|
1353
|
-
|
1625
|
+
args: DictData = {
|
1626
|
+
"result": Result(
|
1627
|
+
run_id=run_id,
|
1628
|
+
parent_run_id=parent_run_id,
|
1629
|
+
status=WAIT,
|
1630
|
+
context=context,
|
1631
|
+
extras=self.extras,
|
1632
|
+
),
|
1633
|
+
"extras": self.extras,
|
1634
|
+
} | param2template(self.args, params, extras=self.extras)
|
1354
1635
|
sig = inspect.signature(call_func)
|
1355
1636
|
necessary_params: list[str] = []
|
1356
1637
|
has_keyword: bool = False
|
@@ -1369,20 +1650,33 @@ class CallStage(BaseRetryStage):
|
|
1369
1650
|
(k.removeprefix("_") not in args and k not in args)
|
1370
1651
|
for k in necessary_params
|
1371
1652
|
):
|
1653
|
+
if "result" in necessary_params:
|
1654
|
+
necessary_params.remove("result")
|
1655
|
+
|
1656
|
+
if "extras" in necessary_params:
|
1657
|
+
necessary_params.remove("extras")
|
1658
|
+
|
1659
|
+
args.pop("result")
|
1660
|
+
args.pop("extras")
|
1372
1661
|
raise ValueError(
|
1373
1662
|
f"Necessary params, ({', '.join(necessary_params)}, ), "
|
1374
|
-
f"does not set to args
|
1663
|
+
f"does not set to args. It already set {list(args.keys())}."
|
1375
1664
|
)
|
1376
1665
|
|
1377
1666
|
if "result" not in sig.parameters and not has_keyword:
|
1378
1667
|
args.pop("result")
|
1379
1668
|
|
1669
|
+
if "extras" not in sig.parameters and not has_keyword:
|
1670
|
+
args.pop("extras")
|
1671
|
+
|
1380
1672
|
if event and event.is_set():
|
1381
1673
|
raise StageCancelError(
|
1382
1674
|
"Execution was canceled from the event before start parallel."
|
1383
1675
|
)
|
1384
1676
|
|
1385
|
-
args = self.validate_model_args(
|
1677
|
+
args: DictData = self.validate_model_args(
|
1678
|
+
call_func, args, run_id, parent_run_id
|
1679
|
+
)
|
1386
1680
|
if inspect.iscoroutinefunction(call_func):
|
1387
1681
|
loop = asyncio.get_event_loop()
|
1388
1682
|
rs: DictData = loop.run_until_complete(
|
@@ -1403,47 +1697,66 @@ class CallStage(BaseRetryStage):
|
|
1403
1697
|
f"serialize, you must set return be `dict` or Pydantic "
|
1404
1698
|
f"model."
|
1405
1699
|
)
|
1406
|
-
return
|
1700
|
+
return Result(
|
1701
|
+
run_id=run_id,
|
1702
|
+
parent_run_id=parent_run_id,
|
1703
|
+
status=SUCCESS,
|
1704
|
+
context=catch(
|
1705
|
+
context=context,
|
1706
|
+
status=SUCCESS,
|
1707
|
+
updated=dump_all(rs, by_alias=True),
|
1708
|
+
),
|
1709
|
+
extras=self.extras,
|
1710
|
+
)
|
1407
1711
|
|
1408
|
-
async def
|
1712
|
+
async def async_process(
|
1409
1713
|
self,
|
1410
1714
|
params: DictData,
|
1715
|
+
run_id: str,
|
1716
|
+
context: DictData,
|
1411
1717
|
*,
|
1412
|
-
|
1718
|
+
parent_run_id: Optional[str] = None,
|
1413
1719
|
event: Optional[Event] = None,
|
1414
1720
|
) -> Result:
|
1415
1721
|
"""Async execution method for this Bash stage that only logging out to
|
1416
1722
|
stdout.
|
1417
1723
|
|
1418
|
-
:
|
1419
|
-
|
1420
|
-
|
1421
|
-
|
1422
|
-
|
1423
|
-
|
1424
|
-
|
1425
|
-
|
1426
|
-
|
1724
|
+
Args:
|
1725
|
+
params: A parameter data that want to use in this
|
1726
|
+
execution.
|
1727
|
+
run_id: A running stage ID.
|
1728
|
+
context: A context data.
|
1729
|
+
parent_run_id: A parent running ID. (Default is None)
|
1730
|
+
event: An event manager that use to track parent process
|
1731
|
+
was not force stopped.
|
1732
|
+
|
1733
|
+
Returns:
|
1734
|
+
Result: The execution result with status and context data.
|
1427
1735
|
"""
|
1428
|
-
|
1429
|
-
run_id=
|
1430
|
-
extras=self.extras,
|
1736
|
+
trace: Trace = get_trace(
|
1737
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1431
1738
|
)
|
1432
|
-
|
1433
1739
|
call_func: TagFunc = extract_call(
|
1434
1740
|
param2template(self.uses, params, extras=self.extras),
|
1435
1741
|
registries=self.extras.get("registry_caller"),
|
1436
1742
|
)()
|
1437
1743
|
|
1438
|
-
await
|
1744
|
+
await trace.ainfo(
|
1439
1745
|
f"[STAGE]: Caller Func: '{call_func.name}@{call_func.tag}'"
|
1440
1746
|
)
|
1441
1747
|
|
1442
1748
|
# VALIDATE: check input task caller parameters that exists before
|
1443
1749
|
# calling.
|
1444
|
-
args: DictData = {
|
1445
|
-
|
1446
|
-
|
1750
|
+
args: DictData = {
|
1751
|
+
"result": Result(
|
1752
|
+
run_id=run_id,
|
1753
|
+
parent_run_id=parent_run_id,
|
1754
|
+
status=WAIT,
|
1755
|
+
context=context,
|
1756
|
+
extras=self.extras,
|
1757
|
+
),
|
1758
|
+
"extras": self.extras,
|
1759
|
+
} | param2template(self.args, params, extras=self.extras)
|
1447
1760
|
sig = inspect.signature(call_func)
|
1448
1761
|
necessary_params: list[str] = []
|
1449
1762
|
has_keyword: bool = False
|
@@ -1462,15 +1775,27 @@ class CallStage(BaseRetryStage):
|
|
1462
1775
|
(k.removeprefix("_") not in args and k not in args)
|
1463
1776
|
for k in necessary_params
|
1464
1777
|
):
|
1778
|
+
if "result" in necessary_params:
|
1779
|
+
necessary_params.remove("result")
|
1780
|
+
|
1781
|
+
if "extras" in necessary_params:
|
1782
|
+
necessary_params.remove("extras")
|
1783
|
+
|
1784
|
+
args.pop("result")
|
1785
|
+
args.pop("extras")
|
1465
1786
|
raise ValueError(
|
1466
1787
|
f"Necessary params, ({', '.join(necessary_params)}, ), "
|
1467
|
-
f"does not set to args
|
1788
|
+
f"does not set to args. It already set {list(args.keys())}."
|
1468
1789
|
)
|
1469
|
-
|
1470
1790
|
if "result" not in sig.parameters and not has_keyword:
|
1471
1791
|
args.pop("result")
|
1472
1792
|
|
1473
|
-
|
1793
|
+
if "extras" not in sig.parameters and not has_keyword:
|
1794
|
+
args.pop("extras")
|
1795
|
+
|
1796
|
+
args: DictData = self.validate_model_args(
|
1797
|
+
call_func, args, run_id, parent_run_id
|
1798
|
+
)
|
1474
1799
|
if inspect.iscoroutinefunction(call_func):
|
1475
1800
|
rs: DictOrModel = await call_func(
|
1476
1801
|
**param2template(args, params, extras=self.extras)
|
@@ -1490,20 +1815,31 @@ class CallStage(BaseRetryStage):
|
|
1490
1815
|
f"serialize, you must set return be `dict` or Pydantic "
|
1491
1816
|
f"model."
|
1492
1817
|
)
|
1493
|
-
return
|
1818
|
+
return Result(
|
1819
|
+
run_id=run_id,
|
1820
|
+
parent_run_id=parent_run_id,
|
1821
|
+
status=SUCCESS,
|
1822
|
+
context=catch(
|
1823
|
+
context=context,
|
1824
|
+
status=SUCCESS,
|
1825
|
+
updated=dump_all(rs, by_alias=True),
|
1826
|
+
),
|
1827
|
+
extras=self.extras,
|
1828
|
+
)
|
1494
1829
|
|
1495
|
-
@staticmethod
|
1496
1830
|
def validate_model_args(
|
1831
|
+
self,
|
1497
1832
|
func: TagFunc,
|
1498
1833
|
args: DictData,
|
1499
|
-
|
1834
|
+
run_id: str,
|
1835
|
+
parent_run_id: Optional[str] = None,
|
1500
1836
|
) -> DictData:
|
1501
1837
|
"""Validate an input arguments before passing to the caller function.
|
1502
1838
|
|
1503
|
-
:
|
1504
|
-
|
1505
|
-
|
1506
|
-
|
1839
|
+
Args:
|
1840
|
+
func: (TagFunc) A tag function that want to get typing.
|
1841
|
+
args: (DictData) An arguments before passing to this tag func.
|
1842
|
+
run_id: A running stage ID.
|
1507
1843
|
|
1508
1844
|
:rtype: DictData
|
1509
1845
|
"""
|
@@ -1529,7 +1865,10 @@ class CallStage(BaseRetryStage):
|
|
1529
1865
|
"Validate argument from the caller function raise invalid type."
|
1530
1866
|
) from e
|
1531
1867
|
except TypeError as e:
|
1532
|
-
|
1868
|
+
trace: Trace = get_trace(
|
1869
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1870
|
+
)
|
1871
|
+
trace.warning(
|
1533
1872
|
f"[STAGE]: Get type hint raise TypeError: {e}, so, it skip "
|
1534
1873
|
f"parsing model args process."
|
1535
1874
|
)
|
@@ -1541,7 +1880,9 @@ class BaseNestedStage(BaseRetryStage, ABC):
|
|
1541
1880
|
is the nested stage or not.
|
1542
1881
|
"""
|
1543
1882
|
|
1544
|
-
def set_outputs(
|
1883
|
+
def set_outputs(
|
1884
|
+
self, output: DictData, to: DictData, info: Optional[DictData] = None
|
1885
|
+
) -> DictData:
|
1545
1886
|
"""Override the set outputs method that support for nested-stage."""
|
1546
1887
|
return super().set_outputs(output, to=to)
|
1547
1888
|
|
@@ -1570,13 +1911,29 @@ class BaseNestedStage(BaseRetryStage, ABC):
|
|
1570
1911
|
else:
|
1571
1912
|
context["errors"] = error.to_dict(with_refs=True)
|
1572
1913
|
|
1573
|
-
async def
|
1914
|
+
async def async_process(
|
1574
1915
|
self,
|
1575
1916
|
params: DictData,
|
1917
|
+
run_id: str,
|
1918
|
+
context: DictData,
|
1576
1919
|
*,
|
1577
|
-
|
1920
|
+
parent_run_id: Optional[str] = None,
|
1578
1921
|
event: Optional[Event] = None,
|
1579
1922
|
) -> Result:
|
1923
|
+
"""Async process for nested-stage do not implement yet.
|
1924
|
+
|
1925
|
+
Args:
|
1926
|
+
params: A parameter data that want to use in this
|
1927
|
+
execution.
|
1928
|
+
run_id: A running stage ID.
|
1929
|
+
context: A context data.
|
1930
|
+
parent_run_id: A parent running ID. (Default is None)
|
1931
|
+
event: An event manager that use to track parent process
|
1932
|
+
was not force stopped.
|
1933
|
+
|
1934
|
+
Returns:
|
1935
|
+
Result: The execution result with status and context data.
|
1936
|
+
"""
|
1580
1937
|
raise NotImplementedError(
|
1581
1938
|
"The nested-stage does not implement the `axecute` method yet."
|
1582
1939
|
)
|
@@ -1606,41 +1963,45 @@ class TriggerStage(BaseNestedStage):
|
|
1606
1963
|
description="A parameter that will pass to workflow execution method.",
|
1607
1964
|
)
|
1608
1965
|
|
1609
|
-
def
|
1966
|
+
def process(
|
1610
1967
|
self,
|
1611
1968
|
params: DictData,
|
1969
|
+
run_id: str,
|
1970
|
+
context: DictData,
|
1612
1971
|
*,
|
1613
|
-
|
1972
|
+
parent_run_id: Optional[str] = None,
|
1614
1973
|
event: Optional[Event] = None,
|
1615
1974
|
) -> Result:
|
1616
1975
|
"""Trigger another workflow execution. It will wait the trigger
|
1617
1976
|
workflow running complete before catching its result and raise error
|
1618
1977
|
when the result status does not be SUCCESS.
|
1619
1978
|
|
1620
|
-
:
|
1621
|
-
|
1622
|
-
|
1623
|
-
|
1624
|
-
|
1625
|
-
|
1626
|
-
|
1979
|
+
Args:
|
1980
|
+
params: A parameter data that want to use in this
|
1981
|
+
execution.
|
1982
|
+
run_id: A running stage ID.
|
1983
|
+
context: A context data.
|
1984
|
+
parent_run_id: A parent running ID. (Default is None)
|
1985
|
+
event: An event manager that use to track parent process
|
1986
|
+
was not force stopped.
|
1987
|
+
|
1988
|
+
Returns:
|
1989
|
+
Result: The execution result with status and context data.
|
1627
1990
|
"""
|
1628
1991
|
from .workflow import Workflow
|
1629
1992
|
|
1630
|
-
|
1631
|
-
run_id=
|
1632
|
-
extras=self.extras,
|
1993
|
+
trace: Trace = get_trace(
|
1994
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1633
1995
|
)
|
1634
|
-
|
1635
1996
|
_trigger: str = param2template(self.trigger, params, extras=self.extras)
|
1997
|
+
trace.info(f"[STAGE]: Load workflow: {_trigger!r}")
|
1636
1998
|
result: Result = Workflow.from_conf(
|
1637
1999
|
name=pass_env(_trigger),
|
1638
2000
|
extras=self.extras,
|
1639
2001
|
).execute(
|
1640
2002
|
# NOTE: Should not use the `pass_env` function on this params parameter.
|
1641
2003
|
params=param2template(self.params, params, extras=self.extras),
|
1642
|
-
run_id=
|
1643
|
-
parent_run_id=result.parent_run_id,
|
2004
|
+
run_id=parent_run_id,
|
1644
2005
|
event=event,
|
1645
2006
|
)
|
1646
2007
|
if result.status == FAILED:
|
@@ -1705,20 +2066,24 @@ class ParallelStage(BaseNestedStage):
|
|
1705
2066
|
alias="max-workers",
|
1706
2067
|
)
|
1707
2068
|
|
1708
|
-
def
|
2069
|
+
def _process_branch(
|
1709
2070
|
self,
|
1710
2071
|
branch: str,
|
1711
2072
|
params: DictData,
|
1712
|
-
|
2073
|
+
run_id: str,
|
2074
|
+
context: DictData,
|
1713
2075
|
*,
|
2076
|
+
parent_run_id: Optional[str] = None,
|
1714
2077
|
event: Optional[Event] = None,
|
1715
|
-
) -> tuple[Status,
|
2078
|
+
) -> tuple[Status, DictData]:
|
1716
2079
|
"""Execute branch that will execute all nested-stage that was set in
|
1717
2080
|
this stage with specific branch ID.
|
1718
2081
|
|
1719
2082
|
:param branch: (str) A branch ID.
|
1720
2083
|
:param params: (DictData) A parameter data.
|
1721
|
-
:param
|
2084
|
+
:param run_id: (str)
|
2085
|
+
:param context: (DictData)
|
2086
|
+
:param parent_run_id: (str | None)
|
1722
2087
|
:param event: (Event) An Event manager instance that use to cancel this
|
1723
2088
|
execution if it forces stopped by parent execution.
|
1724
2089
|
(Default is None)
|
@@ -1728,13 +2093,16 @@ class ParallelStage(BaseNestedStage):
|
|
1728
2093
|
status.
|
1729
2094
|
:raise StageError: If result from a nested-stage return failed status.
|
1730
2095
|
|
1731
|
-
:rtype: tuple[Status,
|
2096
|
+
:rtype: tuple[Status, DictData]
|
1732
2097
|
"""
|
1733
|
-
|
2098
|
+
trace: Trace = get_trace(
|
2099
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2100
|
+
)
|
2101
|
+
trace.debug(f"[STAGE]: Execute Branch: {branch!r}")
|
1734
2102
|
|
1735
2103
|
# NOTE: Create nested-context
|
1736
|
-
|
1737
|
-
|
2104
|
+
current_context: DictData = copy.deepcopy(params)
|
2105
|
+
current_context.update({"branch": branch})
|
1738
2106
|
nestet_context: DictData = {"branch": branch, "stages": {}}
|
1739
2107
|
|
1740
2108
|
total_stage: int = len(self.parallel[branch])
|
@@ -1749,7 +2117,8 @@ class ParallelStage(BaseNestedStage):
|
|
1749
2117
|
"Branch execution was canceled from the event before "
|
1750
2118
|
"start branch execution."
|
1751
2119
|
)
|
1752
|
-
|
2120
|
+
catch(
|
2121
|
+
context=context,
|
1753
2122
|
status=CANCEL,
|
1754
2123
|
parallel={
|
1755
2124
|
branch: {
|
@@ -1764,14 +2133,15 @@ class ParallelStage(BaseNestedStage):
|
|
1764
2133
|
)
|
1765
2134
|
raise StageCancelError(error_msg, refs=branch)
|
1766
2135
|
|
1767
|
-
rs: Result = stage.
|
1768
|
-
params=
|
1769
|
-
run_id=
|
1770
|
-
parent_run_id=result.parent_run_id,
|
2136
|
+
rs: Result = stage.execute(
|
2137
|
+
params=current_context,
|
2138
|
+
run_id=parent_run_id,
|
1771
2139
|
event=event,
|
1772
2140
|
)
|
1773
2141
|
stage.set_outputs(rs.context, to=nestet_context)
|
1774
|
-
stage.set_outputs(
|
2142
|
+
stage.set_outputs(
|
2143
|
+
stage.get_outputs(nestet_context), to=current_context
|
2144
|
+
)
|
1775
2145
|
|
1776
2146
|
if rs.status == SKIP:
|
1777
2147
|
skips[i] = True
|
@@ -1782,7 +2152,8 @@ class ParallelStage(BaseNestedStage):
|
|
1782
2152
|
f"Branch execution was break because its nested-stage, "
|
1783
2153
|
f"{stage.iden!r}, failed."
|
1784
2154
|
)
|
1785
|
-
|
2155
|
+
catch(
|
2156
|
+
context=context,
|
1786
2157
|
status=FAILED,
|
1787
2158
|
parallel={
|
1788
2159
|
branch: {
|
@@ -1802,7 +2173,8 @@ class ParallelStage(BaseNestedStage):
|
|
1802
2173
|
"Branch execution was canceled from the event after "
|
1803
2174
|
"end branch execution."
|
1804
2175
|
)
|
1805
|
-
|
2176
|
+
catch(
|
2177
|
+
context=context,
|
1806
2178
|
status=CANCEL,
|
1807
2179
|
parallel={
|
1808
2180
|
branch: {
|
@@ -1818,7 +2190,8 @@ class ParallelStage(BaseNestedStage):
|
|
1818
2190
|
raise StageCancelError(error_msg, refs=branch)
|
1819
2191
|
|
1820
2192
|
status: Status = SKIP if sum(skips) == total_stage else SUCCESS
|
1821
|
-
return status,
|
2193
|
+
return status, catch(
|
2194
|
+
context=context,
|
1822
2195
|
status=status,
|
1823
2196
|
parallel={
|
1824
2197
|
branch: {
|
@@ -1829,32 +2202,38 @@ class ParallelStage(BaseNestedStage):
|
|
1829
2202
|
},
|
1830
2203
|
)
|
1831
2204
|
|
1832
|
-
def
|
2205
|
+
def process(
|
1833
2206
|
self,
|
1834
2207
|
params: DictData,
|
2208
|
+
run_id: str,
|
2209
|
+
context: DictData,
|
1835
2210
|
*,
|
1836
|
-
|
2211
|
+
parent_run_id: Optional[str] = None,
|
1837
2212
|
event: Optional[Event] = None,
|
1838
2213
|
) -> Result:
|
1839
2214
|
"""Execute parallel each branch via multi-threading pool.
|
1840
2215
|
|
1841
|
-
:
|
1842
|
-
|
1843
|
-
|
1844
|
-
|
1845
|
-
|
1846
|
-
|
1847
|
-
|
2216
|
+
Args:
|
2217
|
+
params: A parameter data that want to use in this
|
2218
|
+
execution.
|
2219
|
+
run_id: A running stage ID.
|
2220
|
+
context: A context data.
|
2221
|
+
parent_run_id: A parent running ID. (Default is None)
|
2222
|
+
event: An event manager that use to track parent process
|
2223
|
+
was not force stopped.
|
2224
|
+
|
2225
|
+
Returns:
|
2226
|
+
Result: The execution result with status and context data.
|
1848
2227
|
"""
|
1849
|
-
|
1850
|
-
run_id=
|
1851
|
-
extras=self.extras,
|
2228
|
+
trace: Trace = get_trace(
|
2229
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1852
2230
|
)
|
1853
2231
|
event: Event = event or Event()
|
1854
|
-
|
1855
|
-
|
2232
|
+
trace.info(f"[STAGE]: Parallel with {self.max_workers} workers.")
|
2233
|
+
catch(
|
2234
|
+
context=context,
|
1856
2235
|
status=WAIT,
|
1857
|
-
|
2236
|
+
updated={"workers": self.max_workers, "parallel": {}},
|
1858
2237
|
)
|
1859
2238
|
len_parallel: int = len(self.parallel)
|
1860
2239
|
if event and event.is_set():
|
@@ -1865,25 +2244,32 @@ class ParallelStage(BaseNestedStage):
|
|
1865
2244
|
with ThreadPoolExecutor(self.max_workers, "stp") as executor:
|
1866
2245
|
futures: list[Future] = [
|
1867
2246
|
executor.submit(
|
1868
|
-
self.
|
2247
|
+
self._process_branch,
|
1869
2248
|
branch=branch,
|
1870
2249
|
params=params,
|
1871
|
-
|
2250
|
+
run_id=run_id,
|
2251
|
+
context=context,
|
2252
|
+
parent_run_id=parent_run_id,
|
1872
2253
|
event=event,
|
1873
2254
|
)
|
1874
2255
|
for branch in self.parallel
|
1875
2256
|
]
|
1876
|
-
|
2257
|
+
errors: DictData = {}
|
1877
2258
|
statuses: list[Status] = [WAIT] * len_parallel
|
1878
2259
|
for i, future in enumerate(as_completed(futures), start=0):
|
1879
2260
|
try:
|
1880
2261
|
statuses[i], _ = future.result()
|
1881
2262
|
except StageError as e:
|
1882
2263
|
statuses[i] = get_status_from_error(e)
|
1883
|
-
self.mark_errors(
|
1884
|
-
|
1885
|
-
|
1886
|
-
|
2264
|
+
self.mark_errors(errors, e)
|
2265
|
+
|
2266
|
+
st: Status = validate_statuses(statuses)
|
2267
|
+
return Result(
|
2268
|
+
run_id=run_id,
|
2269
|
+
parent_run_id=parent_run_id,
|
2270
|
+
status=st,
|
2271
|
+
context=catch(context, status=st, updated=errors),
|
2272
|
+
extras=self.extras,
|
1887
2273
|
)
|
1888
2274
|
|
1889
2275
|
|
@@ -1936,15 +2322,17 @@ class ForEachStage(BaseNestedStage):
|
|
1936
2322
|
),
|
1937
2323
|
)
|
1938
2324
|
|
1939
|
-
def
|
2325
|
+
def _process_item(
|
1940
2326
|
self,
|
1941
2327
|
index: int,
|
1942
2328
|
item: StrOrInt,
|
1943
2329
|
params: DictData,
|
1944
|
-
|
2330
|
+
run_id: str,
|
2331
|
+
context: DictData,
|
1945
2332
|
*,
|
2333
|
+
parent_run_id: Optional[str] = None,
|
1946
2334
|
event: Optional[Event] = None,
|
1947
|
-
) -> tuple[Status,
|
2335
|
+
) -> tuple[Status, DictData]:
|
1948
2336
|
"""Execute item that will execute all nested-stage that was set in this
|
1949
2337
|
stage with specific foreach item.
|
1950
2338
|
|
@@ -1954,7 +2342,9 @@ class ForEachStage(BaseNestedStage):
|
|
1954
2342
|
:param index: (int) An index value of foreach loop.
|
1955
2343
|
:param item: (str | int) An item that want to execution.
|
1956
2344
|
:param params: (DictData) A parameter data.
|
1957
|
-
:param
|
2345
|
+
:param run_id: (str)
|
2346
|
+
:param context: (DictData)
|
2347
|
+
:param parent_run_id: (str | None)
|
1958
2348
|
:param event: (Event) An Event manager instance that use to cancel this
|
1959
2349
|
execution if it forces stopped by parent execution.
|
1960
2350
|
(Default is None)
|
@@ -1968,12 +2358,15 @@ class ForEachStage(BaseNestedStage):
|
|
1968
2358
|
|
1969
2359
|
:rtype: tuple[Status, Result]
|
1970
2360
|
"""
|
1971
|
-
|
2361
|
+
trace: Trace = get_trace(
|
2362
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2363
|
+
)
|
2364
|
+
trace.debug(f"[STAGE]: Execute Item: {item!r}")
|
1972
2365
|
key: StrOrInt = index if self.use_index_as_key else item
|
1973
2366
|
|
1974
2367
|
# NOTE: Create nested-context data from the passing context.
|
1975
|
-
|
1976
|
-
|
2368
|
+
current_context: DictData = copy.deepcopy(params)
|
2369
|
+
current_context.update({"item": item, "loop": index})
|
1977
2370
|
nestet_context: DictData = {"item": item, "stages": {}}
|
1978
2371
|
|
1979
2372
|
total_stage: int = len(self.stages)
|
@@ -1988,7 +2381,8 @@ class ForEachStage(BaseNestedStage):
|
|
1988
2381
|
"Item execution was canceled from the event before start "
|
1989
2382
|
"item execution."
|
1990
2383
|
)
|
1991
|
-
|
2384
|
+
catch(
|
2385
|
+
context=context,
|
1992
2386
|
status=CANCEL,
|
1993
2387
|
foreach={
|
1994
2388
|
key: {
|
@@ -2003,14 +2397,16 @@ class ForEachStage(BaseNestedStage):
|
|
2003
2397
|
)
|
2004
2398
|
raise StageCancelError(error_msg, refs=key)
|
2005
2399
|
|
2006
|
-
|
2007
|
-
|
2008
|
-
|
2009
|
-
|
2400
|
+
# NOTE: Nested-stage execute will pass only params and context only.
|
2401
|
+
rs: Result = stage.execute(
|
2402
|
+
params=current_context,
|
2403
|
+
run_id=parent_run_id,
|
2010
2404
|
event=event,
|
2011
2405
|
)
|
2012
2406
|
stage.set_outputs(rs.context, to=nestet_context)
|
2013
|
-
stage.set_outputs(
|
2407
|
+
stage.set_outputs(
|
2408
|
+
stage.get_outputs(nestet_context), to=current_context
|
2409
|
+
)
|
2014
2410
|
|
2015
2411
|
if rs.status == SKIP:
|
2016
2412
|
skips[i] = True
|
@@ -2021,8 +2417,9 @@ class ForEachStage(BaseNestedStage):
|
|
2021
2417
|
f"Item execution was break because its nested-stage, "
|
2022
2418
|
f"{stage.iden!r}, failed."
|
2023
2419
|
)
|
2024
|
-
|
2025
|
-
|
2420
|
+
trace.warning(f"[STAGE]: {error_msg}")
|
2421
|
+
catch(
|
2422
|
+
context=context,
|
2026
2423
|
status=FAILED,
|
2027
2424
|
foreach={
|
2028
2425
|
key: {
|
@@ -2042,7 +2439,8 @@ class ForEachStage(BaseNestedStage):
|
|
2042
2439
|
"Item execution was canceled from the event after "
|
2043
2440
|
"end item execution."
|
2044
2441
|
)
|
2045
|
-
|
2442
|
+
catch(
|
2443
|
+
context=context,
|
2046
2444
|
status=CANCEL,
|
2047
2445
|
foreach={
|
2048
2446
|
key: {
|
@@ -2058,7 +2456,8 @@ class ForEachStage(BaseNestedStage):
|
|
2058
2456
|
raise StageCancelError(error_msg, refs=key)
|
2059
2457
|
|
2060
2458
|
status: Status = SKIP if sum(skips) == total_stage else SUCCESS
|
2061
|
-
return status,
|
2459
|
+
return status, catch(
|
2460
|
+
context=context,
|
2062
2461
|
status=status,
|
2063
2462
|
foreach={
|
2064
2463
|
key: {
|
@@ -2069,11 +2468,13 @@ class ForEachStage(BaseNestedStage):
|
|
2069
2468
|
},
|
2070
2469
|
)
|
2071
2470
|
|
2072
|
-
def
|
2471
|
+
def process(
|
2073
2472
|
self,
|
2074
2473
|
params: DictData,
|
2474
|
+
run_id: str,
|
2475
|
+
context: DictData,
|
2075
2476
|
*,
|
2076
|
-
|
2477
|
+
parent_run_id: Optional[str] = None,
|
2077
2478
|
event: Optional[Event] = None,
|
2078
2479
|
) -> Result:
|
2079
2480
|
"""Execute the stages that pass each item form the foreach field.
|
@@ -2082,18 +2483,20 @@ class ForEachStage(BaseNestedStage):
|
|
2082
2483
|
value more than 1. It will cancel all nested-stage execution when it has
|
2083
2484
|
any item loop raise failed or canceled error.
|
2084
2485
|
|
2085
|
-
:
|
2086
|
-
|
2087
|
-
|
2088
|
-
|
2089
|
-
|
2090
|
-
|
2091
|
-
|
2092
|
-
|
2486
|
+
Args:
|
2487
|
+
params: A parameter data that want to use in this
|
2488
|
+
execution.
|
2489
|
+
run_id: A running stage ID.
|
2490
|
+
context: A context data.
|
2491
|
+
parent_run_id: A parent running ID. (Default is None)
|
2492
|
+
event: An event manager that use to track parent process
|
2493
|
+
was not force stopped.
|
2494
|
+
|
2495
|
+
Returns:
|
2496
|
+
Result: The execution result with status and context data.
|
2093
2497
|
"""
|
2094
|
-
|
2095
|
-
run_id=
|
2096
|
-
extras=self.extras,
|
2498
|
+
trace: Trace = get_trace(
|
2499
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2097
2500
|
)
|
2098
2501
|
event: Event = event or Event()
|
2099
2502
|
foreach: Union[list[str], list[int]] = pass_env(
|
@@ -2123,8 +2526,12 @@ class ForEachStage(BaseNestedStage):
|
|
2123
2526
|
"duplicate item, it should set `use_index_as_key: true`."
|
2124
2527
|
)
|
2125
2528
|
|
2126
|
-
|
2127
|
-
|
2529
|
+
trace.info(f"[STAGE]: Foreach: {foreach!r}.")
|
2530
|
+
catch(
|
2531
|
+
context=context,
|
2532
|
+
status=WAIT,
|
2533
|
+
updated={"items": foreach, "foreach": {}},
|
2534
|
+
)
|
2128
2535
|
len_foreach: int = len(foreach)
|
2129
2536
|
if event and event.is_set():
|
2130
2537
|
raise StageCancelError(
|
@@ -2134,30 +2541,32 @@ class ForEachStage(BaseNestedStage):
|
|
2134
2541
|
with ThreadPoolExecutor(self.concurrent, "stf") as executor:
|
2135
2542
|
futures: list[Future] = [
|
2136
2543
|
executor.submit(
|
2137
|
-
self.
|
2544
|
+
self._process_item,
|
2138
2545
|
index=i,
|
2139
2546
|
item=item,
|
2140
2547
|
params=params,
|
2141
|
-
|
2548
|
+
run_id=run_id,
|
2549
|
+
context=context,
|
2550
|
+
parent_run_id=parent_run_id,
|
2142
2551
|
event=event,
|
2143
2552
|
)
|
2144
2553
|
for i, item in enumerate(foreach, start=0)
|
2145
2554
|
]
|
2146
2555
|
|
2147
|
-
|
2556
|
+
errors: DictData = {}
|
2148
2557
|
statuses: list[Status] = [WAIT] * len_foreach
|
2149
2558
|
fail_fast: bool = False
|
2150
2559
|
|
2151
2560
|
done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
|
2152
2561
|
if len(list(done)) != len(futures):
|
2153
|
-
|
2562
|
+
trace.warning(
|
2154
2563
|
"[STAGE]: Set the event for stop pending for-each stage."
|
2155
2564
|
)
|
2156
2565
|
event.set()
|
2157
2566
|
for future in not_done:
|
2158
2567
|
future.cancel()
|
2159
2568
|
|
2160
|
-
time.sleep(0.025
|
2569
|
+
time.sleep(0.01) # Reduced from 0.025 for better responsiveness
|
2161
2570
|
nd: str = (
|
2162
2571
|
(
|
2163
2572
|
f", {len(not_done)} item"
|
@@ -2166,18 +2575,17 @@ class ForEachStage(BaseNestedStage):
|
|
2166
2575
|
if not_done
|
2167
2576
|
else ""
|
2168
2577
|
)
|
2169
|
-
|
2170
|
-
f"[STAGE]: ... Foreach-Stage set failed event{nd}"
|
2171
|
-
)
|
2578
|
+
trace.debug(f"[STAGE]: ... Foreach-Stage set failed event{nd}")
|
2172
2579
|
done: Iterator[Future] = as_completed(futures)
|
2173
2580
|
fail_fast = True
|
2174
2581
|
|
2175
2582
|
for i, future in enumerate(done, start=0):
|
2176
2583
|
try:
|
2584
|
+
# NOTE: Ignore returned context because it already updated.
|
2177
2585
|
statuses[i], _ = future.result()
|
2178
2586
|
except StageError as e:
|
2179
2587
|
statuses[i] = get_status_from_error(e)
|
2180
|
-
self.mark_errors(
|
2588
|
+
self.mark_errors(errors, e)
|
2181
2589
|
except CancelledError:
|
2182
2590
|
pass
|
2183
2591
|
|
@@ -2188,7 +2596,13 @@ class ForEachStage(BaseNestedStage):
|
|
2188
2596
|
if fail_fast and status == CANCEL:
|
2189
2597
|
status = FAILED
|
2190
2598
|
|
2191
|
-
return
|
2599
|
+
return Result(
|
2600
|
+
run_id=run_id,
|
2601
|
+
parent_run_id=parent_run_id,
|
2602
|
+
status=status,
|
2603
|
+
context=catch(context, status=status, updated=errors),
|
2604
|
+
extras=self.extras,
|
2605
|
+
)
|
2192
2606
|
|
2193
2607
|
|
2194
2608
|
class UntilStage(BaseNestedStage):
|
@@ -2239,32 +2653,40 @@ class UntilStage(BaseNestedStage):
|
|
2239
2653
|
alias="max-loop",
|
2240
2654
|
)
|
2241
2655
|
|
2242
|
-
def
|
2656
|
+
def _process_loop(
|
2243
2657
|
self,
|
2244
2658
|
item: T,
|
2245
2659
|
loop: int,
|
2246
2660
|
params: DictData,
|
2247
|
-
|
2661
|
+
run_id: str,
|
2662
|
+
context: DictData,
|
2663
|
+
*,
|
2664
|
+
parent_run_id: Optional[str] = None,
|
2248
2665
|
event: Optional[Event] = None,
|
2249
|
-
) -> tuple[Status,
|
2666
|
+
) -> tuple[Status, DictData, T]:
|
2250
2667
|
"""Execute loop that will execute all nested-stage that was set in this
|
2251
2668
|
stage with specific loop and item.
|
2252
2669
|
|
2253
2670
|
:param item: (T) An item that want to execution.
|
2254
2671
|
:param loop: (int) A number of loop.
|
2255
2672
|
:param params: (DictData) A parameter data.
|
2256
|
-
:param
|
2673
|
+
:param run_id: (str)
|
2674
|
+
:param context: (DictData)
|
2675
|
+
:param parent_run_id: (str | None)
|
2257
2676
|
:param event: (Event) An Event manager instance that use to cancel this
|
2258
2677
|
execution if it forces stopped by parent execution.
|
2259
2678
|
|
2260
|
-
:rtype: tuple[Status,
|
2679
|
+
:rtype: tuple[Status, DictData, T]
|
2261
2680
|
:return: Return a pair of Result and changed item.
|
2262
2681
|
"""
|
2263
|
-
|
2682
|
+
trace: Trace = get_trace(
|
2683
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2684
|
+
)
|
2685
|
+
trace.debug(f"[STAGE]: Execute Loop: {loop} (Item {item!r})")
|
2264
2686
|
|
2265
2687
|
# NOTE: Create nested-context
|
2266
|
-
|
2267
|
-
|
2688
|
+
current_context: DictData = copy.deepcopy(params)
|
2689
|
+
current_context.update({"item": item, "loop": loop})
|
2268
2690
|
nestet_context: DictData = {"loop": loop, "item": item, "stages": {}}
|
2269
2691
|
|
2270
2692
|
next_item: Optional[T] = None
|
@@ -2280,7 +2702,8 @@ class UntilStage(BaseNestedStage):
|
|
2280
2702
|
"Loop execution was canceled from the event before start "
|
2281
2703
|
"loop execution."
|
2282
2704
|
)
|
2283
|
-
|
2705
|
+
catch(
|
2706
|
+
context=context,
|
2284
2707
|
status=CANCEL,
|
2285
2708
|
until={
|
2286
2709
|
loop: {
|
@@ -2296,10 +2719,9 @@ class UntilStage(BaseNestedStage):
|
|
2296
2719
|
)
|
2297
2720
|
raise StageCancelError(error_msg, refs=loop)
|
2298
2721
|
|
2299
|
-
rs: Result = stage.
|
2300
|
-
params=
|
2301
|
-
run_id=
|
2302
|
-
parent_run_id=result.parent_run_id,
|
2722
|
+
rs: Result = stage.execute(
|
2723
|
+
params=current_context,
|
2724
|
+
run_id=parent_run_id,
|
2303
2725
|
event=event,
|
2304
2726
|
)
|
2305
2727
|
stage.set_outputs(rs.context, to=nestet_context)
|
@@ -2307,7 +2729,7 @@ class UntilStage(BaseNestedStage):
|
|
2307
2729
|
if "item" in (_output := stage.get_outputs(nestet_context)):
|
2308
2730
|
next_item = _output["item"]
|
2309
2731
|
|
2310
|
-
stage.set_outputs(_output, to=
|
2732
|
+
stage.set_outputs(_output, to=current_context)
|
2311
2733
|
|
2312
2734
|
if rs.status == SKIP:
|
2313
2735
|
skips[i] = True
|
@@ -2318,7 +2740,8 @@ class UntilStage(BaseNestedStage):
|
|
2318
2740
|
f"Loop execution was break because its nested-stage, "
|
2319
2741
|
f"{stage.iden!r}, failed."
|
2320
2742
|
)
|
2321
|
-
|
2743
|
+
catch(
|
2744
|
+
context=context,
|
2322
2745
|
status=FAILED,
|
2323
2746
|
until={
|
2324
2747
|
loop: {
|
@@ -2339,7 +2762,8 @@ class UntilStage(BaseNestedStage):
|
|
2339
2762
|
"Loop execution was canceled from the event after "
|
2340
2763
|
"end loop execution."
|
2341
2764
|
)
|
2342
|
-
|
2765
|
+
catch(
|
2766
|
+
context=context,
|
2343
2767
|
status=CANCEL,
|
2344
2768
|
until={
|
2345
2769
|
loop: {
|
@@ -2358,7 +2782,8 @@ class UntilStage(BaseNestedStage):
|
|
2358
2782
|
status: Status = SKIP if sum(skips) == total_stage else SUCCESS
|
2359
2783
|
return (
|
2360
2784
|
status,
|
2361
|
-
|
2785
|
+
catch(
|
2786
|
+
context=context,
|
2362
2787
|
status=status,
|
2363
2788
|
until={
|
2364
2789
|
loop: {
|
@@ -2372,37 +2797,42 @@ class UntilStage(BaseNestedStage):
|
|
2372
2797
|
next_item,
|
2373
2798
|
)
|
2374
2799
|
|
2375
|
-
def
|
2800
|
+
def process(
|
2376
2801
|
self,
|
2377
2802
|
params: DictData,
|
2803
|
+
run_id: str,
|
2804
|
+
context: DictData,
|
2378
2805
|
*,
|
2379
|
-
|
2806
|
+
parent_run_id: Optional[str] = None,
|
2380
2807
|
event: Optional[Event] = None,
|
2381
2808
|
) -> Result:
|
2382
2809
|
"""Execute until loop with checking the until condition before release
|
2383
2810
|
the next loop.
|
2384
2811
|
|
2385
|
-
:
|
2386
|
-
|
2387
|
-
|
2388
|
-
|
2389
|
-
|
2390
|
-
|
2391
|
-
|
2812
|
+
Args:
|
2813
|
+
params: A parameter data that want to use in this
|
2814
|
+
execution.
|
2815
|
+
run_id: A running stage ID.
|
2816
|
+
context: A context data.
|
2817
|
+
parent_run_id: A parent running ID. (Default is None)
|
2818
|
+
event: An event manager that use to track parent process
|
2819
|
+
was not force stopped.
|
2820
|
+
|
2821
|
+
Returns:
|
2822
|
+
Result: The execution result with status and context data.
|
2392
2823
|
"""
|
2393
|
-
|
2394
|
-
run_id=
|
2395
|
-
extras=self.extras,
|
2824
|
+
trace: Trace = get_trace(
|
2825
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2396
2826
|
)
|
2397
2827
|
event: Event = event or Event()
|
2398
|
-
|
2828
|
+
trace.info(f"[STAGE]: Until: {self.until!r}")
|
2399
2829
|
item: Union[str, int, bool] = pass_env(
|
2400
2830
|
param2template(self.item, params, extras=self.extras)
|
2401
2831
|
)
|
2402
2832
|
loop: int = 1
|
2403
2833
|
until_rs: bool = True
|
2404
2834
|
exceed_loop: bool = False
|
2405
|
-
|
2835
|
+
catch(context=context, status=WAIT, updated={"until": {}})
|
2406
2836
|
statuses: list[Status] = []
|
2407
2837
|
while until_rs and not (exceed_loop := (loop > self.max_loop)):
|
2408
2838
|
|
@@ -2411,18 +2841,20 @@ class UntilStage(BaseNestedStage):
|
|
2411
2841
|
"Execution was canceled from the event before start loop."
|
2412
2842
|
)
|
2413
2843
|
|
2414
|
-
status,
|
2844
|
+
status, context, item = self._process_loop(
|
2415
2845
|
item=item,
|
2416
2846
|
loop=loop,
|
2417
2847
|
params=params,
|
2418
|
-
|
2848
|
+
run_id=run_id,
|
2849
|
+
context=context,
|
2850
|
+
parent_run_id=parent_run_id,
|
2419
2851
|
event=event,
|
2420
2852
|
)
|
2421
2853
|
|
2422
2854
|
loop += 1
|
2423
2855
|
if item is None:
|
2424
2856
|
item: int = loop
|
2425
|
-
|
2857
|
+
trace.warning(
|
2426
2858
|
f"[STAGE]: Return loop not set the item. It uses loop: "
|
2427
2859
|
f"{loop} by default."
|
2428
2860
|
)
|
@@ -2453,7 +2885,15 @@ class UntilStage(BaseNestedStage):
|
|
2453
2885
|
f"loop{'s' if self.max_loop > 1 else ''}."
|
2454
2886
|
)
|
2455
2887
|
raise StageError(error_msg)
|
2456
|
-
|
2888
|
+
|
2889
|
+
st: Status = validate_statuses(statuses)
|
2890
|
+
return Result(
|
2891
|
+
run_id=run_id,
|
2892
|
+
parent_run_id=parent_run_id,
|
2893
|
+
status=st,
|
2894
|
+
context=catch(context, status=st),
|
2895
|
+
extras=self.extras,
|
2896
|
+
)
|
2457
2897
|
|
2458
2898
|
|
2459
2899
|
class Match(BaseModel):
|
@@ -2509,28 +2949,36 @@ class CaseStage(BaseNestedStage):
|
|
2509
2949
|
alias="skip-not-match",
|
2510
2950
|
)
|
2511
2951
|
|
2512
|
-
def
|
2952
|
+
def _process_case(
|
2513
2953
|
self,
|
2514
2954
|
case: str,
|
2515
2955
|
stages: list[Stage],
|
2516
2956
|
params: DictData,
|
2517
|
-
|
2957
|
+
run_id: str,
|
2958
|
+
context: DictData,
|
2518
2959
|
*,
|
2960
|
+
parent_run_id: Optional[str] = None,
|
2519
2961
|
event: Optional[Event] = None,
|
2520
|
-
) ->
|
2962
|
+
) -> tuple[Status, DictData]:
|
2521
2963
|
"""Execute case.
|
2522
2964
|
|
2523
2965
|
:param case: (str) A case that want to execution.
|
2524
2966
|
:param stages: (list[Stage]) A list of stage.
|
2525
2967
|
:param params: (DictData) A parameter data.
|
2526
|
-
:param
|
2968
|
+
:param run_id: (str)
|
2969
|
+
:param context: (DictData)
|
2970
|
+
:param parent_run_id: (str | None)
|
2527
2971
|
:param event: (Event) An Event manager instance that use to cancel this
|
2528
2972
|
execution if it forces stopped by parent execution.
|
2529
2973
|
|
2530
|
-
:rtype:
|
2974
|
+
:rtype: DictData
|
2531
2975
|
"""
|
2532
|
-
|
2533
|
-
|
2976
|
+
trace: Trace = get_trace(
|
2977
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2978
|
+
)
|
2979
|
+
trace.debug(f"[STAGE]: Execute Case: {case!r}")
|
2980
|
+
current_context: DictData = copy.deepcopy(params)
|
2981
|
+
current_context.update({"case": case})
|
2534
2982
|
output: DictData = {"case": case, "stages": {}}
|
2535
2983
|
for stage in stages:
|
2536
2984
|
|
@@ -2542,69 +2990,77 @@ class CaseStage(BaseNestedStage):
|
|
2542
2990
|
"Case-Stage was canceled from event that had set before "
|
2543
2991
|
"stage case execution."
|
2544
2992
|
)
|
2545
|
-
return
|
2993
|
+
return CANCEL, catch(
|
2994
|
+
context=context,
|
2546
2995
|
status=CANCEL,
|
2547
|
-
|
2996
|
+
updated={
|
2548
2997
|
"case": case,
|
2549
2998
|
"stages": filter_func(output.pop("stages", {})),
|
2550
2999
|
"errors": StageError(error_msg).to_dict(),
|
2551
3000
|
},
|
2552
3001
|
)
|
2553
3002
|
|
2554
|
-
rs: Result = stage.
|
2555
|
-
params=
|
2556
|
-
run_id=
|
2557
|
-
parent_run_id=result.parent_run_id,
|
3003
|
+
rs: Result = stage.execute(
|
3004
|
+
params=current_context,
|
3005
|
+
run_id=parent_run_id,
|
2558
3006
|
event=event,
|
2559
3007
|
)
|
2560
3008
|
stage.set_outputs(rs.context, to=output)
|
2561
|
-
stage.set_outputs(stage.get_outputs(output), to=
|
3009
|
+
stage.set_outputs(stage.get_outputs(output), to=current_context)
|
2562
3010
|
|
2563
3011
|
if rs.status == FAILED:
|
2564
3012
|
error_msg: str = (
|
2565
3013
|
f"Case-Stage was break because it has a sub stage, "
|
2566
3014
|
f"{stage.iden}, failed without raise error."
|
2567
3015
|
)
|
2568
|
-
return
|
3016
|
+
return FAILED, catch(
|
3017
|
+
context=context,
|
2569
3018
|
status=FAILED,
|
2570
|
-
|
3019
|
+
updated={
|
2571
3020
|
"case": case,
|
2572
3021
|
"stages": filter_func(output.pop("stages", {})),
|
2573
3022
|
"errors": StageError(error_msg).to_dict(),
|
2574
3023
|
},
|
2575
3024
|
)
|
2576
|
-
return
|
3025
|
+
return SUCCESS, catch(
|
3026
|
+
context=context,
|
2577
3027
|
status=SUCCESS,
|
2578
|
-
|
3028
|
+
updated={
|
2579
3029
|
"case": case,
|
2580
3030
|
"stages": filter_func(output.pop("stages", {})),
|
2581
3031
|
},
|
2582
3032
|
)
|
2583
3033
|
|
2584
|
-
def
|
3034
|
+
def process(
|
2585
3035
|
self,
|
2586
3036
|
params: DictData,
|
3037
|
+
run_id: str,
|
3038
|
+
context: DictData,
|
2587
3039
|
*,
|
2588
|
-
|
3040
|
+
parent_run_id: Optional[str] = None,
|
2589
3041
|
event: Optional[Event] = None,
|
2590
3042
|
) -> Result:
|
2591
3043
|
"""Execute case-match condition that pass to the case field.
|
2592
3044
|
|
2593
|
-
:
|
2594
|
-
|
2595
|
-
|
2596
|
-
|
2597
|
-
|
2598
|
-
|
3045
|
+
Args:
|
3046
|
+
params: A parameter data that want to use in this
|
3047
|
+
execution.
|
3048
|
+
run_id: A running stage ID.
|
3049
|
+
context: A context data.
|
3050
|
+
parent_run_id: A parent running ID. (Default is None)
|
3051
|
+
event: An event manager that use to track parent process
|
3052
|
+
was not force stopped.
|
3053
|
+
|
3054
|
+
Returns:
|
3055
|
+
Result: The execution result with status and context data.
|
2599
3056
|
"""
|
2600
|
-
|
2601
|
-
run_id=
|
2602
|
-
extras=self.extras,
|
3057
|
+
trace: Trace = get_trace(
|
3058
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2603
3059
|
)
|
2604
3060
|
|
2605
3061
|
_case: StrOrNone = param2template(self.case, params, extras=self.extras)
|
2606
3062
|
|
2607
|
-
|
3063
|
+
trace.info(f"[STAGE]: Case: {_case!r}.")
|
2608
3064
|
_else: Optional[Match] = None
|
2609
3065
|
stages: Optional[list[Stage]] = None
|
2610
3066
|
for match in self.match:
|
@@ -2636,9 +3092,21 @@ class CaseStage(BaseNestedStage):
|
|
2636
3092
|
"Execution was canceled from the event before start "
|
2637
3093
|
"case execution."
|
2638
3094
|
)
|
2639
|
-
|
2640
|
-
|
2641
|
-
|
3095
|
+
status, context = self._process_case(
|
3096
|
+
case=_case,
|
3097
|
+
stages=stages,
|
3098
|
+
params=params,
|
3099
|
+
run_id=run_id,
|
3100
|
+
context=context,
|
3101
|
+
parent_run_id=parent_run_id,
|
3102
|
+
event=event,
|
3103
|
+
)
|
3104
|
+
return Result(
|
3105
|
+
run_id=run_id,
|
3106
|
+
parent_run_id=parent_run_id,
|
3107
|
+
status=status,
|
3108
|
+
context=catch(context, status=status),
|
3109
|
+
extras=self.extras,
|
2642
3110
|
)
|
2643
3111
|
|
2644
3112
|
|
@@ -2661,53 +3129,65 @@ class RaiseStage(BaseAsyncStage):
|
|
2661
3129
|
alias="raise",
|
2662
3130
|
)
|
2663
3131
|
|
2664
|
-
def
|
3132
|
+
def process(
|
2665
3133
|
self,
|
2666
3134
|
params: DictData,
|
3135
|
+
run_id: str,
|
3136
|
+
context: DictData,
|
2667
3137
|
*,
|
2668
|
-
|
3138
|
+
parent_run_id: Optional[str] = None,
|
2669
3139
|
event: Optional[Event] = None,
|
2670
3140
|
) -> Result:
|
2671
3141
|
"""Raise the StageError object with the message field execution.
|
2672
3142
|
|
2673
|
-
:
|
2674
|
-
|
2675
|
-
|
2676
|
-
|
3143
|
+
Args:
|
3144
|
+
params: A parameter data that want to use in this
|
3145
|
+
execution.
|
3146
|
+
run_id: A running stage ID.
|
3147
|
+
context: A context data.
|
3148
|
+
parent_run_id: A parent running ID. (Default is None)
|
3149
|
+
event: An event manager that use to track parent process
|
3150
|
+
was not force stopped.
|
3151
|
+
|
3152
|
+
Returns:
|
3153
|
+
Result: The execution result with status and context data.
|
2677
3154
|
"""
|
2678
|
-
|
2679
|
-
run_id=
|
2680
|
-
extras=self.extras,
|
3155
|
+
trace: Trace = get_trace(
|
3156
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2681
3157
|
)
|
2682
3158
|
message: str = param2template(self.message, params, extras=self.extras)
|
2683
|
-
|
3159
|
+
trace.info(f"[STAGE]: Message: ( {message} )")
|
2684
3160
|
raise StageError(message)
|
2685
3161
|
|
2686
|
-
async def
|
3162
|
+
async def async_process(
|
2687
3163
|
self,
|
2688
3164
|
params: DictData,
|
3165
|
+
run_id: str,
|
3166
|
+
context: DictData,
|
2689
3167
|
*,
|
2690
|
-
|
3168
|
+
parent_run_id: Optional[str] = None,
|
2691
3169
|
event: Optional[Event] = None,
|
2692
3170
|
) -> Result:
|
2693
3171
|
"""Async execution method for this Empty stage that only logging out to
|
2694
3172
|
stdout.
|
2695
3173
|
|
2696
|
-
:
|
2697
|
-
|
2698
|
-
|
2699
|
-
|
2700
|
-
|
2701
|
-
|
2702
|
-
|
2703
|
-
|
3174
|
+
Args:
|
3175
|
+
params: A parameter data that want to use in this
|
3176
|
+
execution.
|
3177
|
+
run_id: A running stage ID.
|
3178
|
+
context: A context data.
|
3179
|
+
parent_run_id: A parent running ID. (Default is None)
|
3180
|
+
event: An event manager that use to track parent process
|
3181
|
+
was not force stopped.
|
3182
|
+
|
3183
|
+
Returns:
|
3184
|
+
Result: The execution result with status and context data.
|
2704
3185
|
"""
|
2705
|
-
|
2706
|
-
run_id=
|
2707
|
-
extras=self.extras,
|
3186
|
+
trace: Trace = get_trace(
|
3187
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2708
3188
|
)
|
2709
3189
|
message: str = param2template(self.message, params, extras=self.extras)
|
2710
|
-
await
|
3190
|
+
await trace.ainfo(f"[STAGE]: Execute Raise-Stage: ( {message} )")
|
2711
3191
|
raise StageError(message)
|
2712
3192
|
|
2713
3193
|
|
@@ -2755,20 +3235,25 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2755
3235
|
),
|
2756
3236
|
)
|
2757
3237
|
|
2758
|
-
def
|
3238
|
+
def _process_task(
|
2759
3239
|
self,
|
2760
3240
|
params: DictData,
|
2761
|
-
|
3241
|
+
run_id: str,
|
3242
|
+
context: DictData,
|
3243
|
+
*,
|
3244
|
+
parent_run_id: Optional[str] = None,
|
2762
3245
|
event: Optional[Event] = None,
|
2763
|
-
) ->
|
3246
|
+
) -> DictData:
|
2764
3247
|
"""Execute Docker container task.
|
2765
3248
|
|
2766
3249
|
:param params: (DictData) A parameter data.
|
2767
|
-
:param
|
3250
|
+
:param run_id: (str)
|
3251
|
+
:param context: (DictData)
|
3252
|
+
:param parent_run_id: (str | None)
|
2768
3253
|
:param event: (Event) An Event manager instance that use to cancel this
|
2769
3254
|
execution if it forces stopped by parent execution.
|
2770
3255
|
|
2771
|
-
:rtype:
|
3256
|
+
:rtype: DictData
|
2772
3257
|
"""
|
2773
3258
|
try:
|
2774
3259
|
from docker import DockerClient
|
@@ -2779,6 +3264,9 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2779
3264
|
"by `pip install docker` first."
|
2780
3265
|
) from None
|
2781
3266
|
|
3267
|
+
trace: Trace = get_trace(
|
3268
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
3269
|
+
)
|
2782
3270
|
client = DockerClient(
|
2783
3271
|
base_url="unix://var/run/docker.sock", version="auto"
|
2784
3272
|
)
|
@@ -2793,16 +3281,17 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2793
3281
|
decode=True,
|
2794
3282
|
)
|
2795
3283
|
for line in resp:
|
2796
|
-
|
3284
|
+
trace.info(f"[STAGE]: ... {line}")
|
2797
3285
|
|
2798
3286
|
if event and event.is_set():
|
2799
3287
|
error_msg: str = (
|
2800
3288
|
"Docker-Stage was canceled from event that had set before "
|
2801
3289
|
"run the Docker container."
|
2802
3290
|
)
|
2803
|
-
return
|
3291
|
+
return catch(
|
3292
|
+
context=context,
|
2804
3293
|
status=CANCEL,
|
2805
|
-
|
3294
|
+
updated={"errors": StageError(error_msg).to_dict()},
|
2806
3295
|
)
|
2807
3296
|
|
2808
3297
|
unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S%f}"
|
@@ -2813,7 +3302,7 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2813
3302
|
volumes=pass_env(
|
2814
3303
|
{
|
2815
3304
|
Path.cwd()
|
2816
|
-
/ f".docker.{
|
3305
|
+
/ f".docker.{run_id}.logs": {
|
2817
3306
|
"bind": "/logs",
|
2818
3307
|
"mode": "rw",
|
2819
3308
|
},
|
@@ -2829,7 +3318,7 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2829
3318
|
)
|
2830
3319
|
|
2831
3320
|
for line in container.logs(stream=True, timestamps=True):
|
2832
|
-
|
3321
|
+
trace.info(f"[STAGE]: ... {line.strip().decode()}")
|
2833
3322
|
|
2834
3323
|
# NOTE: This code copy from the docker package.
|
2835
3324
|
exit_status: int = container.wait()["StatusCode"]
|
@@ -2843,36 +3332,42 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2843
3332
|
f"{self.image}:{self.tag}",
|
2844
3333
|
out.decode("utf-8"),
|
2845
3334
|
)
|
2846
|
-
output_file: Path = Path(f".docker.{
|
3335
|
+
output_file: Path = Path(f".docker.{run_id}.logs/outputs.json")
|
2847
3336
|
if not output_file.exists():
|
2848
|
-
return
|
2849
|
-
|
2850
|
-
|
2851
|
-
|
2852
|
-
|
3337
|
+
return catch(context=context, status=SUCCESS)
|
3338
|
+
return catch(
|
3339
|
+
context=context,
|
3340
|
+
status=SUCCESS,
|
3341
|
+
updated=json.loads(output_file.read_text()),
|
3342
|
+
)
|
2853
3343
|
|
2854
|
-
def
|
3344
|
+
def process(
|
2855
3345
|
self,
|
2856
3346
|
params: DictData,
|
3347
|
+
run_id: str,
|
3348
|
+
context: DictData,
|
2857
3349
|
*,
|
2858
|
-
|
3350
|
+
parent_run_id: Optional[str] = None,
|
2859
3351
|
event: Optional[Event] = None,
|
2860
3352
|
) -> Result:
|
2861
3353
|
"""Execute the Docker image via Python API.
|
2862
3354
|
|
2863
|
-
:
|
2864
|
-
|
2865
|
-
|
2866
|
-
|
2867
|
-
|
2868
|
-
|
3355
|
+
Args:
|
3356
|
+
params: A parameter data that want to use in this
|
3357
|
+
execution.
|
3358
|
+
run_id: A running stage ID.
|
3359
|
+
context: A context data.
|
3360
|
+
parent_run_id: A parent running ID. (Default is None)
|
3361
|
+
event: An event manager that use to track parent process
|
3362
|
+
was not force stopped.
|
3363
|
+
|
3364
|
+
Returns:
|
3365
|
+
Result: The execution result with status and context data.
|
2869
3366
|
"""
|
2870
|
-
|
2871
|
-
run_id=
|
2872
|
-
extras=self.extras,
|
3367
|
+
trace: Trace = get_trace(
|
3368
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2873
3369
|
)
|
2874
|
-
|
2875
|
-
result.trace.info(f"[STAGE]: Docker: {self.image}:{self.tag}")
|
3370
|
+
trace.info(f"[STAGE]: Docker: {self.image}:{self.tag}")
|
2876
3371
|
raise NotImplementedError("Docker Stage does not implement yet.")
|
2877
3372
|
|
2878
3373
|
|
@@ -2947,11 +3442,18 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
2947
3442
|
# Note: Remove .py file that use to run Python.
|
2948
3443
|
Path(f"./{f_name}").unlink()
|
2949
3444
|
|
2950
|
-
|
3445
|
+
@staticmethod
|
3446
|
+
def prepare_std(value: str) -> Optional[str]:
|
3447
|
+
"""Prepare returned standard string from subprocess."""
|
3448
|
+
return None if (out := value.strip("\n")) == "" else out
|
3449
|
+
|
3450
|
+
def process(
|
2951
3451
|
self,
|
2952
3452
|
params: DictData,
|
3453
|
+
run_id: str,
|
3454
|
+
context: DictData,
|
2953
3455
|
*,
|
2954
|
-
|
3456
|
+
parent_run_id: Optional[str] = None,
|
2955
3457
|
event: Optional[Event] = None,
|
2956
3458
|
) -> Result:
|
2957
3459
|
"""Execute the Python statement via Python virtual environment.
|
@@ -2960,25 +3462,29 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
2960
3462
|
- Create python file with the `uv` syntax.
|
2961
3463
|
- Execution python file with `uv run` via Python subprocess module.
|
2962
3464
|
|
2963
|
-
:
|
2964
|
-
|
2965
|
-
|
2966
|
-
|
2967
|
-
|
2968
|
-
|
3465
|
+
Args:
|
3466
|
+
params: A parameter data that want to use in this
|
3467
|
+
execution.
|
3468
|
+
run_id: A running stage ID.
|
3469
|
+
context: A context data.
|
3470
|
+
parent_run_id: A parent running ID. (Default is None)
|
3471
|
+
event: An event manager that use to track parent process
|
3472
|
+
was not force stopped.
|
3473
|
+
|
3474
|
+
Returns:
|
3475
|
+
Result: The execution result with status and context data.
|
2969
3476
|
"""
|
2970
|
-
|
2971
|
-
run_id=
|
2972
|
-
extras=self.extras,
|
3477
|
+
trace: Trace = get_trace(
|
3478
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2973
3479
|
)
|
2974
3480
|
run: str = param2template(dedent(self.run), params, extras=self.extras)
|
2975
3481
|
with self.create_py_file(
|
2976
3482
|
py=run,
|
2977
3483
|
values=param2template(self.vars, params, extras=self.extras),
|
2978
3484
|
deps=param2template(self.deps, params, extras=self.extras),
|
2979
|
-
run_id=
|
3485
|
+
run_id=run_id,
|
2980
3486
|
) as py:
|
2981
|
-
|
3487
|
+
trace.debug(f"[STAGE]: Create `{py}` file.")
|
2982
3488
|
rs: CompletedProcess = subprocess.run(
|
2983
3489
|
["python", "-m", "uv", "run", py, "--no-cache"],
|
2984
3490
|
# ["uv", "run", "--python", "3.9", py],
|
@@ -2998,13 +3504,20 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
2998
3504
|
f"Subprocess: {e}\nRunning Statement:\n---\n"
|
2999
3505
|
f"```python\n{run}\n```"
|
3000
3506
|
)
|
3001
|
-
return
|
3507
|
+
return Result(
|
3508
|
+
run_id=run_id,
|
3509
|
+
parent_run_id=parent_run_id,
|
3002
3510
|
status=SUCCESS,
|
3003
|
-
context=
|
3004
|
-
|
3005
|
-
|
3006
|
-
|
3007
|
-
|
3511
|
+
context=catch(
|
3512
|
+
context=context,
|
3513
|
+
status=SUCCESS,
|
3514
|
+
updated={
|
3515
|
+
"return_code": rs.returncode,
|
3516
|
+
"stdout": self.prepare_std(rs.stdout),
|
3517
|
+
"stderr": self.prepare_std(rs.stderr),
|
3518
|
+
},
|
3519
|
+
),
|
3520
|
+
extras=self.extras,
|
3008
3521
|
)
|
3009
3522
|
|
3010
3523
|
|