ddeutil-workflow 0.0.73__py3-none-any.whl → 0.0.75__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__cron.py +20 -12
- ddeutil/workflow/__init__.py +119 -10
- ddeutil/workflow/__types.py +53 -41
- ddeutil/workflow/api/__init__.py +74 -3
- ddeutil/workflow/api/routes/job.py +15 -29
- ddeutil/workflow/api/routes/logs.py +9 -9
- ddeutil/workflow/api/routes/workflows.py +3 -3
- ddeutil/workflow/audits.py +70 -55
- ddeutil/workflow/cli.py +1 -15
- ddeutil/workflow/conf.py +71 -26
- ddeutil/workflow/errors.py +86 -19
- ddeutil/workflow/event.py +268 -169
- ddeutil/workflow/job.py +331 -192
- ddeutil/workflow/params.py +43 -11
- ddeutil/workflow/result.py +96 -70
- ddeutil/workflow/reusables.py +56 -6
- ddeutil/workflow/stages.py +1059 -572
- ddeutil/workflow/traces.py +205 -124
- ddeutil/workflow/utils.py +58 -19
- ddeutil/workflow/workflow.py +435 -296
- {ddeutil_workflow-0.0.73.dist-info → ddeutil_workflow-0.0.75.dist-info}/METADATA +27 -17
- ddeutil_workflow-0.0.75.dist-info/RECORD +30 -0
- ddeutil_workflow-0.0.73.dist-info/RECORD +0 -30
- {ddeutil_workflow-0.0.73.dist-info → ddeutil_workflow-0.0.75.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.73.dist-info → ddeutil_workflow-0.0.75.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.73.dist-info → ddeutil_workflow-0.0.75.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.73.dist-info → ddeutil_workflow-0.0.75.dist-info}/top_level.txt +0 -0
ddeutil/workflow/stages.py
CHANGED
@@ -12,7 +12,7 @@ you can track logs.
|
|
12
12
|
I do not want to handle stage error on this stage execution. I think stage model
|
13
13
|
have a lot of use-case, and it should does not worry about it error output.
|
14
14
|
|
15
|
-
So, I will create `
|
15
|
+
So, I will create `execute` for any exception class that raise from
|
16
16
|
the stage execution method.
|
17
17
|
|
18
18
|
Handler --> Ok --> Result
|
@@ -32,7 +32,7 @@ the stage execution method.
|
|
32
32
|
|-name: ...
|
33
33
|
╰-message: ...
|
34
34
|
|
35
|
-
On the context I/O that pass to a stage object at execute
|
35
|
+
On the context I/O that pass to a stage object at execute step. The
|
36
36
|
execute method receives a `params={"params": {...}}` value for passing template
|
37
37
|
searching.
|
38
38
|
|
@@ -95,6 +95,7 @@ from .result import (
|
|
95
95
|
WAIT,
|
96
96
|
Result,
|
97
97
|
Status,
|
98
|
+
catch,
|
98
99
|
get_status_from_error,
|
99
100
|
validate_statuses,
|
100
101
|
)
|
@@ -105,6 +106,7 @@ from .reusables import (
|
|
105
106
|
not_in_template,
|
106
107
|
param2template,
|
107
108
|
)
|
109
|
+
from .traces import Trace, get_trace
|
108
110
|
from .utils import (
|
109
111
|
delay,
|
110
112
|
dump_all,
|
@@ -119,13 +121,40 @@ DictOrModel = Union[DictData, BaseModel]
|
|
119
121
|
|
120
122
|
|
121
123
|
class BaseStage(BaseModel, ABC):
|
122
|
-
"""
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
124
|
+
"""Abstract base class for all stage implementations.
|
125
|
+
|
126
|
+
BaseStage provides the foundation for all stage types in the workflow system.
|
127
|
+
It defines the common interface and metadata fields that all stages must
|
128
|
+
implement, ensuring consistent behavior across different stage types.
|
129
|
+
|
130
|
+
This abstract class handles core stage functionality including:
|
131
|
+
- Stage identification and naming
|
132
|
+
- Conditional execution logic
|
133
|
+
- Output management and templating
|
134
|
+
- Execution lifecycle management
|
135
|
+
|
136
|
+
Custom stages should inherit from this class and implement the abstract
|
137
|
+
`process()` method to define their specific execution behavior.
|
138
|
+
|
139
|
+
Attributes:
|
140
|
+
extras (dict): Additional configuration parameters
|
141
|
+
id (str, optional): Unique stage identifier for output reference
|
142
|
+
name (str): Human-readable stage name for logging
|
143
|
+
desc (str, optional): Stage description for documentation
|
144
|
+
condition (str, optional): Conditional expression for execution
|
145
|
+
|
146
|
+
Abstract Methods:
|
147
|
+
process: Main execution logic that must be implemented by subclasses
|
148
|
+
|
149
|
+
Example:
|
150
|
+
```python
|
151
|
+
class CustomStage(BaseStage):
|
152
|
+
custom_param: str = Field(description="Custom parameter")
|
153
|
+
|
154
|
+
def process(self, params: dict, **kwargs) -> Result:
|
155
|
+
# Custom execution logic
|
156
|
+
return Result(status=SUCCESS)
|
157
|
+
```
|
129
158
|
"""
|
130
159
|
|
131
160
|
extras: DictData = Field(
|
@@ -162,7 +191,8 @@ class BaseStage(BaseModel, ABC):
|
|
162
191
|
"""Return this stage identity that return the `id` field first and if
|
163
192
|
this `id` field does not set, it will use the `name` field instead.
|
164
193
|
|
165
|
-
:
|
194
|
+
Returns:
|
195
|
+
str: Return an identity of this stage for making output.
|
166
196
|
"""
|
167
197
|
return self.id or self.name
|
168
198
|
|
@@ -194,37 +224,40 @@ class BaseStage(BaseModel, ABC):
|
|
194
224
|
return self
|
195
225
|
|
196
226
|
@abstractmethod
|
197
|
-
def
|
227
|
+
def process(
|
198
228
|
self,
|
199
229
|
params: DictData,
|
230
|
+
run_id: str,
|
231
|
+
context: DictData,
|
200
232
|
*,
|
201
|
-
|
233
|
+
parent_run_id: Optional[str] = None,
|
202
234
|
event: Optional[Event] = None,
|
203
235
|
) -> Result:
|
204
|
-
"""
|
236
|
+
"""Process abstraction method that action something by sub-model class.
|
205
237
|
This is important method that make this class is able to be the stage.
|
206
238
|
|
207
|
-
:
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
239
|
+
Args:
|
240
|
+
params: A parameter data that want to use in this
|
241
|
+
execution.
|
242
|
+
run_id: A running stage ID.
|
243
|
+
context: A context data.
|
244
|
+
parent_run_id: A parent running ID. (Default is None)
|
245
|
+
event: An event manager that use to track parent process
|
246
|
+
was not force stopped.
|
247
|
+
|
248
|
+
Returns:
|
249
|
+
Result: The execution result with status and context data.
|
215
250
|
"""
|
216
|
-
raise NotImplementedError("Stage should implement `
|
251
|
+
raise NotImplementedError("Stage should implement `process` method.")
|
217
252
|
|
218
|
-
def
|
253
|
+
def execute(
|
219
254
|
self,
|
220
255
|
params: DictData,
|
221
256
|
*,
|
222
257
|
run_id: StrOrNone = None,
|
223
|
-
parent_run_id: StrOrNone = None,
|
224
|
-
result: Optional[Result] = None,
|
225
258
|
event: Optional[Event] = None,
|
226
259
|
) -> Union[Result, DictData]:
|
227
|
-
"""Handler stage execution result from the stage `
|
260
|
+
"""Handler stage execution result from the stage `process` method.
|
228
261
|
|
229
262
|
This handler strategy will catch and mapping message to the result
|
230
263
|
context data before returning. All possible status that will return from
|
@@ -250,33 +283,36 @@ class BaseStage(BaseModel, ABC):
|
|
250
283
|
On the last step, it will set the running ID on a return result
|
251
284
|
object from the current stage ID before release the final result.
|
252
285
|
|
253
|
-
:
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
(Default is None)
|
259
|
-
:param event: (Event) An event manager that pass to the stage execution.
|
260
|
-
(Default is None)
|
286
|
+
Args:
|
287
|
+
params: A parameter data.
|
288
|
+
run_id: A running stage ID. (Default is None)
|
289
|
+
event: An event manager that pass to the stage execution.
|
290
|
+
(Default is None)
|
261
291
|
|
262
|
-
:
|
292
|
+
Returns:
|
293
|
+
Result: The execution result with updated status and context.
|
263
294
|
"""
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
extras=self.extras
|
295
|
+
ts: float = time.monotonic()
|
296
|
+
parent_run_id: str = run_id
|
297
|
+
run_id: str = run_id or gen_id(self.iden, unique=True)
|
298
|
+
context: DictData = {}
|
299
|
+
trace: Trace = get_trace(
|
300
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
270
301
|
)
|
271
302
|
try:
|
272
|
-
|
303
|
+
_id: str = (
|
304
|
+
f" with ID: {param2template(self.id, params=params)!r}"
|
305
|
+
if self.id
|
306
|
+
else ""
|
307
|
+
)
|
308
|
+
trace.info(
|
273
309
|
f"[STAGE]: Handler {to_train(self.__class__.__name__)}: "
|
274
|
-
f"{self.name!r}."
|
310
|
+
f"{self.name!r}{_id}."
|
275
311
|
)
|
276
312
|
|
277
313
|
# NOTE: Show the description of this stage before execution.
|
278
314
|
if self.desc:
|
279
|
-
|
315
|
+
trace.debug(f"[STAGE]: Description:||{self.desc}||")
|
280
316
|
|
281
317
|
# VALIDATE: Checking stage condition before execution.
|
282
318
|
if self.is_skipped(params):
|
@@ -287,13 +323,19 @@ class BaseStage(BaseModel, ABC):
|
|
287
323
|
# NOTE: Start call wrapped execution method that will use custom
|
288
324
|
# execution before the real execution from inherit stage model.
|
289
325
|
result_caught: Result = self._execute(
|
290
|
-
params,
|
326
|
+
params,
|
327
|
+
run_id=run_id,
|
328
|
+
context=context,
|
329
|
+
parent_run_id=parent_run_id,
|
330
|
+
event=event,
|
291
331
|
)
|
292
332
|
if result_caught.status == WAIT:
|
293
333
|
raise StageError(
|
294
334
|
"Status from execution should not return waiting status."
|
295
335
|
)
|
296
|
-
return result_caught
|
336
|
+
return result_caught.make_info(
|
337
|
+
{"execution_time": time.monotonic() - ts}
|
338
|
+
)
|
297
339
|
|
298
340
|
# NOTE: Catch this error in this line because the execution can raise
|
299
341
|
# this exception class at other location.
|
@@ -302,48 +344,82 @@ class BaseStage(BaseModel, ABC):
|
|
302
344
|
StageCancelError,
|
303
345
|
StageError,
|
304
346
|
) as e: # pragma: no cov
|
305
|
-
|
347
|
+
trace.info(
|
306
348
|
f"[STAGE]: Handler:||{e.__class__.__name__}: {e}||"
|
307
349
|
f"{traceback.format_exc()}"
|
308
350
|
)
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
351
|
+
st: Status = get_status_from_error(e)
|
352
|
+
return Result(
|
353
|
+
run_id=run_id,
|
354
|
+
parent_run_id=parent_run_id,
|
355
|
+
status=st,
|
356
|
+
context=catch(
|
357
|
+
context,
|
358
|
+
status=st,
|
359
|
+
updated=(
|
360
|
+
None
|
361
|
+
if isinstance(e, StageSkipError)
|
362
|
+
else {"errors": e.to_dict()}
|
363
|
+
),
|
315
364
|
),
|
365
|
+
info={"execution_time": time.monotonic() - ts},
|
366
|
+
extras=self.extras,
|
316
367
|
)
|
317
368
|
except Exception as e:
|
318
|
-
|
369
|
+
trace.error(
|
319
370
|
f"[STAGE]: Error Handler:||{e.__class__.__name__}: {e}||"
|
320
371
|
f"{traceback.format_exc()}"
|
321
372
|
)
|
322
|
-
return
|
373
|
+
return Result(
|
374
|
+
run_id=run_id,
|
375
|
+
parent_run_id=parent_run_id,
|
376
|
+
status=FAILED,
|
377
|
+
context=catch(
|
378
|
+
context, status=FAILED, updated={"errors": to_dict(e)}
|
379
|
+
),
|
380
|
+
info={"execution_time": time.monotonic() - ts},
|
381
|
+
extras=self.extras,
|
382
|
+
)
|
323
383
|
|
324
384
|
def _execute(
|
325
|
-
self,
|
385
|
+
self,
|
386
|
+
params: DictData,
|
387
|
+
run_id: str,
|
388
|
+
context: DictData,
|
389
|
+
parent_run_id: Optional[str] = None,
|
390
|
+
event: Optional[Event] = None,
|
326
391
|
) -> Result:
|
327
|
-
"""Wrapped the
|
392
|
+
"""Wrapped the process method before returning to handler execution.
|
328
393
|
|
329
|
-
:
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
was not force stopped.
|
394
|
+
Args:
|
395
|
+
params: A parameter data that want to use in this
|
396
|
+
execution.
|
397
|
+
event: An event manager that use to track parent process
|
398
|
+
was not force stopped.
|
335
399
|
|
336
|
-
:
|
400
|
+
Returns:
|
401
|
+
Result: The wrapped execution result.
|
337
402
|
"""
|
338
|
-
|
339
|
-
return self.
|
403
|
+
catch(context, status=WAIT)
|
404
|
+
return self.process(
|
405
|
+
params,
|
406
|
+
run_id=run_id,
|
407
|
+
context=context,
|
408
|
+
parent_run_id=parent_run_id,
|
409
|
+
event=event,
|
410
|
+
)
|
340
411
|
|
341
|
-
def set_outputs(
|
412
|
+
def set_outputs(
|
413
|
+
self,
|
414
|
+
output: DictData,
|
415
|
+
to: DictData,
|
416
|
+
info: Optional[DictData] = None,
|
417
|
+
) -> DictData:
|
342
418
|
"""Set an outputs from execution result context to the received context
|
343
419
|
with a `to` input parameter. The result context from stage execution
|
344
420
|
will be set with `outputs` key in this stage ID key.
|
345
421
|
|
346
|
-
For example of setting output method, If you receive
|
422
|
+
For example of setting output method, If you receive process output
|
347
423
|
and want to set on the `to` like;
|
348
424
|
|
349
425
|
... (i) output: {'foo': 'bar', 'skipped': True}
|
@@ -374,6 +450,7 @@ class BaseStage(BaseModel, ABC):
|
|
374
450
|
:param output: (DictData) A result data context that want to extract
|
375
451
|
and transfer to the `outputs` key in receive context.
|
376
452
|
:param to: (DictData) A received context data.
|
453
|
+
:param info: (DictData)
|
377
454
|
|
378
455
|
:rtype: DictData
|
379
456
|
"""
|
@@ -393,7 +470,8 @@ class BaseStage(BaseModel, ABC):
|
|
393
470
|
status: dict[str, Status] = (
|
394
471
|
{"status": output.pop("status")} if "status" in output else {}
|
395
472
|
)
|
396
|
-
|
473
|
+
info: DictData = {"info": info} if info else {}
|
474
|
+
to["stages"][_id] = {"outputs": output} | errors | status | info
|
397
475
|
return to
|
398
476
|
|
399
477
|
def get_outputs(self, output: DictData) -> DictData:
|
@@ -460,6 +538,7 @@ class BaseStage(BaseModel, ABC):
|
|
460
538
|
param2template(self.id, params=params, extras=self.extras)
|
461
539
|
if self.id
|
462
540
|
else gen_id(
|
541
|
+
# NOTE: The name should be non-sensitive case for uniqueness.
|
463
542
|
param2template(self.name, params=params, extras=self.extras)
|
464
543
|
)
|
465
544
|
)
|
@@ -491,67 +570,74 @@ class BaseAsyncStage(BaseStage, ABC):
|
|
491
570
|
"""
|
492
571
|
|
493
572
|
@abstractmethod
|
494
|
-
async def
|
573
|
+
async def async_process(
|
495
574
|
self,
|
496
575
|
params: DictData,
|
576
|
+
run_id: str,
|
577
|
+
context: DictData,
|
497
578
|
*,
|
498
|
-
|
579
|
+
parent_run_id: Optional[str] = None,
|
499
580
|
event: Optional[Event] = None,
|
500
581
|
) -> Result:
|
501
582
|
"""Async execution method for this Empty stage that only logging out to
|
502
583
|
stdout.
|
503
584
|
|
504
|
-
:
|
505
|
-
|
506
|
-
|
507
|
-
|
508
|
-
|
509
|
-
|
510
|
-
|
511
|
-
|
585
|
+
Args:
|
586
|
+
params: A parameter data that want to use in this
|
587
|
+
execution.
|
588
|
+
run_id: A running stage ID.
|
589
|
+
context: A context data.
|
590
|
+
parent_run_id: A parent running ID. (Default is None)
|
591
|
+
event: An event manager that use to track parent process
|
592
|
+
was not force stopped.
|
593
|
+
|
594
|
+
Returns:
|
595
|
+
Result: The execution result with status and context data.
|
512
596
|
"""
|
513
597
|
raise NotImplementedError(
|
514
598
|
"Async Stage should implement `axecute` method."
|
515
599
|
)
|
516
600
|
|
517
|
-
async def
|
601
|
+
async def axecute(
|
518
602
|
self,
|
519
603
|
params: DictData,
|
520
604
|
*,
|
521
605
|
run_id: StrOrNone = None,
|
522
|
-
parent_run_id: StrOrNone = None,
|
523
|
-
result: Optional[Result] = None,
|
524
606
|
event: Optional[Event] = None,
|
525
607
|
) -> Result:
|
526
608
|
"""Async Handler stage execution result from the stage `execute` method.
|
527
609
|
|
528
|
-
:
|
529
|
-
|
530
|
-
|
531
|
-
|
532
|
-
|
533
|
-
|
610
|
+
Args:
|
611
|
+
params: A parameter data that want to use in this
|
612
|
+
execution.
|
613
|
+
run_id: A running stage ID. (Default is None)
|
614
|
+
event: An event manager that use to track parent process
|
615
|
+
was not force stopped.
|
534
616
|
|
535
|
-
:
|
617
|
+
Returns:
|
618
|
+
Result: The execution result with status and context data.
|
536
619
|
"""
|
537
|
-
|
538
|
-
|
539
|
-
|
540
|
-
|
541
|
-
|
542
|
-
extras=self.extras
|
620
|
+
ts: float = time.monotonic()
|
621
|
+
parent_run_id: StrOrNone = run_id
|
622
|
+
run_id: str = run_id or gen_id(self.iden, unique=True)
|
623
|
+
context: DictData = {}
|
624
|
+
trace: Trace = get_trace(
|
625
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
543
626
|
)
|
544
627
|
try:
|
545
|
-
|
628
|
+
_id: str = (
|
629
|
+
f" with ID: {param2template(self.id, params=params)!r}"
|
630
|
+
if self.id
|
631
|
+
else ""
|
632
|
+
)
|
633
|
+
await trace.ainfo(
|
546
634
|
f"[STAGE]: Handler {to_train(self.__class__.__name__)}: "
|
547
|
-
f"{self.name!r}."
|
635
|
+
f"{self.name!r}{_id}."
|
548
636
|
)
|
549
637
|
|
550
638
|
# NOTE: Show the description of this stage before execution.
|
551
639
|
if self.desc:
|
552
|
-
await
|
553
|
-
f"[STAGE]: Description:||{self.desc}||"
|
554
|
-
)
|
640
|
+
await trace.adebug(f"[STAGE]: Description:||{self.desc}||")
|
555
641
|
|
556
642
|
# VALIDATE: Checking stage condition before execution.
|
557
643
|
if self.is_skipped(params=params):
|
@@ -562,7 +648,11 @@ class BaseAsyncStage(BaseStage, ABC):
|
|
562
648
|
# NOTE: Start call wrapped execution method that will use custom
|
563
649
|
# execution before the real execution from inherit stage model.
|
564
650
|
result_caught: Result = await self._axecute(
|
565
|
-
params,
|
651
|
+
params,
|
652
|
+
run_id=run_id,
|
653
|
+
context=context,
|
654
|
+
parent_run_id=parent_run_id,
|
655
|
+
event=event,
|
566
656
|
)
|
567
657
|
if result_caught.status == WAIT:
|
568
658
|
raise StageError(
|
@@ -577,41 +667,68 @@ class BaseAsyncStage(BaseStage, ABC):
|
|
577
667
|
StageCancelError,
|
578
668
|
StageError,
|
579
669
|
) as e: # pragma: no cov
|
580
|
-
await
|
670
|
+
await trace.ainfo(
|
581
671
|
f"[STAGE]: Skip Handler:||{e.__class__.__name__}: {e}||"
|
582
672
|
f"{traceback.format_exc()}"
|
583
673
|
)
|
584
|
-
|
585
|
-
|
586
|
-
|
587
|
-
|
588
|
-
|
589
|
-
|
674
|
+
st: Status = get_status_from_error(e)
|
675
|
+
return Result(
|
676
|
+
run_id=run_id,
|
677
|
+
parent_run_id=parent_run_id,
|
678
|
+
status=st,
|
679
|
+
context=catch(
|
680
|
+
context,
|
681
|
+
status=st,
|
682
|
+
updated=(
|
683
|
+
None
|
684
|
+
if isinstance(e, StageSkipError)
|
685
|
+
else {"status": st, "errors": e.to_dict()}
|
686
|
+
),
|
590
687
|
),
|
688
|
+
info={"execution_time": time.monotonic() - ts},
|
689
|
+
extras=self.extras,
|
591
690
|
)
|
592
691
|
except Exception as e:
|
593
|
-
await
|
692
|
+
await trace.aerror(
|
594
693
|
f"[STAGE]: Error Handler:||{e.__class__.__name__}: {e}||"
|
595
694
|
f"{traceback.format_exc()}"
|
596
695
|
)
|
597
|
-
return
|
696
|
+
return Result(
|
697
|
+
run_id=run_id,
|
698
|
+
parent_run_id=parent_run_id,
|
699
|
+
status=FAILED,
|
700
|
+
context=catch(
|
701
|
+
context, status=FAILED, updated={"errors": to_dict(e)}
|
702
|
+
),
|
703
|
+
info={"execution_time": time.monotonic() - ts},
|
704
|
+
extras=self.extras,
|
705
|
+
)
|
598
706
|
|
599
707
|
async def _axecute(
|
600
|
-
self,
|
708
|
+
self,
|
709
|
+
params: DictData,
|
710
|
+
run_id: str,
|
711
|
+
context: DictData,
|
712
|
+
parent_run_id: Optional[str] = None,
|
713
|
+
event: Optional[Event] = None,
|
601
714
|
) -> Result:
|
602
715
|
"""Wrapped the axecute method before returning to handler axecute.
|
603
716
|
|
604
717
|
:param params: (DictData) A parameter data that want to use in this
|
605
718
|
execution.
|
606
|
-
:param result: (Result) A result object for keeping context and status
|
607
|
-
data.
|
608
719
|
:param event: (Event) An event manager that use to track parent execute
|
609
720
|
was not force stopped.
|
610
721
|
|
611
722
|
:rtype: Result
|
612
723
|
"""
|
613
|
-
|
614
|
-
return await self.
|
724
|
+
catch(context, status=WAIT)
|
725
|
+
return await self.async_process(
|
726
|
+
params,
|
727
|
+
run_id=run_id,
|
728
|
+
context=context,
|
729
|
+
parent_run_id=parent_run_id,
|
730
|
+
event=event,
|
731
|
+
)
|
615
732
|
|
616
733
|
|
617
734
|
class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
@@ -629,16 +746,16 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
629
746
|
def _execute(
|
630
747
|
self,
|
631
748
|
params: DictData,
|
632
|
-
|
633
|
-
|
749
|
+
run_id: str,
|
750
|
+
context: DictData,
|
751
|
+
parent_run_id: Optional[str] = None,
|
752
|
+
event: Optional[Event] = None,
|
634
753
|
) -> Result:
|
635
754
|
"""Wrapped the execute method with retry strategy before returning to
|
636
755
|
handler execute.
|
637
756
|
|
638
757
|
:param params: (DictData) A parameter data that want to use in this
|
639
758
|
execution.
|
640
|
-
:param result: (Result) A result object for keeping context and status
|
641
|
-
data.
|
642
759
|
:param event: (Event) An event manager that use to track parent execute
|
643
760
|
was not force stopped.
|
644
761
|
|
@@ -646,13 +763,18 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
646
763
|
"""
|
647
764
|
current_retry: int = 0
|
648
765
|
exception: Exception
|
766
|
+
catch(context, status=WAIT)
|
767
|
+
trace: Trace = get_trace(
|
768
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
769
|
+
)
|
649
770
|
|
650
771
|
# NOTE: First execution for not pass to retry step if it passes.
|
651
772
|
try:
|
652
|
-
|
653
|
-
return self.execute(
|
773
|
+
return self.process(
|
654
774
|
params | {"retry": current_retry},
|
655
|
-
|
775
|
+
run_id=run_id,
|
776
|
+
context=context,
|
777
|
+
parent_run_id=parent_run_id,
|
656
778
|
event=event,
|
657
779
|
)
|
658
780
|
except Exception as e:
|
@@ -662,28 +784,34 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
662
784
|
if self.retry == 0:
|
663
785
|
raise exception
|
664
786
|
|
665
|
-
|
787
|
+
trace.warning(
|
666
788
|
f"[STAGE]: Retry count: {current_retry} ... "
|
667
789
|
f"( {exception.__class__.__name__} )"
|
668
790
|
)
|
669
791
|
|
670
792
|
while current_retry < (self.retry + 1):
|
671
793
|
try:
|
672
|
-
|
673
|
-
|
794
|
+
catch(
|
795
|
+
context=context,
|
796
|
+
status=WAIT,
|
797
|
+
updated={"retry": current_retry},
|
798
|
+
)
|
799
|
+
return self.process(
|
674
800
|
params | {"retry": current_retry},
|
675
|
-
|
801
|
+
run_id=run_id,
|
802
|
+
context=context,
|
803
|
+
parent_run_id=parent_run_id,
|
676
804
|
event=event,
|
677
805
|
)
|
678
806
|
except Exception as e:
|
679
807
|
current_retry += 1
|
680
|
-
|
808
|
+
trace.warning(
|
681
809
|
f"[STAGE]: Retry count: {current_retry} ... "
|
682
810
|
f"( {e.__class__.__name__} )"
|
683
811
|
)
|
684
812
|
exception = e
|
685
813
|
|
686
|
-
|
814
|
+
trace.error(
|
687
815
|
f"[STAGE]: Reach the maximum of retry number: {self.retry}."
|
688
816
|
)
|
689
817
|
raise exception
|
@@ -691,16 +819,16 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
691
819
|
async def _axecute(
|
692
820
|
self,
|
693
821
|
params: DictData,
|
694
|
-
|
695
|
-
|
822
|
+
run_id: str,
|
823
|
+
context: DictData,
|
824
|
+
parent_run_id: Optional[str] = None,
|
825
|
+
event: Optional[Event] = None,
|
696
826
|
) -> Result:
|
697
827
|
"""Wrapped the axecute method with retry strategy before returning to
|
698
828
|
handler axecute.
|
699
829
|
|
700
830
|
:param params: (DictData) A parameter data that want to use in this
|
701
831
|
execution.
|
702
|
-
:param result: (Result) A result object for keeping context and status
|
703
|
-
data.
|
704
832
|
:param event: (Event) An event manager that use to track parent execute
|
705
833
|
was not force stopped.
|
706
834
|
|
@@ -708,13 +836,18 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
708
836
|
"""
|
709
837
|
current_retry: int = 0
|
710
838
|
exception: Exception
|
839
|
+
catch(context, status=WAIT)
|
840
|
+
trace: Trace = get_trace(
|
841
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
842
|
+
)
|
711
843
|
|
712
844
|
# NOTE: First execution for not pass to retry step if it passes.
|
713
845
|
try:
|
714
|
-
|
715
|
-
return await self.axecute(
|
846
|
+
return await self.async_process(
|
716
847
|
params | {"retry": current_retry},
|
717
|
-
|
848
|
+
run_id=run_id,
|
849
|
+
context=context,
|
850
|
+
parent_run_id=parent_run_id,
|
718
851
|
event=event,
|
719
852
|
)
|
720
853
|
except Exception as e:
|
@@ -724,47 +857,75 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
724
857
|
if self.retry == 0:
|
725
858
|
raise exception
|
726
859
|
|
727
|
-
await
|
860
|
+
await trace.awarning(
|
728
861
|
f"[STAGE]: Retry count: {current_retry} ... "
|
729
862
|
f"( {exception.__class__.__name__} )"
|
730
863
|
)
|
731
864
|
|
732
865
|
while current_retry < (self.retry + 1):
|
733
866
|
try:
|
734
|
-
|
735
|
-
|
867
|
+
catch(
|
868
|
+
context=context,
|
869
|
+
status=WAIT,
|
870
|
+
updated={"retry": current_retry},
|
871
|
+
)
|
872
|
+
return await self.async_process(
|
736
873
|
params | {"retry": current_retry},
|
737
|
-
|
874
|
+
run_id=run_id,
|
875
|
+
context=context,
|
876
|
+
parent_run_id=parent_run_id,
|
738
877
|
event=event,
|
739
878
|
)
|
740
879
|
except Exception as e:
|
741
880
|
current_retry += 1
|
742
|
-
await
|
881
|
+
await trace.awarning(
|
743
882
|
f"[STAGE]: Retry count: {current_retry} ... "
|
744
883
|
f"( {e.__class__.__name__} )"
|
745
884
|
)
|
746
885
|
exception = e
|
747
886
|
|
748
|
-
await
|
887
|
+
await trace.aerror(
|
749
888
|
f"[STAGE]: Reach the maximum of retry number: {self.retry}."
|
750
889
|
)
|
751
890
|
raise exception
|
752
891
|
|
753
892
|
|
754
893
|
class EmptyStage(BaseAsyncStage):
|
755
|
-
"""Empty stage
|
756
|
-
|
757
|
-
|
758
|
-
|
759
|
-
|
760
|
-
|
761
|
-
|
762
|
-
|
763
|
-
|
764
|
-
|
765
|
-
|
766
|
-
|
767
|
-
|
894
|
+
"""Empty stage for logging and debugging workflows.
|
895
|
+
|
896
|
+
EmptyStage is a utility stage that performs no actual work but provides
|
897
|
+
logging output and optional delays. It's commonly used for:
|
898
|
+
- Debugging workflow execution flow
|
899
|
+
- Adding informational messages to workflows
|
900
|
+
- Creating delays between stages
|
901
|
+
- Testing template parameter resolution
|
902
|
+
|
903
|
+
The stage outputs the echo message to stdout and can optionally sleep
|
904
|
+
for a specified duration, making it useful for workflow timing control
|
905
|
+
and debugging scenarios.
|
906
|
+
|
907
|
+
Attributes:
|
908
|
+
echo (str, optional): Message to display during execution
|
909
|
+
sleep (float): Duration to sleep after logging (0-1800 seconds)
|
910
|
+
|
911
|
+
Example:
|
912
|
+
```yaml
|
913
|
+
stages:
|
914
|
+
- name: "Workflow Started"
|
915
|
+
echo: "Beginning data processing workflow"
|
916
|
+
sleep: 2
|
917
|
+
|
918
|
+
- name: "Debug Parameters"
|
919
|
+
echo: "Processing file: ${{ params.filename }}"
|
920
|
+
```
|
921
|
+
|
922
|
+
```python
|
923
|
+
stage = EmptyStage(
|
924
|
+
name="Status Update",
|
925
|
+
echo="Processing completed successfully",
|
926
|
+
sleep=1.0
|
927
|
+
)
|
928
|
+
```
|
768
929
|
"""
|
769
930
|
|
770
931
|
echo: StrOrNone = Field(
|
@@ -781,11 +942,13 @@ class EmptyStage(BaseAsyncStage):
|
|
781
942
|
lt=1800,
|
782
943
|
)
|
783
944
|
|
784
|
-
def
|
945
|
+
def process(
|
785
946
|
self,
|
786
947
|
params: DictData,
|
948
|
+
run_id: str,
|
949
|
+
context: DictData,
|
787
950
|
*,
|
788
|
-
|
951
|
+
parent_run_id: Optional[str] = None,
|
789
952
|
event: Optional[Event] = None,
|
790
953
|
) -> Result:
|
791
954
|
"""Execution method for the Empty stage that do only logging out to
|
@@ -794,16 +957,20 @@ class EmptyStage(BaseAsyncStage):
|
|
794
957
|
The result context should be empty and do not process anything
|
795
958
|
without calling logging function.
|
796
959
|
|
797
|
-
:
|
798
|
-
|
799
|
-
|
800
|
-
|
801
|
-
|
802
|
-
|
960
|
+
Args:
|
961
|
+
params: A parameter data that want to use in this
|
962
|
+
execution.
|
963
|
+
run_id: A running stage ID.
|
964
|
+
context: A context data.
|
965
|
+
parent_run_id: A parent running ID. (Default is None)
|
966
|
+
event: An event manager that use to track parent process
|
967
|
+
was not force stopped.
|
968
|
+
|
969
|
+
Returns:
|
970
|
+
Result: The execution result with status and context data.
|
803
971
|
"""
|
804
|
-
|
805
|
-
run_id=
|
806
|
-
extras=self.extras,
|
972
|
+
trace: Trace = get_trace(
|
973
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
807
974
|
)
|
808
975
|
message: str = (
|
809
976
|
param2template(
|
@@ -818,35 +985,46 @@ class EmptyStage(BaseAsyncStage):
|
|
818
985
|
"Execution was canceled from the event before start parallel."
|
819
986
|
)
|
820
987
|
|
821
|
-
|
988
|
+
trace.info(f"[STAGE]: Message: ( {message} )")
|
822
989
|
if self.sleep > 0:
|
823
990
|
if self.sleep > 5:
|
824
|
-
|
991
|
+
trace.info(f"[STAGE]: Sleep ... ({self.sleep} sec)")
|
825
992
|
time.sleep(self.sleep)
|
826
|
-
return
|
993
|
+
return Result(
|
994
|
+
run_id=run_id,
|
995
|
+
parent_run_id=parent_run_id,
|
996
|
+
status=SUCCESS,
|
997
|
+
context=catch(context=context, status=SUCCESS),
|
998
|
+
extras=self.extras,
|
999
|
+
)
|
827
1000
|
|
828
|
-
async def
|
1001
|
+
async def async_process(
|
829
1002
|
self,
|
830
1003
|
params: DictData,
|
1004
|
+
run_id: str,
|
1005
|
+
context: DictData,
|
831
1006
|
*,
|
832
|
-
|
1007
|
+
parent_run_id: Optional[str] = None,
|
833
1008
|
event: Optional[Event] = None,
|
834
1009
|
) -> Result:
|
835
1010
|
"""Async execution method for this Empty stage that only logging out to
|
836
1011
|
stdout.
|
837
1012
|
|
838
|
-
:
|
839
|
-
|
840
|
-
|
841
|
-
|
842
|
-
|
843
|
-
|
1013
|
+
Args:
|
1014
|
+
params: A parameter data that want to use in this
|
1015
|
+
execution.
|
1016
|
+
run_id: A running stage ID.
|
1017
|
+
context: A context data.
|
1018
|
+
parent_run_id: A parent running ID. (Default is None)
|
1019
|
+
event: An event manager that use to track parent process
|
1020
|
+
was not force stopped.
|
1021
|
+
|
1022
|
+
Returns:
|
1023
|
+
Result: The execution result with status and context data.
|
844
1024
|
"""
|
845
|
-
|
846
|
-
run_id=
|
847
|
-
extras=self.extras,
|
1025
|
+
trace: Trace = get_trace(
|
1026
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
848
1027
|
)
|
849
|
-
|
850
1028
|
message: str = (
|
851
1029
|
param2template(
|
852
1030
|
dedent(self.echo.strip("\n")), params, extras=self.extras
|
@@ -860,14 +1038,18 @@ class EmptyStage(BaseAsyncStage):
|
|
860
1038
|
"Execution was canceled from the event before start parallel."
|
861
1039
|
)
|
862
1040
|
|
863
|
-
|
1041
|
+
trace.info(f"[STAGE]: Message: ( {message} )")
|
864
1042
|
if self.sleep > 0:
|
865
1043
|
if self.sleep > 5:
|
866
|
-
await
|
867
|
-
f"[STAGE]: Sleep ... ({self.sleep} sec)"
|
868
|
-
)
|
1044
|
+
await trace.ainfo(f"[STAGE]: Sleep ... ({self.sleep} sec)")
|
869
1045
|
await asyncio.sleep(self.sleep)
|
870
|
-
return
|
1046
|
+
return Result(
|
1047
|
+
run_id=run_id,
|
1048
|
+
parent_run_id=parent_run_id,
|
1049
|
+
status=SUCCESS,
|
1050
|
+
context=catch(context=context, status=SUCCESS),
|
1051
|
+
extras=self.extras,
|
1052
|
+
)
|
871
1053
|
|
872
1054
|
|
873
1055
|
class BashStage(BaseRetryStage):
|
@@ -975,27 +1157,38 @@ class BashStage(BaseRetryStage):
|
|
975
1157
|
# Note: Remove .sh file that use to run bash.
|
976
1158
|
Path(f"./{f_name}").unlink()
|
977
1159
|
|
978
|
-
|
1160
|
+
@staticmethod
|
1161
|
+
def prepare_std(value: str) -> Optional[str]:
|
1162
|
+
"""Prepare returned standard string from subprocess."""
|
1163
|
+
return None if (out := value.strip("\n")) == "" else out
|
1164
|
+
|
1165
|
+
def process(
|
979
1166
|
self,
|
980
1167
|
params: DictData,
|
1168
|
+
run_id: str,
|
1169
|
+
context: DictData,
|
981
1170
|
*,
|
982
|
-
|
1171
|
+
parent_run_id: Optional[str] = None,
|
983
1172
|
event: Optional[Event] = None,
|
984
1173
|
) -> Result:
|
985
1174
|
"""Execute bash statement with the Python build-in `subprocess` package.
|
986
1175
|
It will catch result from the `subprocess.run` returning output like
|
987
1176
|
`return_code`, `stdout`, and `stderr`.
|
988
1177
|
|
989
|
-
:
|
990
|
-
|
991
|
-
|
992
|
-
|
993
|
-
|
994
|
-
|
1178
|
+
Args:
|
1179
|
+
params: A parameter data that want to use in this
|
1180
|
+
execution.
|
1181
|
+
run_id: A running stage ID.
|
1182
|
+
context: A context data.
|
1183
|
+
parent_run_id: A parent running ID. (Default is None)
|
1184
|
+
event: An event manager that use to track parent process
|
1185
|
+
was not force stopped.
|
1186
|
+
|
1187
|
+
Returns:
|
1188
|
+
Result: The execution result with status and context data.
|
995
1189
|
"""
|
996
|
-
|
997
|
-
run_id=
|
998
|
-
extras=self.extras,
|
1190
|
+
trace: Trace = get_trace(
|
1191
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
999
1192
|
)
|
1000
1193
|
bash: str = param2template(
|
1001
1194
|
dedent(self.bash.strip("\n")), params, extras=self.extras
|
@@ -1003,9 +1196,9 @@ class BashStage(BaseRetryStage):
|
|
1003
1196
|
with self.create_sh_file(
|
1004
1197
|
bash=bash,
|
1005
1198
|
env=param2template(self.env, params, extras=self.extras),
|
1006
|
-
run_id=
|
1199
|
+
run_id=run_id,
|
1007
1200
|
) as sh:
|
1008
|
-
|
1201
|
+
trace.debug(f"[STAGE]: Create `{sh[1]}` file.")
|
1009
1202
|
rs: CompletedProcess = subprocess.run(
|
1010
1203
|
sh,
|
1011
1204
|
shell=False,
|
@@ -1018,35 +1211,48 @@ class BashStage(BaseRetryStage):
|
|
1018
1211
|
e: str = rs.stderr.removesuffix("\n")
|
1019
1212
|
e_bash: str = bash.replace("\n", "\n\t")
|
1020
1213
|
raise StageError(f"Subprocess: {e}\n\t```bash\n\t{e_bash}\n\t```")
|
1021
|
-
return
|
1214
|
+
return Result(
|
1215
|
+
run_id=run_id,
|
1216
|
+
parent_run_id=parent_run_id,
|
1022
1217
|
status=SUCCESS,
|
1023
|
-
context=
|
1024
|
-
|
1025
|
-
|
1026
|
-
|
1027
|
-
|
1218
|
+
context=catch(
|
1219
|
+
context=context,
|
1220
|
+
status=SUCCESS,
|
1221
|
+
updated={
|
1222
|
+
"return_code": rs.returncode,
|
1223
|
+
"stdout": self.prepare_std(rs.stdout),
|
1224
|
+
"stderr": self.prepare_std(rs.stderr),
|
1225
|
+
},
|
1226
|
+
),
|
1227
|
+
extras=self.extras,
|
1028
1228
|
)
|
1029
1229
|
|
1030
|
-
async def
|
1230
|
+
async def async_process(
|
1031
1231
|
self,
|
1032
1232
|
params: DictData,
|
1233
|
+
run_id: str,
|
1234
|
+
context: DictData,
|
1033
1235
|
*,
|
1034
|
-
|
1236
|
+
parent_run_id: Optional[str] = None,
|
1035
1237
|
event: Optional[Event] = None,
|
1036
1238
|
) -> Result:
|
1037
1239
|
"""Async execution method for this Bash stage that only logging out to
|
1038
1240
|
stdout.
|
1039
1241
|
|
1040
|
-
:
|
1041
|
-
|
1042
|
-
|
1043
|
-
|
1044
|
-
|
1045
|
-
|
1242
|
+
Args:
|
1243
|
+
params: A parameter data that want to use in this
|
1244
|
+
execution.
|
1245
|
+
run_id: A running stage ID.
|
1246
|
+
context: A context data.
|
1247
|
+
parent_run_id: A parent running ID. (Default is None)
|
1248
|
+
event: An event manager that use to track parent process
|
1249
|
+
was not force stopped.
|
1250
|
+
|
1251
|
+
Returns:
|
1252
|
+
Result: The execution result with status and context data.
|
1046
1253
|
"""
|
1047
|
-
|
1048
|
-
run_id=
|
1049
|
-
extras=self.extras,
|
1254
|
+
trace: Trace = get_trace(
|
1255
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1050
1256
|
)
|
1051
1257
|
bash: str = param2template(
|
1052
1258
|
dedent(self.bash.strip("\n")), params, extras=self.extras
|
@@ -1054,9 +1260,9 @@ class BashStage(BaseRetryStage):
|
|
1054
1260
|
async with self.async_create_sh_file(
|
1055
1261
|
bash=bash,
|
1056
1262
|
env=param2template(self.env, params, extras=self.extras),
|
1057
|
-
run_id=
|
1263
|
+
run_id=run_id,
|
1058
1264
|
) as sh:
|
1059
|
-
await
|
1265
|
+
await trace.adebug(f"[STAGE]: Create `{sh[1]}` file.")
|
1060
1266
|
rs: CompletedProcess = subprocess.run(
|
1061
1267
|
sh,
|
1062
1268
|
shell=False,
|
@@ -1070,13 +1276,20 @@ class BashStage(BaseRetryStage):
|
|
1070
1276
|
e: str = rs.stderr.removesuffix("\n")
|
1071
1277
|
e_bash: str = bash.replace("\n", "\n\t")
|
1072
1278
|
raise StageError(f"Subprocess: {e}\n\t```bash\n\t{e_bash}\n\t```")
|
1073
|
-
return
|
1279
|
+
return Result(
|
1280
|
+
run_id=run_id,
|
1281
|
+
parent_run_id=parent_run_id,
|
1074
1282
|
status=SUCCESS,
|
1075
|
-
context=
|
1076
|
-
|
1077
|
-
|
1078
|
-
|
1079
|
-
|
1283
|
+
context=catch(
|
1284
|
+
context=context,
|
1285
|
+
status=SUCCESS,
|
1286
|
+
updated={
|
1287
|
+
"return_code": rs.returncode,
|
1288
|
+
"stdout": self.prepare_std(rs.stdout),
|
1289
|
+
"stderr": self.prepare_std(rs.stderr),
|
1290
|
+
},
|
1291
|
+
),
|
1292
|
+
extras=self.extras,
|
1080
1293
|
)
|
1081
1294
|
|
1082
1295
|
|
@@ -1135,13 +1348,16 @@ class PyStage(BaseRetryStage):
|
|
1135
1348
|
|
1136
1349
|
yield value
|
1137
1350
|
|
1138
|
-
def set_outputs(
|
1351
|
+
def set_outputs(
|
1352
|
+
self, output: DictData, to: DictData, info: Optional[DictData] = None
|
1353
|
+
) -> DictData:
|
1139
1354
|
"""Override set an outputs method for the Python execution process that
|
1140
1355
|
extract output from all the locals values.
|
1141
1356
|
|
1142
1357
|
:param output: (DictData) An output data that want to extract to an
|
1143
1358
|
output key.
|
1144
1359
|
:param to: (DictData) A context data that want to add output result.
|
1360
|
+
:param info: (DictData)
|
1145
1361
|
|
1146
1362
|
:rtype: DictData
|
1147
1363
|
"""
|
@@ -1152,33 +1368,47 @@ class PyStage(BaseRetryStage):
|
|
1152
1368
|
to.update({k: gb[k] for k in to if k in gb})
|
1153
1369
|
return to
|
1154
1370
|
|
1155
|
-
def
|
1371
|
+
def process(
|
1156
1372
|
self,
|
1157
1373
|
params: DictData,
|
1374
|
+
run_id: str,
|
1375
|
+
context: DictData,
|
1158
1376
|
*,
|
1159
|
-
|
1377
|
+
parent_run_id: Optional[str] = None,
|
1160
1378
|
event: Optional[Event] = None,
|
1161
1379
|
) -> Result:
|
1162
1380
|
"""Execute the Python statement that pass all globals and input params
|
1163
1381
|
to globals argument on `exec` build-in function.
|
1164
1382
|
|
1165
|
-
:
|
1166
|
-
|
1167
|
-
|
1168
|
-
|
1169
|
-
|
1170
|
-
|
1171
|
-
|
1383
|
+
Args:
|
1384
|
+
params: A parameter data that want to use in this
|
1385
|
+
execution.
|
1386
|
+
run_id: A running stage ID.
|
1387
|
+
context: A context data.
|
1388
|
+
parent_run_id: A parent running ID. (Default is None)
|
1389
|
+
event: An event manager that use to track parent process
|
1390
|
+
was not force stopped.
|
1391
|
+
|
1392
|
+
Returns:
|
1393
|
+
Result: The execution result with status and context data.
|
1172
1394
|
"""
|
1173
|
-
|
1174
|
-
run_id=
|
1175
|
-
extras=self.extras,
|
1395
|
+
trace: Trace = get_trace(
|
1396
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1176
1397
|
)
|
1398
|
+
trace.info("[STAGE]: Prepare `globals` and `locals` variables.")
|
1177
1399
|
lc: DictData = {}
|
1178
1400
|
gb: DictData = (
|
1179
1401
|
globals()
|
1180
1402
|
| param2template(self.vars, params, extras=self.extras)
|
1181
|
-
| {
|
1403
|
+
| {
|
1404
|
+
"result": Result(
|
1405
|
+
run_id=run_id,
|
1406
|
+
parent_run_id=parent_run_id,
|
1407
|
+
status=WAIT,
|
1408
|
+
context=context,
|
1409
|
+
extras=self.extras,
|
1410
|
+
)
|
1411
|
+
}
|
1182
1412
|
)
|
1183
1413
|
|
1184
1414
|
# WARNING: The exec build-in function is very dangerous. So, it
|
@@ -1190,55 +1420,76 @@ class PyStage(BaseRetryStage):
|
|
1190
1420
|
gb,
|
1191
1421
|
lc,
|
1192
1422
|
)
|
1193
|
-
|
1194
|
-
|
1423
|
+
return Result(
|
1424
|
+
run_id=run_id,
|
1425
|
+
parent_run_id=parent_run_id,
|
1195
1426
|
status=SUCCESS,
|
1196
|
-
context=
|
1197
|
-
|
1198
|
-
|
1199
|
-
|
1200
|
-
for k in
|
1201
|
-
|
1202
|
-
|
1203
|
-
|
1204
|
-
|
1205
|
-
|
1206
|
-
|
1207
|
-
|
1208
|
-
|
1427
|
+
context=catch(
|
1428
|
+
context=context,
|
1429
|
+
status=SUCCESS,
|
1430
|
+
updated={
|
1431
|
+
"locals": {k: lc[k] for k in self.filter_locals(lc)},
|
1432
|
+
"globals": {
|
1433
|
+
k: gb[k]
|
1434
|
+
for k in gb
|
1435
|
+
if (
|
1436
|
+
not k.startswith("__")
|
1437
|
+
and k != "annotations"
|
1438
|
+
and not ismodule(gb[k])
|
1439
|
+
and not isclass(gb[k])
|
1440
|
+
and not isfunction(gb[k])
|
1441
|
+
and k in params
|
1442
|
+
)
|
1443
|
+
},
|
1209
1444
|
},
|
1210
|
-
|
1445
|
+
),
|
1446
|
+
extras=self.extras,
|
1211
1447
|
)
|
1212
1448
|
|
1213
|
-
async def
|
1449
|
+
async def async_process(
|
1214
1450
|
self,
|
1215
1451
|
params: DictData,
|
1452
|
+
run_id: str,
|
1453
|
+
context: DictData,
|
1216
1454
|
*,
|
1217
|
-
|
1455
|
+
parent_run_id: Optional[str] = None,
|
1218
1456
|
event: Optional[Event] = None,
|
1219
1457
|
) -> Result:
|
1220
1458
|
"""Async execution method for this Bash stage that only logging out to
|
1221
1459
|
stdout.
|
1222
1460
|
|
1223
|
-
:param params: (DictData) A parameter data.
|
1224
|
-
:param result: (Result) A Result instance for return context and status.
|
1225
|
-
:param event: (Event) An Event manager instance that use to cancel this
|
1226
|
-
execution if it forces stopped by parent execution.
|
1227
|
-
|
1228
1461
|
References:
|
1229
1462
|
- https://stackoverflow.com/questions/44859165/async-exec-in-python
|
1230
1463
|
|
1231
|
-
:
|
1464
|
+
Args:
|
1465
|
+
params: A parameter data that want to use in this
|
1466
|
+
execution.
|
1467
|
+
run_id: A running stage ID.
|
1468
|
+
context: A context data.
|
1469
|
+
parent_run_id: A parent running ID. (Default is None)
|
1470
|
+
event: An event manager that use to track parent process
|
1471
|
+
was not force stopped.
|
1472
|
+
|
1473
|
+
Returns:
|
1474
|
+
Result: The execution result with status and context data.
|
1232
1475
|
"""
|
1233
|
-
|
1234
|
-
run_id=
|
1235
|
-
extras=self.extras,
|
1476
|
+
trace: Trace = get_trace(
|
1477
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1236
1478
|
)
|
1479
|
+
await trace.ainfo("[STAGE]: Prepare `globals` and `locals` variables.")
|
1237
1480
|
lc: DictData = {}
|
1238
1481
|
gb: DictData = (
|
1239
1482
|
globals()
|
1240
1483
|
| param2template(self.vars, params, extras=self.extras)
|
1241
|
-
| {
|
1484
|
+
| {
|
1485
|
+
"result": Result(
|
1486
|
+
run_id=run_id,
|
1487
|
+
parent_run_id=parent_run_id,
|
1488
|
+
status=WAIT,
|
1489
|
+
context=context,
|
1490
|
+
extras=self.extras,
|
1491
|
+
)
|
1492
|
+
}
|
1242
1493
|
)
|
1243
1494
|
# WARNING: The exec build-in function is very dangerous. So, it
|
1244
1495
|
# should use the re module to validate exec-string before running.
|
@@ -1247,23 +1498,30 @@ class PyStage(BaseRetryStage):
|
|
1247
1498
|
gb,
|
1248
1499
|
lc,
|
1249
1500
|
)
|
1250
|
-
return
|
1501
|
+
return Result(
|
1502
|
+
run_id=run_id,
|
1503
|
+
parent_run_id=parent_run_id,
|
1251
1504
|
status=SUCCESS,
|
1252
|
-
context=
|
1253
|
-
|
1254
|
-
|
1255
|
-
|
1256
|
-
for k in
|
1257
|
-
|
1258
|
-
|
1259
|
-
|
1260
|
-
|
1261
|
-
|
1262
|
-
|
1263
|
-
|
1264
|
-
|
1505
|
+
context=catch(
|
1506
|
+
context=context,
|
1507
|
+
status=SUCCESS,
|
1508
|
+
updated={
|
1509
|
+
"locals": {k: lc[k] for k in self.filter_locals(lc)},
|
1510
|
+
"globals": {
|
1511
|
+
k: gb[k]
|
1512
|
+
for k in gb
|
1513
|
+
if (
|
1514
|
+
not k.startswith("__")
|
1515
|
+
and k != "annotations"
|
1516
|
+
and not ismodule(gb[k])
|
1517
|
+
and not isclass(gb[k])
|
1518
|
+
and not isfunction(gb[k])
|
1519
|
+
and k in params
|
1520
|
+
)
|
1521
|
+
},
|
1265
1522
|
},
|
1266
|
-
|
1523
|
+
),
|
1524
|
+
extras=self.extras,
|
1267
1525
|
)
|
1268
1526
|
|
1269
1527
|
|
@@ -1313,6 +1571,13 @@ class CallStage(BaseRetryStage):
|
|
1313
1571
|
|
1314
1572
|
@field_validator("args", mode="before")
|
1315
1573
|
def __validate_args_key(cls, value: Any) -> Any:
|
1574
|
+
"""Validate argument keys on the ``args`` field should not include the
|
1575
|
+
special keys.
|
1576
|
+
|
1577
|
+
:param value: (Any) A value that want to check the special keys.
|
1578
|
+
|
1579
|
+
:rtype: Any
|
1580
|
+
"""
|
1316
1581
|
if isinstance(value, dict):
|
1317
1582
|
if any(k in value for k in ("result", "extras")):
|
1318
1583
|
raise ValueError(
|
@@ -1322,45 +1587,49 @@ class CallStage(BaseRetryStage):
|
|
1322
1587
|
)
|
1323
1588
|
return value
|
1324
1589
|
|
1325
|
-
def
|
1590
|
+
def process(
|
1326
1591
|
self,
|
1327
1592
|
params: DictData,
|
1593
|
+
run_id: str,
|
1594
|
+
context: DictData,
|
1328
1595
|
*,
|
1329
|
-
|
1596
|
+
parent_run_id: Optional[str] = None,
|
1330
1597
|
event: Optional[Event] = None,
|
1331
1598
|
) -> Result:
|
1332
1599
|
"""Execute this caller function with its argument parameter.
|
1333
1600
|
|
1334
|
-
:
|
1335
|
-
|
1336
|
-
|
1337
|
-
|
1338
|
-
|
1339
|
-
|
1340
|
-
|
1341
|
-
|
1342
|
-
|
1343
|
-
|
1344
|
-
|
1601
|
+
Args:
|
1602
|
+
params: A parameter data that want to use in this
|
1603
|
+
execution.
|
1604
|
+
run_id: A running stage ID.
|
1605
|
+
context: A context data.
|
1606
|
+
parent_run_id: A parent running ID. (Default is None)
|
1607
|
+
event: An event manager that use to track parent process
|
1608
|
+
was not force stopped.
|
1609
|
+
|
1610
|
+
Returns:
|
1611
|
+
Result: The execution result with status and context data.
|
1345
1612
|
"""
|
1346
|
-
|
1347
|
-
run_id=
|
1348
|
-
extras=self.extras,
|
1613
|
+
trace: Trace = get_trace(
|
1614
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1349
1615
|
)
|
1350
|
-
|
1351
1616
|
call_func: TagFunc = extract_call(
|
1352
1617
|
param2template(self.uses, params, extras=self.extras),
|
1353
1618
|
registries=self.extras.get("registry_caller"),
|
1354
1619
|
)()
|
1355
1620
|
|
1356
|
-
|
1357
|
-
f"[STAGE]: Caller Func: '{call_func.name}@{call_func.tag}'"
|
1358
|
-
)
|
1621
|
+
trace.info(f"[STAGE]: Caller Func: '{call_func.name}@{call_func.tag}'")
|
1359
1622
|
|
1360
1623
|
# VALIDATE: check input task caller parameters that exists before
|
1361
1624
|
# calling.
|
1362
1625
|
args: DictData = {
|
1363
|
-
"result":
|
1626
|
+
"result": Result(
|
1627
|
+
run_id=run_id,
|
1628
|
+
parent_run_id=parent_run_id,
|
1629
|
+
status=WAIT,
|
1630
|
+
context=context,
|
1631
|
+
extras=self.extras,
|
1632
|
+
),
|
1364
1633
|
"extras": self.extras,
|
1365
1634
|
} | param2template(self.args, params, extras=self.extras)
|
1366
1635
|
sig = inspect.signature(call_func)
|
@@ -1381,8 +1650,12 @@ class CallStage(BaseRetryStage):
|
|
1381
1650
|
(k.removeprefix("_") not in args and k not in args)
|
1382
1651
|
for k in necessary_params
|
1383
1652
|
):
|
1384
|
-
|
1385
|
-
|
1653
|
+
if "result" in necessary_params:
|
1654
|
+
necessary_params.remove("result")
|
1655
|
+
|
1656
|
+
if "extras" in necessary_params:
|
1657
|
+
necessary_params.remove("extras")
|
1658
|
+
|
1386
1659
|
args.pop("result")
|
1387
1660
|
args.pop("extras")
|
1388
1661
|
raise ValueError(
|
@@ -1401,7 +1674,9 @@ class CallStage(BaseRetryStage):
|
|
1401
1674
|
"Execution was canceled from the event before start parallel."
|
1402
1675
|
)
|
1403
1676
|
|
1404
|
-
args = self.validate_model_args(
|
1677
|
+
args: DictData = self.validate_model_args(
|
1678
|
+
call_func, args, run_id, parent_run_id
|
1679
|
+
)
|
1405
1680
|
if inspect.iscoroutinefunction(call_func):
|
1406
1681
|
loop = asyncio.get_event_loop()
|
1407
1682
|
rs: DictData = loop.run_until_complete(
|
@@ -1422,46 +1697,64 @@ class CallStage(BaseRetryStage):
|
|
1422
1697
|
f"serialize, you must set return be `dict` or Pydantic "
|
1423
1698
|
f"model."
|
1424
1699
|
)
|
1425
|
-
return
|
1700
|
+
return Result(
|
1701
|
+
run_id=run_id,
|
1702
|
+
parent_run_id=parent_run_id,
|
1703
|
+
status=SUCCESS,
|
1704
|
+
context=catch(
|
1705
|
+
context=context,
|
1706
|
+
status=SUCCESS,
|
1707
|
+
updated=dump_all(rs, by_alias=True),
|
1708
|
+
),
|
1709
|
+
extras=self.extras,
|
1710
|
+
)
|
1426
1711
|
|
1427
|
-
async def
|
1712
|
+
async def async_process(
|
1428
1713
|
self,
|
1429
1714
|
params: DictData,
|
1715
|
+
run_id: str,
|
1716
|
+
context: DictData,
|
1430
1717
|
*,
|
1431
|
-
|
1718
|
+
parent_run_id: Optional[str] = None,
|
1432
1719
|
event: Optional[Event] = None,
|
1433
1720
|
) -> Result:
|
1434
1721
|
"""Async execution method for this Bash stage that only logging out to
|
1435
1722
|
stdout.
|
1436
1723
|
|
1437
|
-
:
|
1438
|
-
|
1439
|
-
|
1440
|
-
|
1441
|
-
|
1442
|
-
|
1443
|
-
|
1444
|
-
|
1445
|
-
|
1724
|
+
Args:
|
1725
|
+
params: A parameter data that want to use in this
|
1726
|
+
execution.
|
1727
|
+
run_id: A running stage ID.
|
1728
|
+
context: A context data.
|
1729
|
+
parent_run_id: A parent running ID. (Default is None)
|
1730
|
+
event: An event manager that use to track parent process
|
1731
|
+
was not force stopped.
|
1732
|
+
|
1733
|
+
Returns:
|
1734
|
+
Result: The execution result with status and context data.
|
1446
1735
|
"""
|
1447
|
-
|
1448
|
-
run_id=
|
1449
|
-
extras=self.extras,
|
1736
|
+
trace: Trace = get_trace(
|
1737
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1450
1738
|
)
|
1451
|
-
|
1452
1739
|
call_func: TagFunc = extract_call(
|
1453
1740
|
param2template(self.uses, params, extras=self.extras),
|
1454
1741
|
registries=self.extras.get("registry_caller"),
|
1455
1742
|
)()
|
1456
1743
|
|
1457
|
-
await
|
1744
|
+
await trace.ainfo(
|
1458
1745
|
f"[STAGE]: Caller Func: '{call_func.name}@{call_func.tag}'"
|
1459
1746
|
)
|
1460
1747
|
|
1461
1748
|
# VALIDATE: check input task caller parameters that exists before
|
1462
1749
|
# calling.
|
1463
1750
|
args: DictData = {
|
1464
|
-
"result":
|
1751
|
+
"result": Result(
|
1752
|
+
run_id=run_id,
|
1753
|
+
parent_run_id=parent_run_id,
|
1754
|
+
status=WAIT,
|
1755
|
+
context=context,
|
1756
|
+
extras=self.extras,
|
1757
|
+
),
|
1465
1758
|
"extras": self.extras,
|
1466
1759
|
} | param2template(self.args, params, extras=self.extras)
|
1467
1760
|
sig = inspect.signature(call_func)
|
@@ -1482,8 +1775,12 @@ class CallStage(BaseRetryStage):
|
|
1482
1775
|
(k.removeprefix("_") not in args and k not in args)
|
1483
1776
|
for k in necessary_params
|
1484
1777
|
):
|
1485
|
-
|
1486
|
-
|
1778
|
+
if "result" in necessary_params:
|
1779
|
+
necessary_params.remove("result")
|
1780
|
+
|
1781
|
+
if "extras" in necessary_params:
|
1782
|
+
necessary_params.remove("extras")
|
1783
|
+
|
1487
1784
|
args.pop("result")
|
1488
1785
|
args.pop("extras")
|
1489
1786
|
raise ValueError(
|
@@ -1496,7 +1793,9 @@ class CallStage(BaseRetryStage):
|
|
1496
1793
|
if "extras" not in sig.parameters and not has_keyword:
|
1497
1794
|
args.pop("extras")
|
1498
1795
|
|
1499
|
-
args: DictData = self.validate_model_args(
|
1796
|
+
args: DictData = self.validate_model_args(
|
1797
|
+
call_func, args, run_id, parent_run_id
|
1798
|
+
)
|
1500
1799
|
if inspect.iscoroutinefunction(call_func):
|
1501
1800
|
rs: DictOrModel = await call_func(
|
1502
1801
|
**param2template(args, params, extras=self.extras)
|
@@ -1516,20 +1815,31 @@ class CallStage(BaseRetryStage):
|
|
1516
1815
|
f"serialize, you must set return be `dict` or Pydantic "
|
1517
1816
|
f"model."
|
1518
1817
|
)
|
1519
|
-
return
|
1818
|
+
return Result(
|
1819
|
+
run_id=run_id,
|
1820
|
+
parent_run_id=parent_run_id,
|
1821
|
+
status=SUCCESS,
|
1822
|
+
context=catch(
|
1823
|
+
context=context,
|
1824
|
+
status=SUCCESS,
|
1825
|
+
updated=dump_all(rs, by_alias=True),
|
1826
|
+
),
|
1827
|
+
extras=self.extras,
|
1828
|
+
)
|
1520
1829
|
|
1521
|
-
@staticmethod
|
1522
1830
|
def validate_model_args(
|
1831
|
+
self,
|
1523
1832
|
func: TagFunc,
|
1524
1833
|
args: DictData,
|
1525
|
-
|
1834
|
+
run_id: str,
|
1835
|
+
parent_run_id: Optional[str] = None,
|
1526
1836
|
) -> DictData:
|
1527
1837
|
"""Validate an input arguments before passing to the caller function.
|
1528
1838
|
|
1529
|
-
:
|
1530
|
-
|
1531
|
-
|
1532
|
-
|
1839
|
+
Args:
|
1840
|
+
func: (TagFunc) A tag function that want to get typing.
|
1841
|
+
args: (DictData) An arguments before passing to this tag func.
|
1842
|
+
run_id: A running stage ID.
|
1533
1843
|
|
1534
1844
|
:rtype: DictData
|
1535
1845
|
"""
|
@@ -1555,7 +1865,10 @@ class CallStage(BaseRetryStage):
|
|
1555
1865
|
"Validate argument from the caller function raise invalid type."
|
1556
1866
|
) from e
|
1557
1867
|
except TypeError as e:
|
1558
|
-
|
1868
|
+
trace: Trace = get_trace(
|
1869
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1870
|
+
)
|
1871
|
+
trace.warning(
|
1559
1872
|
f"[STAGE]: Get type hint raise TypeError: {e}, so, it skip "
|
1560
1873
|
f"parsing model args process."
|
1561
1874
|
)
|
@@ -1567,7 +1880,9 @@ class BaseNestedStage(BaseRetryStage, ABC):
|
|
1567
1880
|
is the nested stage or not.
|
1568
1881
|
"""
|
1569
1882
|
|
1570
|
-
def set_outputs(
|
1883
|
+
def set_outputs(
|
1884
|
+
self, output: DictData, to: DictData, info: Optional[DictData] = None
|
1885
|
+
) -> DictData:
|
1571
1886
|
"""Override the set outputs method that support for nested-stage."""
|
1572
1887
|
return super().set_outputs(output, to=to)
|
1573
1888
|
|
@@ -1596,13 +1911,29 @@ class BaseNestedStage(BaseRetryStage, ABC):
|
|
1596
1911
|
else:
|
1597
1912
|
context["errors"] = error.to_dict(with_refs=True)
|
1598
1913
|
|
1599
|
-
async def
|
1914
|
+
async def async_process(
|
1600
1915
|
self,
|
1601
1916
|
params: DictData,
|
1917
|
+
run_id: str,
|
1918
|
+
context: DictData,
|
1602
1919
|
*,
|
1603
|
-
|
1920
|
+
parent_run_id: Optional[str] = None,
|
1604
1921
|
event: Optional[Event] = None,
|
1605
1922
|
) -> Result:
|
1923
|
+
"""Async process for nested-stage do not implement yet.
|
1924
|
+
|
1925
|
+
Args:
|
1926
|
+
params: A parameter data that want to use in this
|
1927
|
+
execution.
|
1928
|
+
run_id: A running stage ID.
|
1929
|
+
context: A context data.
|
1930
|
+
parent_run_id: A parent running ID. (Default is None)
|
1931
|
+
event: An event manager that use to track parent process
|
1932
|
+
was not force stopped.
|
1933
|
+
|
1934
|
+
Returns:
|
1935
|
+
Result: The execution result with status and context data.
|
1936
|
+
"""
|
1606
1937
|
raise NotImplementedError(
|
1607
1938
|
"The nested-stage does not implement the `axecute` method yet."
|
1608
1939
|
)
|
@@ -1632,41 +1963,45 @@ class TriggerStage(BaseNestedStage):
|
|
1632
1963
|
description="A parameter that will pass to workflow execution method.",
|
1633
1964
|
)
|
1634
1965
|
|
1635
|
-
def
|
1966
|
+
def process(
|
1636
1967
|
self,
|
1637
1968
|
params: DictData,
|
1969
|
+
run_id: str,
|
1970
|
+
context: DictData,
|
1638
1971
|
*,
|
1639
|
-
|
1972
|
+
parent_run_id: Optional[str] = None,
|
1640
1973
|
event: Optional[Event] = None,
|
1641
1974
|
) -> Result:
|
1642
1975
|
"""Trigger another workflow execution. It will wait the trigger
|
1643
1976
|
workflow running complete before catching its result and raise error
|
1644
1977
|
when the result status does not be SUCCESS.
|
1645
1978
|
|
1646
|
-
:
|
1647
|
-
|
1648
|
-
|
1649
|
-
|
1650
|
-
|
1651
|
-
|
1652
|
-
|
1979
|
+
Args:
|
1980
|
+
params: A parameter data that want to use in this
|
1981
|
+
execution.
|
1982
|
+
run_id: A running stage ID.
|
1983
|
+
context: A context data.
|
1984
|
+
parent_run_id: A parent running ID. (Default is None)
|
1985
|
+
event: An event manager that use to track parent process
|
1986
|
+
was not force stopped.
|
1987
|
+
|
1988
|
+
Returns:
|
1989
|
+
Result: The execution result with status and context data.
|
1653
1990
|
"""
|
1654
1991
|
from .workflow import Workflow
|
1655
1992
|
|
1656
|
-
|
1657
|
-
run_id=
|
1658
|
-
extras=self.extras,
|
1993
|
+
trace: Trace = get_trace(
|
1994
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1659
1995
|
)
|
1660
|
-
|
1661
1996
|
_trigger: str = param2template(self.trigger, params, extras=self.extras)
|
1997
|
+
trace.info(f"[STAGE]: Load workflow: {_trigger!r}")
|
1662
1998
|
result: Result = Workflow.from_conf(
|
1663
1999
|
name=pass_env(_trigger),
|
1664
2000
|
extras=self.extras,
|
1665
2001
|
).execute(
|
1666
2002
|
# NOTE: Should not use the `pass_env` function on this params parameter.
|
1667
2003
|
params=param2template(self.params, params, extras=self.extras),
|
1668
|
-
run_id=
|
1669
|
-
parent_run_id=result.parent_run_id,
|
2004
|
+
run_id=parent_run_id,
|
1670
2005
|
event=event,
|
1671
2006
|
)
|
1672
2007
|
if result.status == FAILED:
|
@@ -1731,20 +2066,24 @@ class ParallelStage(BaseNestedStage):
|
|
1731
2066
|
alias="max-workers",
|
1732
2067
|
)
|
1733
2068
|
|
1734
|
-
def
|
2069
|
+
def _process_branch(
|
1735
2070
|
self,
|
1736
2071
|
branch: str,
|
1737
2072
|
params: DictData,
|
1738
|
-
|
2073
|
+
run_id: str,
|
2074
|
+
context: DictData,
|
1739
2075
|
*,
|
2076
|
+
parent_run_id: Optional[str] = None,
|
1740
2077
|
event: Optional[Event] = None,
|
1741
|
-
) -> tuple[Status,
|
2078
|
+
) -> tuple[Status, DictData]:
|
1742
2079
|
"""Execute branch that will execute all nested-stage that was set in
|
1743
2080
|
this stage with specific branch ID.
|
1744
2081
|
|
1745
2082
|
:param branch: (str) A branch ID.
|
1746
2083
|
:param params: (DictData) A parameter data.
|
1747
|
-
:param
|
2084
|
+
:param run_id: (str)
|
2085
|
+
:param context: (DictData)
|
2086
|
+
:param parent_run_id: (str | None)
|
1748
2087
|
:param event: (Event) An Event manager instance that use to cancel this
|
1749
2088
|
execution if it forces stopped by parent execution.
|
1750
2089
|
(Default is None)
|
@@ -1754,13 +2093,16 @@ class ParallelStage(BaseNestedStage):
|
|
1754
2093
|
status.
|
1755
2094
|
:raise StageError: If result from a nested-stage return failed status.
|
1756
2095
|
|
1757
|
-
:rtype: tuple[Status,
|
2096
|
+
:rtype: tuple[Status, DictData]
|
1758
2097
|
"""
|
1759
|
-
|
2098
|
+
trace: Trace = get_trace(
|
2099
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2100
|
+
)
|
2101
|
+
trace.debug(f"[STAGE]: Execute Branch: {branch!r}")
|
1760
2102
|
|
1761
2103
|
# NOTE: Create nested-context
|
1762
|
-
|
1763
|
-
|
2104
|
+
current_context: DictData = copy.deepcopy(params)
|
2105
|
+
current_context.update({"branch": branch})
|
1764
2106
|
nestet_context: DictData = {"branch": branch, "stages": {}}
|
1765
2107
|
|
1766
2108
|
total_stage: int = len(self.parallel[branch])
|
@@ -1775,7 +2117,8 @@ class ParallelStage(BaseNestedStage):
|
|
1775
2117
|
"Branch execution was canceled from the event before "
|
1776
2118
|
"start branch execution."
|
1777
2119
|
)
|
1778
|
-
|
2120
|
+
catch(
|
2121
|
+
context=context,
|
1779
2122
|
status=CANCEL,
|
1780
2123
|
parallel={
|
1781
2124
|
branch: {
|
@@ -1790,14 +2133,15 @@ class ParallelStage(BaseNestedStage):
|
|
1790
2133
|
)
|
1791
2134
|
raise StageCancelError(error_msg, refs=branch)
|
1792
2135
|
|
1793
|
-
rs: Result = stage.
|
1794
|
-
params=
|
1795
|
-
run_id=
|
1796
|
-
parent_run_id=result.parent_run_id,
|
2136
|
+
rs: Result = stage.execute(
|
2137
|
+
params=current_context,
|
2138
|
+
run_id=parent_run_id,
|
1797
2139
|
event=event,
|
1798
2140
|
)
|
1799
2141
|
stage.set_outputs(rs.context, to=nestet_context)
|
1800
|
-
stage.set_outputs(
|
2142
|
+
stage.set_outputs(
|
2143
|
+
stage.get_outputs(nestet_context), to=current_context
|
2144
|
+
)
|
1801
2145
|
|
1802
2146
|
if rs.status == SKIP:
|
1803
2147
|
skips[i] = True
|
@@ -1808,7 +2152,8 @@ class ParallelStage(BaseNestedStage):
|
|
1808
2152
|
f"Branch execution was break because its nested-stage, "
|
1809
2153
|
f"{stage.iden!r}, failed."
|
1810
2154
|
)
|
1811
|
-
|
2155
|
+
catch(
|
2156
|
+
context=context,
|
1812
2157
|
status=FAILED,
|
1813
2158
|
parallel={
|
1814
2159
|
branch: {
|
@@ -1828,7 +2173,8 @@ class ParallelStage(BaseNestedStage):
|
|
1828
2173
|
"Branch execution was canceled from the event after "
|
1829
2174
|
"end branch execution."
|
1830
2175
|
)
|
1831
|
-
|
2176
|
+
catch(
|
2177
|
+
context=context,
|
1832
2178
|
status=CANCEL,
|
1833
2179
|
parallel={
|
1834
2180
|
branch: {
|
@@ -1844,7 +2190,8 @@ class ParallelStage(BaseNestedStage):
|
|
1844
2190
|
raise StageCancelError(error_msg, refs=branch)
|
1845
2191
|
|
1846
2192
|
status: Status = SKIP if sum(skips) == total_stage else SUCCESS
|
1847
|
-
return status,
|
2193
|
+
return status, catch(
|
2194
|
+
context=context,
|
1848
2195
|
status=status,
|
1849
2196
|
parallel={
|
1850
2197
|
branch: {
|
@@ -1855,32 +2202,38 @@ class ParallelStage(BaseNestedStage):
|
|
1855
2202
|
},
|
1856
2203
|
)
|
1857
2204
|
|
1858
|
-
def
|
2205
|
+
def process(
|
1859
2206
|
self,
|
1860
2207
|
params: DictData,
|
2208
|
+
run_id: str,
|
2209
|
+
context: DictData,
|
1861
2210
|
*,
|
1862
|
-
|
2211
|
+
parent_run_id: Optional[str] = None,
|
1863
2212
|
event: Optional[Event] = None,
|
1864
2213
|
) -> Result:
|
1865
2214
|
"""Execute parallel each branch via multi-threading pool.
|
1866
2215
|
|
1867
|
-
:
|
1868
|
-
|
1869
|
-
|
1870
|
-
|
1871
|
-
|
1872
|
-
|
1873
|
-
|
2216
|
+
Args:
|
2217
|
+
params: A parameter data that want to use in this
|
2218
|
+
execution.
|
2219
|
+
run_id: A running stage ID.
|
2220
|
+
context: A context data.
|
2221
|
+
parent_run_id: A parent running ID. (Default is None)
|
2222
|
+
event: An event manager that use to track parent process
|
2223
|
+
was not force stopped.
|
2224
|
+
|
2225
|
+
Returns:
|
2226
|
+
Result: The execution result with status and context data.
|
1874
2227
|
"""
|
1875
|
-
|
1876
|
-
run_id=
|
1877
|
-
extras=self.extras,
|
2228
|
+
trace: Trace = get_trace(
|
2229
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1878
2230
|
)
|
1879
2231
|
event: Event = event or Event()
|
1880
|
-
|
1881
|
-
|
2232
|
+
trace.info(f"[STAGE]: Parallel with {self.max_workers} workers.")
|
2233
|
+
catch(
|
2234
|
+
context=context,
|
1882
2235
|
status=WAIT,
|
1883
|
-
|
2236
|
+
updated={"workers": self.max_workers, "parallel": {}},
|
1884
2237
|
)
|
1885
2238
|
len_parallel: int = len(self.parallel)
|
1886
2239
|
if event and event.is_set():
|
@@ -1891,25 +2244,32 @@ class ParallelStage(BaseNestedStage):
|
|
1891
2244
|
with ThreadPoolExecutor(self.max_workers, "stp") as executor:
|
1892
2245
|
futures: list[Future] = [
|
1893
2246
|
executor.submit(
|
1894
|
-
self.
|
2247
|
+
self._process_branch,
|
1895
2248
|
branch=branch,
|
1896
2249
|
params=params,
|
1897
|
-
|
2250
|
+
run_id=run_id,
|
2251
|
+
context=context,
|
2252
|
+
parent_run_id=parent_run_id,
|
1898
2253
|
event=event,
|
1899
2254
|
)
|
1900
2255
|
for branch in self.parallel
|
1901
2256
|
]
|
1902
|
-
|
2257
|
+
errors: DictData = {}
|
1903
2258
|
statuses: list[Status] = [WAIT] * len_parallel
|
1904
2259
|
for i, future in enumerate(as_completed(futures), start=0):
|
1905
2260
|
try:
|
1906
2261
|
statuses[i], _ = future.result()
|
1907
2262
|
except StageError as e:
|
1908
2263
|
statuses[i] = get_status_from_error(e)
|
1909
|
-
self.mark_errors(
|
1910
|
-
|
1911
|
-
|
1912
|
-
|
2264
|
+
self.mark_errors(errors, e)
|
2265
|
+
|
2266
|
+
st: Status = validate_statuses(statuses)
|
2267
|
+
return Result(
|
2268
|
+
run_id=run_id,
|
2269
|
+
parent_run_id=parent_run_id,
|
2270
|
+
status=st,
|
2271
|
+
context=catch(context, status=st, updated=errors),
|
2272
|
+
extras=self.extras,
|
1913
2273
|
)
|
1914
2274
|
|
1915
2275
|
|
@@ -1962,15 +2322,17 @@ class ForEachStage(BaseNestedStage):
|
|
1962
2322
|
),
|
1963
2323
|
)
|
1964
2324
|
|
1965
|
-
def
|
2325
|
+
def _process_item(
|
1966
2326
|
self,
|
1967
2327
|
index: int,
|
1968
2328
|
item: StrOrInt,
|
1969
2329
|
params: DictData,
|
1970
|
-
|
2330
|
+
run_id: str,
|
2331
|
+
context: DictData,
|
1971
2332
|
*,
|
2333
|
+
parent_run_id: Optional[str] = None,
|
1972
2334
|
event: Optional[Event] = None,
|
1973
|
-
) -> tuple[Status,
|
2335
|
+
) -> tuple[Status, DictData]:
|
1974
2336
|
"""Execute item that will execute all nested-stage that was set in this
|
1975
2337
|
stage with specific foreach item.
|
1976
2338
|
|
@@ -1980,7 +2342,9 @@ class ForEachStage(BaseNestedStage):
|
|
1980
2342
|
:param index: (int) An index value of foreach loop.
|
1981
2343
|
:param item: (str | int) An item that want to execution.
|
1982
2344
|
:param params: (DictData) A parameter data.
|
1983
|
-
:param
|
2345
|
+
:param run_id: (str)
|
2346
|
+
:param context: (DictData)
|
2347
|
+
:param parent_run_id: (str | None)
|
1984
2348
|
:param event: (Event) An Event manager instance that use to cancel this
|
1985
2349
|
execution if it forces stopped by parent execution.
|
1986
2350
|
(Default is None)
|
@@ -1994,12 +2358,15 @@ class ForEachStage(BaseNestedStage):
|
|
1994
2358
|
|
1995
2359
|
:rtype: tuple[Status, Result]
|
1996
2360
|
"""
|
1997
|
-
|
2361
|
+
trace: Trace = get_trace(
|
2362
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2363
|
+
)
|
2364
|
+
trace.debug(f"[STAGE]: Execute Item: {item!r}")
|
1998
2365
|
key: StrOrInt = index if self.use_index_as_key else item
|
1999
2366
|
|
2000
2367
|
# NOTE: Create nested-context data from the passing context.
|
2001
|
-
|
2002
|
-
|
2368
|
+
current_context: DictData = copy.deepcopy(params)
|
2369
|
+
current_context.update({"item": item, "loop": index})
|
2003
2370
|
nestet_context: DictData = {"item": item, "stages": {}}
|
2004
2371
|
|
2005
2372
|
total_stage: int = len(self.stages)
|
@@ -2014,7 +2381,8 @@ class ForEachStage(BaseNestedStage):
|
|
2014
2381
|
"Item execution was canceled from the event before start "
|
2015
2382
|
"item execution."
|
2016
2383
|
)
|
2017
|
-
|
2384
|
+
catch(
|
2385
|
+
context=context,
|
2018
2386
|
status=CANCEL,
|
2019
2387
|
foreach={
|
2020
2388
|
key: {
|
@@ -2029,14 +2397,16 @@ class ForEachStage(BaseNestedStage):
|
|
2029
2397
|
)
|
2030
2398
|
raise StageCancelError(error_msg, refs=key)
|
2031
2399
|
|
2032
|
-
|
2033
|
-
|
2034
|
-
|
2035
|
-
|
2400
|
+
# NOTE: Nested-stage execute will pass only params and context only.
|
2401
|
+
rs: Result = stage.execute(
|
2402
|
+
params=current_context,
|
2403
|
+
run_id=parent_run_id,
|
2036
2404
|
event=event,
|
2037
2405
|
)
|
2038
2406
|
stage.set_outputs(rs.context, to=nestet_context)
|
2039
|
-
stage.set_outputs(
|
2407
|
+
stage.set_outputs(
|
2408
|
+
stage.get_outputs(nestet_context), to=current_context
|
2409
|
+
)
|
2040
2410
|
|
2041
2411
|
if rs.status == SKIP:
|
2042
2412
|
skips[i] = True
|
@@ -2047,8 +2417,9 @@ class ForEachStage(BaseNestedStage):
|
|
2047
2417
|
f"Item execution was break because its nested-stage, "
|
2048
2418
|
f"{stage.iden!r}, failed."
|
2049
2419
|
)
|
2050
|
-
|
2051
|
-
|
2420
|
+
trace.warning(f"[STAGE]: {error_msg}")
|
2421
|
+
catch(
|
2422
|
+
context=context,
|
2052
2423
|
status=FAILED,
|
2053
2424
|
foreach={
|
2054
2425
|
key: {
|
@@ -2068,7 +2439,8 @@ class ForEachStage(BaseNestedStage):
|
|
2068
2439
|
"Item execution was canceled from the event after "
|
2069
2440
|
"end item execution."
|
2070
2441
|
)
|
2071
|
-
|
2442
|
+
catch(
|
2443
|
+
context=context,
|
2072
2444
|
status=CANCEL,
|
2073
2445
|
foreach={
|
2074
2446
|
key: {
|
@@ -2084,7 +2456,8 @@ class ForEachStage(BaseNestedStage):
|
|
2084
2456
|
raise StageCancelError(error_msg, refs=key)
|
2085
2457
|
|
2086
2458
|
status: Status = SKIP if sum(skips) == total_stage else SUCCESS
|
2087
|
-
return status,
|
2459
|
+
return status, catch(
|
2460
|
+
context=context,
|
2088
2461
|
status=status,
|
2089
2462
|
foreach={
|
2090
2463
|
key: {
|
@@ -2095,11 +2468,13 @@ class ForEachStage(BaseNestedStage):
|
|
2095
2468
|
},
|
2096
2469
|
)
|
2097
2470
|
|
2098
|
-
def
|
2471
|
+
def process(
|
2099
2472
|
self,
|
2100
2473
|
params: DictData,
|
2474
|
+
run_id: str,
|
2475
|
+
context: DictData,
|
2101
2476
|
*,
|
2102
|
-
|
2477
|
+
parent_run_id: Optional[str] = None,
|
2103
2478
|
event: Optional[Event] = None,
|
2104
2479
|
) -> Result:
|
2105
2480
|
"""Execute the stages that pass each item form the foreach field.
|
@@ -2108,18 +2483,20 @@ class ForEachStage(BaseNestedStage):
|
|
2108
2483
|
value more than 1. It will cancel all nested-stage execution when it has
|
2109
2484
|
any item loop raise failed or canceled error.
|
2110
2485
|
|
2111
|
-
:
|
2112
|
-
|
2113
|
-
|
2114
|
-
|
2115
|
-
|
2116
|
-
|
2117
|
-
|
2118
|
-
|
2486
|
+
Args:
|
2487
|
+
params: A parameter data that want to use in this
|
2488
|
+
execution.
|
2489
|
+
run_id: A running stage ID.
|
2490
|
+
context: A context data.
|
2491
|
+
parent_run_id: A parent running ID. (Default is None)
|
2492
|
+
event: An event manager that use to track parent process
|
2493
|
+
was not force stopped.
|
2494
|
+
|
2495
|
+
Returns:
|
2496
|
+
Result: The execution result with status and context data.
|
2119
2497
|
"""
|
2120
|
-
|
2121
|
-
run_id=
|
2122
|
-
extras=self.extras,
|
2498
|
+
trace: Trace = get_trace(
|
2499
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2123
2500
|
)
|
2124
2501
|
event: Event = event or Event()
|
2125
2502
|
foreach: Union[list[str], list[int]] = pass_env(
|
@@ -2149,8 +2526,12 @@ class ForEachStage(BaseNestedStage):
|
|
2149
2526
|
"duplicate item, it should set `use_index_as_key: true`."
|
2150
2527
|
)
|
2151
2528
|
|
2152
|
-
|
2153
|
-
|
2529
|
+
trace.info(f"[STAGE]: Foreach: {foreach!r}.")
|
2530
|
+
catch(
|
2531
|
+
context=context,
|
2532
|
+
status=WAIT,
|
2533
|
+
updated={"items": foreach, "foreach": {}},
|
2534
|
+
)
|
2154
2535
|
len_foreach: int = len(foreach)
|
2155
2536
|
if event and event.is_set():
|
2156
2537
|
raise StageCancelError(
|
@@ -2160,30 +2541,32 @@ class ForEachStage(BaseNestedStage):
|
|
2160
2541
|
with ThreadPoolExecutor(self.concurrent, "stf") as executor:
|
2161
2542
|
futures: list[Future] = [
|
2162
2543
|
executor.submit(
|
2163
|
-
self.
|
2544
|
+
self._process_item,
|
2164
2545
|
index=i,
|
2165
2546
|
item=item,
|
2166
2547
|
params=params,
|
2167
|
-
|
2548
|
+
run_id=run_id,
|
2549
|
+
context=context,
|
2550
|
+
parent_run_id=parent_run_id,
|
2168
2551
|
event=event,
|
2169
2552
|
)
|
2170
2553
|
for i, item in enumerate(foreach, start=0)
|
2171
2554
|
]
|
2172
2555
|
|
2173
|
-
|
2556
|
+
errors: DictData = {}
|
2174
2557
|
statuses: list[Status] = [WAIT] * len_foreach
|
2175
2558
|
fail_fast: bool = False
|
2176
2559
|
|
2177
2560
|
done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
|
2178
2561
|
if len(list(done)) != len(futures):
|
2179
|
-
|
2562
|
+
trace.warning(
|
2180
2563
|
"[STAGE]: Set the event for stop pending for-each stage."
|
2181
2564
|
)
|
2182
2565
|
event.set()
|
2183
2566
|
for future in not_done:
|
2184
2567
|
future.cancel()
|
2185
2568
|
|
2186
|
-
time.sleep(0.025
|
2569
|
+
time.sleep(0.01) # Reduced from 0.025 for better responsiveness
|
2187
2570
|
nd: str = (
|
2188
2571
|
(
|
2189
2572
|
f", {len(not_done)} item"
|
@@ -2192,18 +2575,17 @@ class ForEachStage(BaseNestedStage):
|
|
2192
2575
|
if not_done
|
2193
2576
|
else ""
|
2194
2577
|
)
|
2195
|
-
|
2196
|
-
f"[STAGE]: ... Foreach-Stage set failed event{nd}"
|
2197
|
-
)
|
2578
|
+
trace.debug(f"[STAGE]: ... Foreach-Stage set failed event{nd}")
|
2198
2579
|
done: Iterator[Future] = as_completed(futures)
|
2199
2580
|
fail_fast = True
|
2200
2581
|
|
2201
2582
|
for i, future in enumerate(done, start=0):
|
2202
2583
|
try:
|
2584
|
+
# NOTE: Ignore returned context because it already updated.
|
2203
2585
|
statuses[i], _ = future.result()
|
2204
2586
|
except StageError as e:
|
2205
2587
|
statuses[i] = get_status_from_error(e)
|
2206
|
-
self.mark_errors(
|
2588
|
+
self.mark_errors(errors, e)
|
2207
2589
|
except CancelledError:
|
2208
2590
|
pass
|
2209
2591
|
|
@@ -2214,7 +2596,13 @@ class ForEachStage(BaseNestedStage):
|
|
2214
2596
|
if fail_fast and status == CANCEL:
|
2215
2597
|
status = FAILED
|
2216
2598
|
|
2217
|
-
return
|
2599
|
+
return Result(
|
2600
|
+
run_id=run_id,
|
2601
|
+
parent_run_id=parent_run_id,
|
2602
|
+
status=status,
|
2603
|
+
context=catch(context, status=status, updated=errors),
|
2604
|
+
extras=self.extras,
|
2605
|
+
)
|
2218
2606
|
|
2219
2607
|
|
2220
2608
|
class UntilStage(BaseNestedStage):
|
@@ -2265,32 +2653,40 @@ class UntilStage(BaseNestedStage):
|
|
2265
2653
|
alias="max-loop",
|
2266
2654
|
)
|
2267
2655
|
|
2268
|
-
def
|
2656
|
+
def _process_loop(
|
2269
2657
|
self,
|
2270
2658
|
item: T,
|
2271
2659
|
loop: int,
|
2272
2660
|
params: DictData,
|
2273
|
-
|
2661
|
+
run_id: str,
|
2662
|
+
context: DictData,
|
2663
|
+
*,
|
2664
|
+
parent_run_id: Optional[str] = None,
|
2274
2665
|
event: Optional[Event] = None,
|
2275
|
-
) -> tuple[Status,
|
2666
|
+
) -> tuple[Status, DictData, T]:
|
2276
2667
|
"""Execute loop that will execute all nested-stage that was set in this
|
2277
2668
|
stage with specific loop and item.
|
2278
2669
|
|
2279
2670
|
:param item: (T) An item that want to execution.
|
2280
2671
|
:param loop: (int) A number of loop.
|
2281
2672
|
:param params: (DictData) A parameter data.
|
2282
|
-
:param
|
2673
|
+
:param run_id: (str)
|
2674
|
+
:param context: (DictData)
|
2675
|
+
:param parent_run_id: (str | None)
|
2283
2676
|
:param event: (Event) An Event manager instance that use to cancel this
|
2284
2677
|
execution if it forces stopped by parent execution.
|
2285
2678
|
|
2286
|
-
:rtype: tuple[Status,
|
2679
|
+
:rtype: tuple[Status, DictData, T]
|
2287
2680
|
:return: Return a pair of Result and changed item.
|
2288
2681
|
"""
|
2289
|
-
|
2682
|
+
trace: Trace = get_trace(
|
2683
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2684
|
+
)
|
2685
|
+
trace.debug(f"[STAGE]: Execute Loop: {loop} (Item {item!r})")
|
2290
2686
|
|
2291
2687
|
# NOTE: Create nested-context
|
2292
|
-
|
2293
|
-
|
2688
|
+
current_context: DictData = copy.deepcopy(params)
|
2689
|
+
current_context.update({"item": item, "loop": loop})
|
2294
2690
|
nestet_context: DictData = {"loop": loop, "item": item, "stages": {}}
|
2295
2691
|
|
2296
2692
|
next_item: Optional[T] = None
|
@@ -2306,7 +2702,8 @@ class UntilStage(BaseNestedStage):
|
|
2306
2702
|
"Loop execution was canceled from the event before start "
|
2307
2703
|
"loop execution."
|
2308
2704
|
)
|
2309
|
-
|
2705
|
+
catch(
|
2706
|
+
context=context,
|
2310
2707
|
status=CANCEL,
|
2311
2708
|
until={
|
2312
2709
|
loop: {
|
@@ -2322,10 +2719,9 @@ class UntilStage(BaseNestedStage):
|
|
2322
2719
|
)
|
2323
2720
|
raise StageCancelError(error_msg, refs=loop)
|
2324
2721
|
|
2325
|
-
rs: Result = stage.
|
2326
|
-
params=
|
2327
|
-
run_id=
|
2328
|
-
parent_run_id=result.parent_run_id,
|
2722
|
+
rs: Result = stage.execute(
|
2723
|
+
params=current_context,
|
2724
|
+
run_id=parent_run_id,
|
2329
2725
|
event=event,
|
2330
2726
|
)
|
2331
2727
|
stage.set_outputs(rs.context, to=nestet_context)
|
@@ -2333,7 +2729,7 @@ class UntilStage(BaseNestedStage):
|
|
2333
2729
|
if "item" in (_output := stage.get_outputs(nestet_context)):
|
2334
2730
|
next_item = _output["item"]
|
2335
2731
|
|
2336
|
-
stage.set_outputs(_output, to=
|
2732
|
+
stage.set_outputs(_output, to=current_context)
|
2337
2733
|
|
2338
2734
|
if rs.status == SKIP:
|
2339
2735
|
skips[i] = True
|
@@ -2344,7 +2740,8 @@ class UntilStage(BaseNestedStage):
|
|
2344
2740
|
f"Loop execution was break because its nested-stage, "
|
2345
2741
|
f"{stage.iden!r}, failed."
|
2346
2742
|
)
|
2347
|
-
|
2743
|
+
catch(
|
2744
|
+
context=context,
|
2348
2745
|
status=FAILED,
|
2349
2746
|
until={
|
2350
2747
|
loop: {
|
@@ -2365,7 +2762,8 @@ class UntilStage(BaseNestedStage):
|
|
2365
2762
|
"Loop execution was canceled from the event after "
|
2366
2763
|
"end loop execution."
|
2367
2764
|
)
|
2368
|
-
|
2765
|
+
catch(
|
2766
|
+
context=context,
|
2369
2767
|
status=CANCEL,
|
2370
2768
|
until={
|
2371
2769
|
loop: {
|
@@ -2384,7 +2782,8 @@ class UntilStage(BaseNestedStage):
|
|
2384
2782
|
status: Status = SKIP if sum(skips) == total_stage else SUCCESS
|
2385
2783
|
return (
|
2386
2784
|
status,
|
2387
|
-
|
2785
|
+
catch(
|
2786
|
+
context=context,
|
2388
2787
|
status=status,
|
2389
2788
|
until={
|
2390
2789
|
loop: {
|
@@ -2398,37 +2797,42 @@ class UntilStage(BaseNestedStage):
|
|
2398
2797
|
next_item,
|
2399
2798
|
)
|
2400
2799
|
|
2401
|
-
def
|
2800
|
+
def process(
|
2402
2801
|
self,
|
2403
2802
|
params: DictData,
|
2803
|
+
run_id: str,
|
2804
|
+
context: DictData,
|
2404
2805
|
*,
|
2405
|
-
|
2806
|
+
parent_run_id: Optional[str] = None,
|
2406
2807
|
event: Optional[Event] = None,
|
2407
2808
|
) -> Result:
|
2408
2809
|
"""Execute until loop with checking the until condition before release
|
2409
2810
|
the next loop.
|
2410
2811
|
|
2411
|
-
:
|
2412
|
-
|
2413
|
-
|
2414
|
-
|
2415
|
-
|
2416
|
-
|
2417
|
-
|
2812
|
+
Args:
|
2813
|
+
params: A parameter data that want to use in this
|
2814
|
+
execution.
|
2815
|
+
run_id: A running stage ID.
|
2816
|
+
context: A context data.
|
2817
|
+
parent_run_id: A parent running ID. (Default is None)
|
2818
|
+
event: An event manager that use to track parent process
|
2819
|
+
was not force stopped.
|
2820
|
+
|
2821
|
+
Returns:
|
2822
|
+
Result: The execution result with status and context data.
|
2418
2823
|
"""
|
2419
|
-
|
2420
|
-
run_id=
|
2421
|
-
extras=self.extras,
|
2824
|
+
trace: Trace = get_trace(
|
2825
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2422
2826
|
)
|
2423
2827
|
event: Event = event or Event()
|
2424
|
-
|
2828
|
+
trace.info(f"[STAGE]: Until: {self.until!r}")
|
2425
2829
|
item: Union[str, int, bool] = pass_env(
|
2426
2830
|
param2template(self.item, params, extras=self.extras)
|
2427
2831
|
)
|
2428
2832
|
loop: int = 1
|
2429
2833
|
until_rs: bool = True
|
2430
2834
|
exceed_loop: bool = False
|
2431
|
-
|
2835
|
+
catch(context=context, status=WAIT, updated={"until": {}})
|
2432
2836
|
statuses: list[Status] = []
|
2433
2837
|
while until_rs and not (exceed_loop := (loop > self.max_loop)):
|
2434
2838
|
|
@@ -2437,18 +2841,20 @@ class UntilStage(BaseNestedStage):
|
|
2437
2841
|
"Execution was canceled from the event before start loop."
|
2438
2842
|
)
|
2439
2843
|
|
2440
|
-
status,
|
2844
|
+
status, context, item = self._process_loop(
|
2441
2845
|
item=item,
|
2442
2846
|
loop=loop,
|
2443
2847
|
params=params,
|
2444
|
-
|
2848
|
+
run_id=run_id,
|
2849
|
+
context=context,
|
2850
|
+
parent_run_id=parent_run_id,
|
2445
2851
|
event=event,
|
2446
2852
|
)
|
2447
2853
|
|
2448
2854
|
loop += 1
|
2449
2855
|
if item is None:
|
2450
2856
|
item: int = loop
|
2451
|
-
|
2857
|
+
trace.warning(
|
2452
2858
|
f"[STAGE]: Return loop not set the item. It uses loop: "
|
2453
2859
|
f"{loop} by default."
|
2454
2860
|
)
|
@@ -2479,7 +2885,15 @@ class UntilStage(BaseNestedStage):
|
|
2479
2885
|
f"loop{'s' if self.max_loop > 1 else ''}."
|
2480
2886
|
)
|
2481
2887
|
raise StageError(error_msg)
|
2482
|
-
|
2888
|
+
|
2889
|
+
st: Status = validate_statuses(statuses)
|
2890
|
+
return Result(
|
2891
|
+
run_id=run_id,
|
2892
|
+
parent_run_id=parent_run_id,
|
2893
|
+
status=st,
|
2894
|
+
context=catch(context, status=st),
|
2895
|
+
extras=self.extras,
|
2896
|
+
)
|
2483
2897
|
|
2484
2898
|
|
2485
2899
|
class Match(BaseModel):
|
@@ -2535,28 +2949,36 @@ class CaseStage(BaseNestedStage):
|
|
2535
2949
|
alias="skip-not-match",
|
2536
2950
|
)
|
2537
2951
|
|
2538
|
-
def
|
2952
|
+
def _process_case(
|
2539
2953
|
self,
|
2540
2954
|
case: str,
|
2541
2955
|
stages: list[Stage],
|
2542
2956
|
params: DictData,
|
2543
|
-
|
2957
|
+
run_id: str,
|
2958
|
+
context: DictData,
|
2544
2959
|
*,
|
2960
|
+
parent_run_id: Optional[str] = None,
|
2545
2961
|
event: Optional[Event] = None,
|
2546
|
-
) ->
|
2962
|
+
) -> tuple[Status, DictData]:
|
2547
2963
|
"""Execute case.
|
2548
2964
|
|
2549
2965
|
:param case: (str) A case that want to execution.
|
2550
2966
|
:param stages: (list[Stage]) A list of stage.
|
2551
2967
|
:param params: (DictData) A parameter data.
|
2552
|
-
:param
|
2968
|
+
:param run_id: (str)
|
2969
|
+
:param context: (DictData)
|
2970
|
+
:param parent_run_id: (str | None)
|
2553
2971
|
:param event: (Event) An Event manager instance that use to cancel this
|
2554
2972
|
execution if it forces stopped by parent execution.
|
2555
2973
|
|
2556
|
-
:rtype:
|
2974
|
+
:rtype: DictData
|
2557
2975
|
"""
|
2558
|
-
|
2559
|
-
|
2976
|
+
trace: Trace = get_trace(
|
2977
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2978
|
+
)
|
2979
|
+
trace.debug(f"[STAGE]: Execute Case: {case!r}")
|
2980
|
+
current_context: DictData = copy.deepcopy(params)
|
2981
|
+
current_context.update({"case": case})
|
2560
2982
|
output: DictData = {"case": case, "stages": {}}
|
2561
2983
|
for stage in stages:
|
2562
2984
|
|
@@ -2568,69 +2990,77 @@ class CaseStage(BaseNestedStage):
|
|
2568
2990
|
"Case-Stage was canceled from event that had set before "
|
2569
2991
|
"stage case execution."
|
2570
2992
|
)
|
2571
|
-
return
|
2993
|
+
return CANCEL, catch(
|
2994
|
+
context=context,
|
2572
2995
|
status=CANCEL,
|
2573
|
-
|
2996
|
+
updated={
|
2574
2997
|
"case": case,
|
2575
2998
|
"stages": filter_func(output.pop("stages", {})),
|
2576
2999
|
"errors": StageError(error_msg).to_dict(),
|
2577
3000
|
},
|
2578
3001
|
)
|
2579
3002
|
|
2580
|
-
rs: Result = stage.
|
2581
|
-
params=
|
2582
|
-
run_id=
|
2583
|
-
parent_run_id=result.parent_run_id,
|
3003
|
+
rs: Result = stage.execute(
|
3004
|
+
params=current_context,
|
3005
|
+
run_id=parent_run_id,
|
2584
3006
|
event=event,
|
2585
3007
|
)
|
2586
3008
|
stage.set_outputs(rs.context, to=output)
|
2587
|
-
stage.set_outputs(stage.get_outputs(output), to=
|
3009
|
+
stage.set_outputs(stage.get_outputs(output), to=current_context)
|
2588
3010
|
|
2589
3011
|
if rs.status == FAILED:
|
2590
3012
|
error_msg: str = (
|
2591
3013
|
f"Case-Stage was break because it has a sub stage, "
|
2592
3014
|
f"{stage.iden}, failed without raise error."
|
2593
3015
|
)
|
2594
|
-
return
|
3016
|
+
return FAILED, catch(
|
3017
|
+
context=context,
|
2595
3018
|
status=FAILED,
|
2596
|
-
|
3019
|
+
updated={
|
2597
3020
|
"case": case,
|
2598
3021
|
"stages": filter_func(output.pop("stages", {})),
|
2599
3022
|
"errors": StageError(error_msg).to_dict(),
|
2600
3023
|
},
|
2601
3024
|
)
|
2602
|
-
return
|
3025
|
+
return SUCCESS, catch(
|
3026
|
+
context=context,
|
2603
3027
|
status=SUCCESS,
|
2604
|
-
|
3028
|
+
updated={
|
2605
3029
|
"case": case,
|
2606
3030
|
"stages": filter_func(output.pop("stages", {})),
|
2607
3031
|
},
|
2608
3032
|
)
|
2609
3033
|
|
2610
|
-
def
|
3034
|
+
def process(
|
2611
3035
|
self,
|
2612
3036
|
params: DictData,
|
3037
|
+
run_id: str,
|
3038
|
+
context: DictData,
|
2613
3039
|
*,
|
2614
|
-
|
3040
|
+
parent_run_id: Optional[str] = None,
|
2615
3041
|
event: Optional[Event] = None,
|
2616
3042
|
) -> Result:
|
2617
3043
|
"""Execute case-match condition that pass to the case field.
|
2618
3044
|
|
2619
|
-
:
|
2620
|
-
|
2621
|
-
|
2622
|
-
|
2623
|
-
|
2624
|
-
|
3045
|
+
Args:
|
3046
|
+
params: A parameter data that want to use in this
|
3047
|
+
execution.
|
3048
|
+
run_id: A running stage ID.
|
3049
|
+
context: A context data.
|
3050
|
+
parent_run_id: A parent running ID. (Default is None)
|
3051
|
+
event: An event manager that use to track parent process
|
3052
|
+
was not force stopped.
|
3053
|
+
|
3054
|
+
Returns:
|
3055
|
+
Result: The execution result with status and context data.
|
2625
3056
|
"""
|
2626
|
-
|
2627
|
-
run_id=
|
2628
|
-
extras=self.extras,
|
3057
|
+
trace: Trace = get_trace(
|
3058
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2629
3059
|
)
|
2630
3060
|
|
2631
3061
|
_case: StrOrNone = param2template(self.case, params, extras=self.extras)
|
2632
3062
|
|
2633
|
-
|
3063
|
+
trace.info(f"[STAGE]: Case: {_case!r}.")
|
2634
3064
|
_else: Optional[Match] = None
|
2635
3065
|
stages: Optional[list[Stage]] = None
|
2636
3066
|
for match in self.match:
|
@@ -2662,9 +3092,21 @@ class CaseStage(BaseNestedStage):
|
|
2662
3092
|
"Execution was canceled from the event before start "
|
2663
3093
|
"case execution."
|
2664
3094
|
)
|
2665
|
-
|
2666
|
-
|
2667
|
-
|
3095
|
+
status, context = self._process_case(
|
3096
|
+
case=_case,
|
3097
|
+
stages=stages,
|
3098
|
+
params=params,
|
3099
|
+
run_id=run_id,
|
3100
|
+
context=context,
|
3101
|
+
parent_run_id=parent_run_id,
|
3102
|
+
event=event,
|
3103
|
+
)
|
3104
|
+
return Result(
|
3105
|
+
run_id=run_id,
|
3106
|
+
parent_run_id=parent_run_id,
|
3107
|
+
status=status,
|
3108
|
+
context=catch(context, status=status),
|
3109
|
+
extras=self.extras,
|
2668
3110
|
)
|
2669
3111
|
|
2670
3112
|
|
@@ -2687,53 +3129,65 @@ class RaiseStage(BaseAsyncStage):
|
|
2687
3129
|
alias="raise",
|
2688
3130
|
)
|
2689
3131
|
|
2690
|
-
def
|
3132
|
+
def process(
|
2691
3133
|
self,
|
2692
3134
|
params: DictData,
|
3135
|
+
run_id: str,
|
3136
|
+
context: DictData,
|
2693
3137
|
*,
|
2694
|
-
|
3138
|
+
parent_run_id: Optional[str] = None,
|
2695
3139
|
event: Optional[Event] = None,
|
2696
3140
|
) -> Result:
|
2697
3141
|
"""Raise the StageError object with the message field execution.
|
2698
3142
|
|
2699
|
-
:
|
2700
|
-
|
2701
|
-
|
2702
|
-
|
3143
|
+
Args:
|
3144
|
+
params: A parameter data that want to use in this
|
3145
|
+
execution.
|
3146
|
+
run_id: A running stage ID.
|
3147
|
+
context: A context data.
|
3148
|
+
parent_run_id: A parent running ID. (Default is None)
|
3149
|
+
event: An event manager that use to track parent process
|
3150
|
+
was not force stopped.
|
3151
|
+
|
3152
|
+
Returns:
|
3153
|
+
Result: The execution result with status and context data.
|
2703
3154
|
"""
|
2704
|
-
|
2705
|
-
run_id=
|
2706
|
-
extras=self.extras,
|
3155
|
+
trace: Trace = get_trace(
|
3156
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2707
3157
|
)
|
2708
3158
|
message: str = param2template(self.message, params, extras=self.extras)
|
2709
|
-
|
3159
|
+
trace.info(f"[STAGE]: Message: ( {message} )")
|
2710
3160
|
raise StageError(message)
|
2711
3161
|
|
2712
|
-
async def
|
3162
|
+
async def async_process(
|
2713
3163
|
self,
|
2714
3164
|
params: DictData,
|
3165
|
+
run_id: str,
|
3166
|
+
context: DictData,
|
2715
3167
|
*,
|
2716
|
-
|
3168
|
+
parent_run_id: Optional[str] = None,
|
2717
3169
|
event: Optional[Event] = None,
|
2718
3170
|
) -> Result:
|
2719
3171
|
"""Async execution method for this Empty stage that only logging out to
|
2720
3172
|
stdout.
|
2721
3173
|
|
2722
|
-
:
|
2723
|
-
|
2724
|
-
|
2725
|
-
|
2726
|
-
|
2727
|
-
|
2728
|
-
|
2729
|
-
|
3174
|
+
Args:
|
3175
|
+
params: A parameter data that want to use in this
|
3176
|
+
execution.
|
3177
|
+
run_id: A running stage ID.
|
3178
|
+
context: A context data.
|
3179
|
+
parent_run_id: A parent running ID. (Default is None)
|
3180
|
+
event: An event manager that use to track parent process
|
3181
|
+
was not force stopped.
|
3182
|
+
|
3183
|
+
Returns:
|
3184
|
+
Result: The execution result with status and context data.
|
2730
3185
|
"""
|
2731
|
-
|
2732
|
-
run_id=
|
2733
|
-
extras=self.extras,
|
3186
|
+
trace: Trace = get_trace(
|
3187
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2734
3188
|
)
|
2735
3189
|
message: str = param2template(self.message, params, extras=self.extras)
|
2736
|
-
await
|
3190
|
+
await trace.ainfo(f"[STAGE]: Execute Raise-Stage: ( {message} )")
|
2737
3191
|
raise StageError(message)
|
2738
3192
|
|
2739
3193
|
|
@@ -2781,20 +3235,25 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2781
3235
|
),
|
2782
3236
|
)
|
2783
3237
|
|
2784
|
-
def
|
3238
|
+
def _process_task(
|
2785
3239
|
self,
|
2786
3240
|
params: DictData,
|
2787
|
-
|
3241
|
+
run_id: str,
|
3242
|
+
context: DictData,
|
3243
|
+
*,
|
3244
|
+
parent_run_id: Optional[str] = None,
|
2788
3245
|
event: Optional[Event] = None,
|
2789
|
-
) ->
|
3246
|
+
) -> DictData:
|
2790
3247
|
"""Execute Docker container task.
|
2791
3248
|
|
2792
3249
|
:param params: (DictData) A parameter data.
|
2793
|
-
:param
|
3250
|
+
:param run_id: (str)
|
3251
|
+
:param context: (DictData)
|
3252
|
+
:param parent_run_id: (str | None)
|
2794
3253
|
:param event: (Event) An Event manager instance that use to cancel this
|
2795
3254
|
execution if it forces stopped by parent execution.
|
2796
3255
|
|
2797
|
-
:rtype:
|
3256
|
+
:rtype: DictData
|
2798
3257
|
"""
|
2799
3258
|
try:
|
2800
3259
|
from docker import DockerClient
|
@@ -2805,6 +3264,9 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2805
3264
|
"by `pip install docker` first."
|
2806
3265
|
) from None
|
2807
3266
|
|
3267
|
+
trace: Trace = get_trace(
|
3268
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
3269
|
+
)
|
2808
3270
|
client = DockerClient(
|
2809
3271
|
base_url="unix://var/run/docker.sock", version="auto"
|
2810
3272
|
)
|
@@ -2819,16 +3281,17 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2819
3281
|
decode=True,
|
2820
3282
|
)
|
2821
3283
|
for line in resp:
|
2822
|
-
|
3284
|
+
trace.info(f"[STAGE]: ... {line}")
|
2823
3285
|
|
2824
3286
|
if event and event.is_set():
|
2825
3287
|
error_msg: str = (
|
2826
3288
|
"Docker-Stage was canceled from event that had set before "
|
2827
3289
|
"run the Docker container."
|
2828
3290
|
)
|
2829
|
-
return
|
3291
|
+
return catch(
|
3292
|
+
context=context,
|
2830
3293
|
status=CANCEL,
|
2831
|
-
|
3294
|
+
updated={"errors": StageError(error_msg).to_dict()},
|
2832
3295
|
)
|
2833
3296
|
|
2834
3297
|
unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S%f}"
|
@@ -2839,7 +3302,7 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2839
3302
|
volumes=pass_env(
|
2840
3303
|
{
|
2841
3304
|
Path.cwd()
|
2842
|
-
/ f".docker.{
|
3305
|
+
/ f".docker.{run_id}.logs": {
|
2843
3306
|
"bind": "/logs",
|
2844
3307
|
"mode": "rw",
|
2845
3308
|
},
|
@@ -2855,7 +3318,7 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2855
3318
|
)
|
2856
3319
|
|
2857
3320
|
for line in container.logs(stream=True, timestamps=True):
|
2858
|
-
|
3321
|
+
trace.info(f"[STAGE]: ... {line.strip().decode()}")
|
2859
3322
|
|
2860
3323
|
# NOTE: This code copy from the docker package.
|
2861
3324
|
exit_status: int = container.wait()["StatusCode"]
|
@@ -2869,36 +3332,42 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2869
3332
|
f"{self.image}:{self.tag}",
|
2870
3333
|
out.decode("utf-8"),
|
2871
3334
|
)
|
2872
|
-
output_file: Path = Path(f".docker.{
|
3335
|
+
output_file: Path = Path(f".docker.{run_id}.logs/outputs.json")
|
2873
3336
|
if not output_file.exists():
|
2874
|
-
return
|
2875
|
-
|
2876
|
-
|
2877
|
-
|
2878
|
-
|
3337
|
+
return catch(context=context, status=SUCCESS)
|
3338
|
+
return catch(
|
3339
|
+
context=context,
|
3340
|
+
status=SUCCESS,
|
3341
|
+
updated=json.loads(output_file.read_text()),
|
3342
|
+
)
|
2879
3343
|
|
2880
|
-
def
|
3344
|
+
def process(
|
2881
3345
|
self,
|
2882
3346
|
params: DictData,
|
3347
|
+
run_id: str,
|
3348
|
+
context: DictData,
|
2883
3349
|
*,
|
2884
|
-
|
3350
|
+
parent_run_id: Optional[str] = None,
|
2885
3351
|
event: Optional[Event] = None,
|
2886
3352
|
) -> Result:
|
2887
3353
|
"""Execute the Docker image via Python API.
|
2888
3354
|
|
2889
|
-
:
|
2890
|
-
|
2891
|
-
|
2892
|
-
|
2893
|
-
|
2894
|
-
|
3355
|
+
Args:
|
3356
|
+
params: A parameter data that want to use in this
|
3357
|
+
execution.
|
3358
|
+
run_id: A running stage ID.
|
3359
|
+
context: A context data.
|
3360
|
+
parent_run_id: A parent running ID. (Default is None)
|
3361
|
+
event: An event manager that use to track parent process
|
3362
|
+
was not force stopped.
|
3363
|
+
|
3364
|
+
Returns:
|
3365
|
+
Result: The execution result with status and context data.
|
2895
3366
|
"""
|
2896
|
-
|
2897
|
-
run_id=
|
2898
|
-
extras=self.extras,
|
3367
|
+
trace: Trace = get_trace(
|
3368
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2899
3369
|
)
|
2900
|
-
|
2901
|
-
result.trace.info(f"[STAGE]: Docker: {self.image}:{self.tag}")
|
3370
|
+
trace.info(f"[STAGE]: Docker: {self.image}:{self.tag}")
|
2902
3371
|
raise NotImplementedError("Docker Stage does not implement yet.")
|
2903
3372
|
|
2904
3373
|
|
@@ -2973,11 +3442,18 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
2973
3442
|
# Note: Remove .py file that use to run Python.
|
2974
3443
|
Path(f"./{f_name}").unlink()
|
2975
3444
|
|
2976
|
-
|
3445
|
+
@staticmethod
|
3446
|
+
def prepare_std(value: str) -> Optional[str]:
|
3447
|
+
"""Prepare returned standard string from subprocess."""
|
3448
|
+
return None if (out := value.strip("\n")) == "" else out
|
3449
|
+
|
3450
|
+
def process(
|
2977
3451
|
self,
|
2978
3452
|
params: DictData,
|
3453
|
+
run_id: str,
|
3454
|
+
context: DictData,
|
2979
3455
|
*,
|
2980
|
-
|
3456
|
+
parent_run_id: Optional[str] = None,
|
2981
3457
|
event: Optional[Event] = None,
|
2982
3458
|
) -> Result:
|
2983
3459
|
"""Execute the Python statement via Python virtual environment.
|
@@ -2986,25 +3462,29 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
2986
3462
|
- Create python file with the `uv` syntax.
|
2987
3463
|
- Execution python file with `uv run` via Python subprocess module.
|
2988
3464
|
|
2989
|
-
:
|
2990
|
-
|
2991
|
-
|
2992
|
-
|
2993
|
-
|
2994
|
-
|
3465
|
+
Args:
|
3466
|
+
params: A parameter data that want to use in this
|
3467
|
+
execution.
|
3468
|
+
run_id: A running stage ID.
|
3469
|
+
context: A context data.
|
3470
|
+
parent_run_id: A parent running ID. (Default is None)
|
3471
|
+
event: An event manager that use to track parent process
|
3472
|
+
was not force stopped.
|
3473
|
+
|
3474
|
+
Returns:
|
3475
|
+
Result: The execution result with status and context data.
|
2995
3476
|
"""
|
2996
|
-
|
2997
|
-
run_id=
|
2998
|
-
extras=self.extras,
|
3477
|
+
trace: Trace = get_trace(
|
3478
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2999
3479
|
)
|
3000
3480
|
run: str = param2template(dedent(self.run), params, extras=self.extras)
|
3001
3481
|
with self.create_py_file(
|
3002
3482
|
py=run,
|
3003
3483
|
values=param2template(self.vars, params, extras=self.extras),
|
3004
3484
|
deps=param2template(self.deps, params, extras=self.extras),
|
3005
|
-
run_id=
|
3485
|
+
run_id=run_id,
|
3006
3486
|
) as py:
|
3007
|
-
|
3487
|
+
trace.debug(f"[STAGE]: Create `{py}` file.")
|
3008
3488
|
rs: CompletedProcess = subprocess.run(
|
3009
3489
|
["python", "-m", "uv", "run", py, "--no-cache"],
|
3010
3490
|
# ["uv", "run", "--python", "3.9", py],
|
@@ -3024,13 +3504,20 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
3024
3504
|
f"Subprocess: {e}\nRunning Statement:\n---\n"
|
3025
3505
|
f"```python\n{run}\n```"
|
3026
3506
|
)
|
3027
|
-
return
|
3507
|
+
return Result(
|
3508
|
+
run_id=run_id,
|
3509
|
+
parent_run_id=parent_run_id,
|
3028
3510
|
status=SUCCESS,
|
3029
|
-
context=
|
3030
|
-
|
3031
|
-
|
3032
|
-
|
3033
|
-
|
3511
|
+
context=catch(
|
3512
|
+
context=context,
|
3513
|
+
status=SUCCESS,
|
3514
|
+
updated={
|
3515
|
+
"return_code": rs.returncode,
|
3516
|
+
"stdout": self.prepare_std(rs.stdout),
|
3517
|
+
"stderr": self.prepare_std(rs.stderr),
|
3518
|
+
},
|
3519
|
+
),
|
3520
|
+
extras=self.extras,
|
3034
3521
|
)
|
3035
3522
|
|
3036
3523
|
|