ddeutil-workflow 0.0.49__py3-none-any.whl → 0.0.51__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ddeutil/workflow/job.py CHANGED
@@ -3,7 +3,6 @@
3
3
  # Licensed under the MIT License. See LICENSE in the project root for
4
4
  # license information.
5
5
  # ------------------------------------------------------------------------------
6
- # [x] Use dynamic config
7
6
  """Job Model that use for keeping stages and node that running its stages.
8
7
  The job handle the lineage of stages and location of execution of stages that
9
8
  mean the job model able to define `runs-on` key that allow you to run this
@@ -28,19 +27,18 @@ from threading import Event
28
27
  from typing import Annotated, Any, Literal, Optional, Union
29
28
 
30
29
  from ddeutil.core import freeze_args
31
- from pydantic import BaseModel, ConfigDict, Discriminator, Field, Tag
30
+ from pydantic import BaseModel, ConfigDict, Discriminator, Field, SecretStr, Tag
32
31
  from pydantic.functional_validators import field_validator, model_validator
33
32
  from typing_extensions import Self
34
33
 
35
- from .__types import DictData, DictStr, Matrix, TupleStr
36
- from .conf import dynamic
34
+ from .__types import DictData, DictStr, Matrix
37
35
  from .exceptions import (
38
36
  JobException,
39
37
  StageException,
40
38
  UtilException,
41
39
  to_dict,
42
40
  )
43
- from .result import FAILED, SKIP, SUCCESS, WAIT, Result, Status
41
+ from .result import CANCEL, FAILED, SKIP, SUCCESS, WAIT, Result, Status
44
42
  from .reusables import has_template, param2template
45
43
  from .stages import Stage
46
44
  from .utils import cross_product, filter_func, gen_id
@@ -48,20 +46,6 @@ from .utils import cross_product, filter_func, gen_id
48
46
  MatrixFilter = list[dict[str, Union[str, int]]]
49
47
 
50
48
 
51
- __all__: TupleStr = (
52
- "Strategy",
53
- "Job",
54
- "TriggerRules",
55
- "RunsOn",
56
- "RunsOnLocal",
57
- "RunsOnSelfHosted",
58
- "RunsOnK8s",
59
- "make",
60
- "local_execute_strategy",
61
- "local_execute",
62
- )
63
-
64
-
65
49
  @freeze_args
66
50
  @lru_cache
67
51
  def make(
@@ -121,7 +105,6 @@ def make(
121
105
 
122
106
  add.append(inc)
123
107
 
124
- # NOTE: Merge all matrix together.
125
108
  final.extend(add)
126
109
  return final
127
110
 
@@ -194,24 +177,25 @@ class Strategy(BaseModel):
194
177
  return make(self.matrix, self.include, self.exclude)
195
178
 
196
179
 
197
- class TriggerRules(str, Enum):
180
+ class Rule(str, Enum):
198
181
  """Trigger rules enum object."""
199
182
 
200
- all_success: str = "all_success"
201
- all_failed: str = "all_failed"
202
- all_done: str = "all_done"
203
- one_failed: str = "one_failed"
204
- one_success: str = "one_success"
205
- none_failed: str = "none_failed"
206
- none_skipped: str = "none_skipped"
183
+ ALL_SUCCESS: str = "all_success"
184
+ ALL_FAILED: str = "all_failed"
185
+ ALL_DONE: str = "all_done"
186
+ ONE_FAILED: str = "one_failed"
187
+ ONE_SUCCESS: str = "one_success"
188
+ NONE_FAILED: str = "none_failed"
189
+ NONE_SKIPPED: str = "none_skipped"
207
190
 
208
191
 
209
- class RunsOnType(str, Enum):
192
+ class RunsOn(str, Enum):
210
193
  """Runs-On enum object."""
211
194
 
212
195
  LOCAL: str = "local"
213
196
  SELF_HOSTED: str = "self_hosted"
214
- K8S: str = "k8s"
197
+ AZ_BATCH: str = "azure_batch"
198
+ DOCKER: str = "docker"
215
199
 
216
200
 
217
201
  class BaseRunsOn(BaseModel): # pragma: no cov
@@ -221,47 +205,79 @@ class BaseRunsOn(BaseModel): # pragma: no cov
221
205
 
222
206
  model_config = ConfigDict(use_enum_values=True)
223
207
 
224
- type: Literal[RunsOnType.LOCAL]
208
+ type: RunsOn = Field(description="A runs-on type.")
225
209
  args: DictData = Field(
226
210
  default_factory=dict,
227
211
  alias="with",
212
+ description=(
213
+ "An argument that pass to the runs-on execution function. This "
214
+ "args will override by this child-model with specific args model."
215
+ ),
228
216
  )
229
217
 
230
218
 
231
- class RunsOnLocal(BaseRunsOn): # pragma: no cov
219
+ class OnLocal(BaseRunsOn): # pragma: no cov
232
220
  """Runs-on local."""
233
221
 
234
- type: Literal[RunsOnType.LOCAL] = Field(default=RunsOnType.LOCAL)
222
+ type: Literal[RunsOn.LOCAL] = Field(default=RunsOn.LOCAL)
235
223
 
236
224
 
237
225
  class SelfHostedArgs(BaseModel):
238
- host: str
226
+ """Self-Hosted arguments."""
227
+
228
+ host: str = Field(description="A host URL of the target self-hosted.")
239
229
 
240
230
 
241
- class RunsOnSelfHosted(BaseRunsOn): # pragma: no cov
231
+ class OnSelfHosted(BaseRunsOn): # pragma: no cov
242
232
  """Runs-on self-hosted."""
243
233
 
244
- type: Literal[RunsOnType.SELF_HOSTED] = Field(
245
- default=RunsOnType.SELF_HOSTED
246
- )
234
+ type: Literal[RunsOn.SELF_HOSTED] = Field(default=RunsOn.SELF_HOSTED)
247
235
  args: SelfHostedArgs = Field(alias="with")
248
236
 
249
237
 
250
- class RunsOnK8s(BaseRunsOn): # pragma: no cov
251
- """Runs-on Kubernetes."""
238
+ class AzBatchArgs(BaseModel):
239
+ batch_account_name: str
240
+ batch_account_key: SecretStr
241
+ batch_account_url: str
242
+ storage_account_name: str
243
+ storage_account_key: SecretStr
244
+
245
+
246
+ class OnAzBatch(BaseRunsOn): # pragma: no cov
247
+
248
+ type: Literal[RunsOn.AZ_BATCH] = Field(default=RunsOn.AZ_BATCH)
249
+ args: AzBatchArgs = Field(alias="with")
250
+
251
+
252
+ class DockerArgs(BaseModel):
253
+ image: str = Field(
254
+ default="ubuntu-latest",
255
+ description=(
256
+ "An image that want to run like `ubuntu-22.04`, `windows-latest`, "
257
+ ", `ubuntu-24.04-arm`, or `macos-14`"
258
+ ),
259
+ )
260
+ env: DictData = Field(default_factory=dict)
261
+ volume: DictData = Field(default_factory=dict)
262
+
252
263
 
253
- type: Literal[RunsOnType.K8S] = Field(default=RunsOnType.K8S)
264
+ class OnDocker(BaseRunsOn): # pragma: no cov
265
+ """Runs-on Docker container."""
266
+
267
+ type: Literal[RunsOn.DOCKER] = Field(default=RunsOn.DOCKER)
268
+ args: DockerArgs = Field(alias="with", default_factory=DockerArgs)
254
269
 
255
270
 
256
271
  def get_discriminator_runs_on(model: dict[str, Any]) -> str:
272
+ """Get discriminator of the RunsOn models."""
257
273
  return model.get("type", "local")
258
274
 
259
275
 
260
- RunsOn = Annotated[
276
+ RunsOnModel = Annotated[
261
277
  Union[
262
- Annotated[RunsOnK8s, Tag(RunsOnType.K8S)],
263
- Annotated[RunsOnSelfHosted, Tag(RunsOnType.SELF_HOSTED)],
264
- Annotated[RunsOnLocal, Tag(RunsOnType.LOCAL)],
278
+ Annotated[OnSelfHosted, Tag(RunsOn.SELF_HOSTED)],
279
+ Annotated[OnDocker, Tag(RunsOn.DOCKER)],
280
+ Annotated[OnLocal, Tag(RunsOn.LOCAL)],
265
281
  ],
266
282
  Discriminator(get_discriminator_runs_on),
267
283
  ]
@@ -290,7 +306,6 @@ class Job(BaseModel):
290
306
  ... "name": "Some stage",
291
307
  ... "run": "print('Hello World')",
292
308
  ... },
293
- ... ...
294
309
  ... ],
295
310
  ... }
296
311
  """
@@ -305,8 +320,8 @@ class Job(BaseModel):
305
320
  default=None,
306
321
  description="A job description that can be string of markdown content.",
307
322
  )
308
- runs_on: RunsOn = Field(
309
- default_factory=RunsOnLocal,
323
+ runs_on: RunsOnModel = Field(
324
+ default_factory=OnLocal,
310
325
  description="A target node for this job to use for execution.",
311
326
  alias="runs-on",
312
327
  )
@@ -319,8 +334,8 @@ class Job(BaseModel):
319
334
  default_factory=list,
320
335
  description="A list of Stage of this job.",
321
336
  )
322
- trigger_rule: TriggerRules = Field(
323
- default=TriggerRules.all_success,
337
+ trigger_rule: Rule = Field(
338
+ default=Rule.ALL_SUCCESS,
324
339
  description=(
325
340
  "A trigger rule of tracking needed jobs if feature will use when "
326
341
  "the `raise_error` did not set from job and stage executions."
@@ -329,7 +344,7 @@ class Job(BaseModel):
329
344
  )
330
345
  needs: list[str] = Field(
331
346
  default_factory=list,
332
- description="A list of the job ID that want to run before this job.",
347
+ description="A list of the job that want to run before this job model.",
333
348
  )
334
349
  strategy: Strategy = Field(
335
350
  default_factory=Strategy,
@@ -361,7 +376,7 @@ class Job(BaseModel):
361
376
  name: str = stage.iden
362
377
  if name in rs:
363
378
  raise ValueError(
364
- "Stage name in jobs object should not be duplicate."
379
+ f"Stage name, {name!r}, should not be duplicate."
365
380
  )
366
381
  rs.append(name)
367
382
  return value
@@ -374,7 +389,9 @@ class Job(BaseModel):
374
389
  """
375
390
  # VALIDATE: Validate job id should not dynamic with params template.
376
391
  if has_template(self.id):
377
- raise ValueError("Job ID should not has any template.")
392
+ raise ValueError(
393
+ f"Job ID, {self.id!r}, should not has any template."
394
+ )
378
395
 
379
396
  return self
380
397
 
@@ -392,7 +409,7 @@ class Job(BaseModel):
392
409
  if self.extras:
393
410
  stage.extras = self.extras
394
411
  return stage
395
- raise ValueError(f"Stage ID {stage_id} does not exists")
412
+ raise ValueError(f"Stage {stage_id!r} does not exists in this job.")
396
413
 
397
414
  def check_needs(
398
415
  self,
@@ -421,27 +438,27 @@ class Job(BaseModel):
421
438
  return WAIT
422
439
  elif all("skipped" in need_exist[job] for job in need_exist):
423
440
  return SKIP
424
- elif self.trigger_rule == TriggerRules.all_done:
441
+ elif self.trigger_rule == Rule.ALL_DONE:
425
442
  return SUCCESS
426
- elif self.trigger_rule == TriggerRules.all_success:
443
+ elif self.trigger_rule == Rule.ALL_SUCCESS:
427
444
  rs = all(
428
445
  k not in need_exist[job]
429
446
  for k in ("errors", "skipped")
430
447
  for job in need_exist
431
448
  )
432
- elif self.trigger_rule == TriggerRules.all_failed:
449
+ elif self.trigger_rule == Rule.ALL_FAILED:
433
450
  rs = all("errors" in need_exist[job] for job in need_exist)
434
- elif self.trigger_rule == TriggerRules.one_success:
451
+ elif self.trigger_rule == Rule.ONE_SUCCESS:
435
452
  rs = sum(
436
453
  k not in need_exist[job]
437
454
  for k in ("errors", "skipped")
438
455
  for job in need_exist
439
456
  ) + 1 == len(self.needs)
440
- elif self.trigger_rule == TriggerRules.one_failed:
457
+ elif self.trigger_rule == Rule.ONE_FAILED:
441
458
  rs = sum("errors" in need_exist[job] for job in need_exist) == 1
442
- elif self.trigger_rule == TriggerRules.none_skipped:
459
+ elif self.trigger_rule == Rule.NONE_SKIPPED:
443
460
  rs = all("skipped" not in need_exist[job] for job in need_exist)
444
- elif self.trigger_rule == TriggerRules.none_failed:
461
+ elif self.trigger_rule == Rule.NONE_FAILED:
445
462
  rs = all("errors" not in need_exist[job] for job in need_exist)
446
463
  else: # pragma: no cov
447
464
  raise NotImplementedError(
@@ -480,8 +497,8 @@ class Job(BaseModel):
480
497
  if not isinstance(rs, bool):
481
498
  raise TypeError("Return type of condition does not be boolean")
482
499
  return not rs
483
- except Exception as err:
484
- raise JobException(f"{err.__class__.__name__}: {err}") from err
500
+ except Exception as e:
501
+ raise JobException(f"{e.__class__.__name__}: {e}") from e
485
502
 
486
503
  def set_outputs(
487
504
  self,
@@ -499,14 +516,14 @@ class Job(BaseModel):
499
516
  ... (i) output: {'strategy-01': bar, 'strategy-02': bar}
500
517
  ... (ii) to: {'jobs': {}}
501
518
 
502
- The result of the `to` variable will be;
519
+ The result of the `to` argument will be;
503
520
 
504
521
  ... (iii) to: {
505
522
  'jobs': {
506
523
  '<job-id>': {
507
524
  'strategies': {
508
525
  'strategy-01': bar,
509
- 'strategy-02': bar
526
+ 'strategy-02': bar,
510
527
  }
511
528
  }
512
529
  }
@@ -529,22 +546,27 @@ class Job(BaseModel):
529
546
  "This job do not set the ID before setting execution output."
530
547
  )
531
548
 
532
- # NOTE: If the job ID did not set, it will use index of jobs key
533
- # instead.
534
549
  _id: str = self.id or job_id
535
-
536
550
  errors: DictData = (
537
551
  {"errors": output.pop("errors", {})} if "errors" in output else {}
538
552
  )
553
+ skipping: dict[str, bool] = (
554
+ {"skipped": output.pop("skipped", False)}
555
+ if "skipped" in output
556
+ else {}
557
+ )
539
558
 
540
- if "SKIP" in output: # pragma: no cov
541
- to["jobs"][_id] = output["SKIP"]
542
- elif self.strategy.is_set():
543
- to["jobs"][_id] = {"strategies": output, **errors}
559
+ if self.strategy.is_set():
560
+ to["jobs"][_id] = {"strategies": output, **skipping, **errors}
561
+ elif len(k := output.keys()) > 1: # pragma: no cov
562
+ raise JobException(
563
+ "Strategy output from execution return more than one ID while "
564
+ "this job does not set strategy."
565
+ )
544
566
  else:
545
- _output = output.get(next(iter(output), "FIRST"), {})
567
+ _output: DictData = {} if len(k) == 0 else output[list(k)[0]]
546
568
  _output.pop("matrix", {})
547
- to["jobs"][_id] = {**_output, **errors}
569
+ to["jobs"][_id] = {**_output, **skipping, **errors}
548
570
  return to
549
571
 
550
572
  def execute(
@@ -555,11 +577,15 @@ class Job(BaseModel):
555
577
  parent_run_id: str | None = None,
556
578
  result: Result | None = None,
557
579
  event: Event | None = None,
580
+ raise_error: bool = True,
558
581
  ) -> Result:
559
582
  """Job execution with passing dynamic parameters from the workflow
560
583
  execution. It will generate matrix values at the first step and run
561
584
  multithread on this metrics to the `stages` field of this job.
562
585
 
586
+ This method be execution routing for call dynamic execution function
587
+ with specific target `runs-on` value.
588
+
563
589
  :param params: An input parameters that use on job execution.
564
590
  :param run_id: (str) A job running ID.
565
591
  :param parent_run_id: (str) A parent workflow running ID.
@@ -567,6 +593,10 @@ class Job(BaseModel):
567
593
  data.
568
594
  :param event: (Event) An event manager that pass to the
569
595
  PoolThreadExecutor.
596
+ :param raise_error: (bool) A flag that all this method raise error to
597
+ the strategy execution. Default is `True`.
598
+
599
+ :raise NotImplementedError: If the `runs-on` value does not implement.
570
600
 
571
601
  :rtype: Result
572
602
  """
@@ -578,26 +608,36 @@ class Job(BaseModel):
578
608
  extras=self.extras,
579
609
  )
580
610
 
581
- if self.runs_on.type == RunsOnType.LOCAL:
611
+ result.trace.info(
612
+ f"[JOB]: Execute: {self.id!r} on {self.runs_on.type!r}"
613
+ )
614
+ if self.runs_on.type == RunsOn.LOCAL:
582
615
  return local_execute(
583
- job=self,
584
- params=params,
585
- result=result,
616
+ self,
617
+ params,
618
+ run_id=run_id,
619
+ parent_run_id=parent_run_id,
586
620
  event=event,
621
+ raise_error=raise_error,
587
622
  )
588
- elif self.runs_on.type == RunsOnType.SELF_HOSTED: # pragma: no cov
589
- pass
590
- elif self.runs_on.type == RunsOnType.K8S: # pragma: no cov
623
+ elif self.runs_on.type == RunsOn.SELF_HOSTED: # pragma: no cov
591
624
  pass
625
+ elif self.runs_on.type == RunsOn.DOCKER: # pragma: no cov
626
+ docker_execution(
627
+ self,
628
+ params,
629
+ run_id=run_id,
630
+ parent_run_id=parent_run_id,
631
+ event=event,
632
+ raise_error=raise_error,
633
+ )
592
634
 
593
635
  # pragma: no cov
594
636
  result.trace.error(
595
- f"[JOB]: Job executor does not support for runs-on type: "
596
- f"{self.runs_on.type} yet"
637
+ f"[JOB]: Execution not support runs-on: {self.runs_on.type!r} yet."
597
638
  )
598
639
  raise NotImplementedError(
599
- f"The job runs-on other type: {self.runs_on.type} does not "
600
- f"support yet."
640
+ f"Execution runs-on type: {self.runs_on.type} does not support yet."
601
641
  )
602
642
 
603
643
 
@@ -608,7 +648,7 @@ def local_execute_strategy(
608
648
  *,
609
649
  result: Result | None = None,
610
650
  event: Event | None = None,
611
- raise_error: bool | None = None,
651
+ raise_error: bool = True,
612
652
  ) -> Result:
613
653
  """Local job strategy execution with passing dynamic parameters from the
614
654
  workflow execution to strategy matrix.
@@ -619,6 +659,8 @@ def local_execute_strategy(
619
659
 
620
660
  The result of this execution will return result with strategy ID
621
661
  that generated from the `gen_id` function with an input strategy value.
662
+ For each stage that execution with this strategy metrix, it will use the
663
+ `set_outputs` method for reconstruct result context data.
622
664
 
623
665
  :raise JobException: If it has any error from `StageException` or
624
666
  `UtilException`.
@@ -642,8 +684,10 @@ def local_execute_strategy(
642
684
  context.update({"matrix": strategy, "stages": {}})
643
685
 
644
686
  if strategy:
645
- result.trace.info(f"[JOB]: Execute Strategy ID: {strategy_id}")
646
- result.trace.info(f"[JOB]: ... Matrix: {strategy_id}")
687
+ result.trace.info(f"[JOB]: Execute Strategy: {strategy_id!r}")
688
+ result.trace.info(f"[JOB]: ... matrix: {strategy!r}")
689
+ else:
690
+ result.trace.info("[JOB]: Execute Empty-Strategy")
647
691
 
648
692
  for stage in job.stages:
649
693
 
@@ -651,7 +695,7 @@ def local_execute_strategy(
651
695
  stage.extras = job.extras
652
696
 
653
697
  if stage.is_skipped(params=context):
654
- result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
698
+ result.trace.info(f"[JOB]: Skip Stage: {stage.iden!r}")
655
699
  stage.set_outputs(output={"skipped": True}, to=context)
656
700
  continue
657
701
 
@@ -661,17 +705,18 @@ def local_execute_strategy(
661
705
  "strategy execution."
662
706
  )
663
707
  return result.catch(
664
- status=FAILED,
708
+ status=CANCEL,
665
709
  context={
666
710
  strategy_id: {
667
711
  "matrix": strategy,
668
- "stages": context.pop("stages", {}),
712
+ "stages": filter_func(context.pop("stages", {})),
669
713
  "errors": JobException(error_msg).to_dict(),
670
714
  },
671
715
  },
672
716
  )
673
717
 
674
718
  try:
719
+ result.trace.info(f"[JOB]: Execute Stage: {stage.iden!r}")
675
720
  rs: Result = stage.handler_execute(
676
721
  params=context,
677
722
  run_id=result.run_id,
@@ -679,31 +724,11 @@ def local_execute_strategy(
679
724
  event=event,
680
725
  )
681
726
  stage.set_outputs(rs.context, to=context)
682
- if rs.status == FAILED:
683
- error_msg: str = (
684
- f"Job strategy was break because it has a stage, "
685
- f"{stage.iden}, failed without raise error."
686
- )
687
- return result.catch(
688
- status=FAILED,
689
- context={
690
- strategy_id: {
691
- "matrix": strategy,
692
- "stages": context.pop("stages", {}),
693
- "errors": JobException(error_msg).to_dict(),
694
- },
695
- },
696
- )
697
-
698
- except (StageException, UtilException) as err:
699
- result.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
700
- do_raise: bool = dynamic(
701
- "job_raise_error", f=raise_error, extras=job.extras
702
- )
703
- if do_raise:
727
+ except (StageException, UtilException) as e:
728
+ result.trace.error(f"[JOB]: {e.__class__.__name__}: {e}")
729
+ if raise_error:
704
730
  raise JobException(
705
- f"Stage execution error: {err.__class__.__name__}: "
706
- f"{err}"
731
+ f"Stage execution error: {e.__class__.__name__}: {e}"
707
732
  ) from None
708
733
 
709
734
  return result.catch(
@@ -711,8 +736,24 @@ def local_execute_strategy(
711
736
  context={
712
737
  strategy_id: {
713
738
  "matrix": strategy,
714
- "stages": context.pop("stages", {}),
715
- "errors": err.to_dict(),
739
+ "stages": filter_func(context.pop("stages", {})),
740
+ "errors": e.to_dict(),
741
+ },
742
+ },
743
+ )
744
+
745
+ if rs.status == FAILED:
746
+ error_msg: str = (
747
+ f"Job strategy was break because stage, {stage.iden}, "
748
+ f"failed without raise error."
749
+ )
750
+ return result.catch(
751
+ status=FAILED,
752
+ context={
753
+ strategy_id: {
754
+ "matrix": strategy,
755
+ "stages": filter_func(context.pop("stages", {})),
756
+ "errors": JobException(error_msg).to_dict(),
716
757
  },
717
758
  },
718
759
  )
@@ -734,9 +775,8 @@ def local_execute(
734
775
  *,
735
776
  run_id: str | None = None,
736
777
  parent_run_id: str | None = None,
737
- result: Result | None = None,
738
778
  event: Event | None = None,
739
- raise_error: bool | None = None,
779
+ raise_error: bool = True,
740
780
  ) -> Result:
741
781
  """Local job execution with passing dynamic parameters from the workflow
742
782
  execution or itself execution. It will generate matrix values at the first
@@ -749,21 +789,19 @@ def local_execute(
749
789
  :param params: (DictData) An input parameters that use on job execution.
750
790
  :param run_id: (str) A job running ID for this execution.
751
791
  :param parent_run_id: (str) A parent workflow running ID for this release.
752
- :param result: (Result) A result object for keeping context and status
753
- data.
754
792
  :param event: (Event) An event manager that pass to the PoolThreadExecutor.
755
793
  :param raise_error: (bool) A flag that all this method raise error to the
756
- strategy execution.
794
+ strategy execution. Default is `True`.
757
795
 
758
796
  :rtype: Result
759
797
  """
760
798
  result: Result = Result.construct_with_rs_or_id(
761
- result,
762
799
  run_id=run_id,
763
800
  parent_run_id=parent_run_id,
764
801
  id_logic=(job.id or "not-set"),
765
802
  extras=job.extras,
766
803
  )
804
+
767
805
  event: Event = Event() if event is None else event
768
806
 
769
807
  # NOTE: Normal Job execution without parallel strategy matrix. It uses
@@ -774,7 +812,7 @@ def local_execute(
774
812
 
775
813
  if event and event.is_set(): # pragma: no cov
776
814
  return result.catch(
777
- status=FAILED,
815
+ status=CANCEL,
778
816
  context={
779
817
  "errors": JobException(
780
818
  "Job strategy was canceled from event that had set "
@@ -784,15 +822,15 @@ def local_execute(
784
822
  )
785
823
 
786
824
  local_execute_strategy(
787
- job=job,
788
- strategy=strategy,
789
- params=params,
825
+ job,
826
+ strategy,
827
+ params,
790
828
  result=result,
791
829
  event=event,
792
830
  raise_error=raise_error,
793
831
  )
794
832
 
795
- return result.catch(status=result.status)
833
+ return result
796
834
 
797
835
  fail_fast_flag: bool = job.strategy.fail_fast
798
836
  ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
@@ -803,7 +841,7 @@ def local_execute(
803
841
 
804
842
  if event and event.is_set(): # pragma: no cov
805
843
  return result.catch(
806
- status=FAILED,
844
+ status=CANCEL,
807
845
  context={
808
846
  "errors": JobException(
809
847
  "Job strategy was canceled from event that had set "
@@ -842,7 +880,7 @@ def local_execute(
842
880
 
843
881
  if len(done) != len(futures):
844
882
  result.trace.warning(
845
- "[JOB]: Set the event for stop running stage."
883
+ "[JOB]: Set event for stop pending stage future."
846
884
  )
847
885
  event.set()
848
886
  for future in not_done:
@@ -851,18 +889,17 @@ def local_execute(
851
889
  nd: str = (
852
890
  f", the strategies do not run is {not_done}" if not_done else ""
853
891
  )
854
- result.trace.debug(f"[JOB]: Strategy is set Fail Fast{nd}")
892
+ result.trace.debug(f"[JOB]: Strategy set Fail-Fast{nd}")
855
893
 
856
894
  for future in done:
857
895
  try:
858
896
  future.result()
859
- except JobException as err:
897
+ except JobException as e:
860
898
  status = FAILED
861
899
  result.trace.error(
862
- f"[JOB]: {ls} Catch:\n\t{err.__class__.__name__}:"
863
- f"\n\t{err}"
900
+ f"[JOB]: {ls} Catch:\n\t{e.__class__.__name__}:\n\t{e}"
864
901
  )
865
- context.update({"errors": err.to_dict()})
902
+ context.update({"errors": e.to_dict()})
866
903
 
867
904
  return result.catch(status=status, context=context)
868
905
 
@@ -873,9 +910,8 @@ def self_hosted_execute(
873
910
  *,
874
911
  run_id: str | None = None,
875
912
  parent_run_id: str | None = None,
876
- result: Result | None = None,
877
913
  event: Event | None = None,
878
- raise_error: bool | None = None,
914
+ raise_error: bool = True,
879
915
  ) -> Result: # pragma: no cov
880
916
  """Self-Hosted job execution with passing dynamic parameters from the
881
917
  workflow execution or itself execution. It will make request to the
@@ -885,8 +921,6 @@ def self_hosted_execute(
885
921
  :param params: (DictData) An input parameters that use on job execution.
886
922
  :param run_id: (str) A job running ID for this execution.
887
923
  :param parent_run_id: (str) A parent workflow running ID for this release.
888
- :param result: (Result) A result object for keeping context and status
889
- data.
890
924
  :param event: (Event) An event manager that pass to the PoolThreadExecutor.
891
925
  :param raise_error: (bool) A flag that all this method raise error to the
892
926
  strategy execution.
@@ -894,7 +928,6 @@ def self_hosted_execute(
894
928
  :rtype: Result
895
929
  """
896
930
  result: Result = Result.construct_with_rs_or_id(
897
- result,
898
931
  run_id=run_id,
899
932
  parent_run_id=parent_run_id,
900
933
  id_logic=(job.id or "not-set"),
@@ -903,7 +936,7 @@ def self_hosted_execute(
903
936
 
904
937
  if event and event.is_set():
905
938
  return result.catch(
906
- status=FAILED,
939
+ status=CANCEL,
907
940
  context={
908
941
  "errors": JobException(
909
942
  "Job self-hosted execution was canceled from event that "
@@ -929,10 +962,7 @@ def self_hosted_execute(
929
962
  return result.catch(status=FAILED, context={"errors": to_dict(e)})
930
963
 
931
964
  if resp.status_code != 200:
932
- do_raise: bool = dynamic(
933
- "job_raise_error", f=raise_error, extras=job.extras
934
- )
935
- if do_raise:
965
+ if raise_error:
936
966
  raise JobException(
937
967
  f"Job execution error from request to self-hosted: "
938
968
  f"{job.runs_on.args.host!r}"
@@ -940,3 +970,97 @@ def self_hosted_execute(
940
970
 
941
971
  return result.catch(status=FAILED)
942
972
  return result.catch(status=SUCCESS)
973
+
974
+
975
+ def azure_batch_execute(
976
+ job: Job,
977
+ params: DictData,
978
+ *,
979
+ run_id: str | None = None,
980
+ parent_run_id: str | None = None,
981
+ event: Event | None = None,
982
+ raise_error: bool | None = None,
983
+ ) -> Result: # pragma no cov
984
+ """Azure Batch job execution that will run all job's stages on the Azure
985
+ Batch Node and extract the result file to be returning context result.
986
+
987
+ Steps:
988
+ - Create a Batch account and a Batch pool.
989
+ - Create a Batch job and add tasks to the job. Each task represents a
990
+ command to run on a compute node.
991
+ - Specify the command to run the Python script in the task. You can use
992
+ the cmd /c command to run the script with the Python interpreter.
993
+ - Upload the Python script and any required input files to Azure Storage
994
+ Account.
995
+ - Configure the task to download the input files from Azure Storage to
996
+ the compute node before running the script.
997
+ - Monitor the job and retrieve the output files from Azure Storage.
998
+
999
+ References:
1000
+ - https://docs.azure.cn/en-us/batch/tutorial-parallel-python
1001
+
1002
+ :param job:
1003
+ :param params:
1004
+ :param run_id:
1005
+ :param parent_run_id:
1006
+ :param event:
1007
+ :param raise_error:
1008
+
1009
+ :rtype: Result
1010
+ """
1011
+ result: Result = Result.construct_with_rs_or_id(
1012
+ run_id=run_id,
1013
+ parent_run_id=parent_run_id,
1014
+ id_logic=(job.id or "not-set"),
1015
+ extras=job.extras,
1016
+ )
1017
+ if event and event.is_set():
1018
+ return result.catch(
1019
+ status=CANCEL,
1020
+ context={
1021
+ "errors": JobException(
1022
+ "Job azure-batch execution was canceled from event that "
1023
+ "had set before start execution."
1024
+ ).to_dict()
1025
+ },
1026
+ )
1027
+ print(params)
1028
+ print(raise_error)
1029
+ return result.catch(status=SUCCESS)
1030
+
1031
+
1032
+ def docker_execution(
1033
+ job: Job,
1034
+ params: DictData,
1035
+ *,
1036
+ run_id: str | None = None,
1037
+ parent_run_id: str | None = None,
1038
+ event: Event | None = None,
1039
+ raise_error: bool | None = None,
1040
+ ):
1041
+ """Docker job execution.
1042
+
1043
+ Steps:
1044
+ - Pull the image
1045
+ - Install this workflow package
1046
+ - Start push job to run to target Docker container.
1047
+ """
1048
+ result: Result = Result.construct_with_rs_or_id(
1049
+ run_id=run_id,
1050
+ parent_run_id=parent_run_id,
1051
+ id_logic=(job.id or "not-set"),
1052
+ extras=job.extras,
1053
+ )
1054
+ if event and event.is_set():
1055
+ return result.catch(
1056
+ status=CANCEL,
1057
+ context={
1058
+ "errors": JobException(
1059
+ "Job Docker execution was canceled from event that "
1060
+ "had set before start execution."
1061
+ ).to_dict()
1062
+ },
1063
+ )
1064
+ print(params)
1065
+ print(raise_error)
1066
+ return result.catch(status=SUCCESS)