ddeutil-workflow 0.0.64__py3-none-any.whl → 0.0.66__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,9 +8,6 @@ ReleaseQueue, and Workflow models.
8
8
 
9
9
  This package implement timeout strategy on the workflow execution layer only
10
10
  because the main propose of this package is using Workflow to be orchestrator.
11
-
12
- ReleaseQueue is the memory storage of Release for tracking this release
13
- already run or pending in the current session.
14
11
  """
15
12
  from __future__ import annotations
16
13
 
@@ -21,288 +18,57 @@ from concurrent.futures import (
21
18
  ThreadPoolExecutor,
22
19
  as_completed,
23
20
  )
24
- from dataclasses import field
25
- from datetime import datetime, timedelta
21
+ from datetime import datetime
26
22
  from enum import Enum
27
- from functools import partial, total_ordering
28
- from heapq import heappop, heappush
29
23
  from pathlib import Path
30
24
  from queue import Queue
31
25
  from textwrap import dedent
32
- from threading import Event, Lock
33
- from typing import Any, Optional, Union
26
+ from threading import Event
27
+ from typing import Any, Optional
34
28
  from zoneinfo import ZoneInfo
35
29
 
36
- from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
37
- from pydantic.dataclasses import dataclass
30
+ from pydantic import BaseModel, Field, ValidationInfo
38
31
  from pydantic.functional_validators import field_validator, model_validator
39
32
  from typing_extensions import Self
40
33
 
41
- from .__cron import CronRunner
34
+ from . import get_status_from_error
42
35
  from .__types import DictData
43
36
  from .conf import FileLoad, Loader, dynamic
37
+ from .errors import WorkflowCancelError, WorkflowError, WorkflowTimeoutError
44
38
  from .event import Crontab
45
- from .exceptions import WorkflowException
46
39
  from .job import Job
47
40
  from .logs import Audit, get_audit
48
41
  from .params import Param
49
- from .result import CANCEL, FAILED, SKIP, SUCCESS, WAIT, Result
42
+ from .result import (
43
+ CANCEL,
44
+ FAILED,
45
+ SKIP,
46
+ SUCCESS,
47
+ WAIT,
48
+ Result,
49
+ Status,
50
+ validate_statuses,
51
+ )
50
52
  from .reusables import has_template, param2template
51
53
  from .utils import (
52
- clear_tz,
53
54
  gen_id,
54
- get_dt_now,
55
- reach_next_minute,
56
55
  replace_sec,
57
- wait_until_next_minute,
58
56
  )
59
57
 
60
58
 
61
59
  class ReleaseType(str, Enum):
62
- """Release Type Enum support the type field on the Release dataclass."""
60
+ """Release Type Enum."""
63
61
 
64
- DEFAULT = "manual"
65
- SCHEDULE = "schedule"
66
- POKING = "poking"
62
+ NORMAL = "normal"
63
+ RERUN = "rerun"
64
+ EVENT = "event"
67
65
  FORCE = "force"
68
66
 
69
67
 
70
- @total_ordering
71
- @dataclass(config=ConfigDict(use_enum_values=True))
72
- class Release:
73
- """Release object that use for represent the release datetime."""
74
-
75
- date: datetime = Field(
76
- description=(
77
- "A release date that should has second and millisecond equal 0."
78
- )
79
- )
80
- type: ReleaseType = Field(
81
- default=ReleaseType.DEFAULT,
82
- description="A type of release that create before start execution.",
83
- )
84
-
85
- def __repr__(self) -> str:
86
- """Override __repr__ method for represent value of `date` field.
87
-
88
- :rtype: str
89
- """
90
- return repr(f"{self.date:%Y-%m-%d %H:%M:%S}")
91
-
92
- def __str__(self) -> str:
93
- """Override string value of this release object with the `date` field.
94
-
95
- :rtype: str
96
- """
97
- return f"{self.date:%Y-%m-%d %H:%M:%S}"
98
-
99
- @classmethod
100
- def from_dt(cls, dt: Union[datetime, str]) -> Self:
101
- """Construct Release object from `datetime` or `str` objects.
102
-
103
- This method will replace second and millisecond value to 0 and
104
- replace timezone to the `tz` config setting or extras overriding before
105
- create Release object.
106
-
107
- :param dt: (Union[datetime, str]) A datetime object or string that want to
108
- construct to the Release object.
109
-
110
- :raise TypeError: If the type of the dt argument does not valid with
111
- datetime or str object.
112
-
113
- :rtype: Release
114
- """
115
- if isinstance(dt, str):
116
- dt: datetime = datetime.fromisoformat(dt)
117
- elif not isinstance(dt, datetime):
118
- raise TypeError(
119
- f"The `from_dt` need the `dt` parameter type be `str` or "
120
- f"`datetime` only, not {type(dt)}."
121
- )
122
- return cls(date=replace_sec(dt.replace(tzinfo=None)))
123
-
124
- def __eq__(self, other: Union[Release, datetime]) -> bool:
125
- """Override equal property that will compare only the same type or
126
- datetime.
127
-
128
- :rtype: bool
129
- """
130
- if isinstance(other, self.__class__):
131
- return self.date == other.date
132
- elif isinstance(other, datetime):
133
- return self.date == other
134
- return NotImplemented
135
-
136
- def __lt__(self, other: Union[Release, datetime]) -> bool:
137
- """Override less-than property that will compare only the same type or
138
- datetime.
139
-
140
- :rtype: bool
141
- """
142
- if isinstance(other, self.__class__):
143
- return self.date < other.date
144
- elif isinstance(other, datetime):
145
- return self.date < other
146
- return NotImplemented
147
-
148
-
149
- class ReleaseQueue:
150
- """ReleaseQueue object that is storage management of Release objects on
151
- the memory with list object.
152
- """
153
-
154
- def __init__(
155
- self,
156
- queue: Optional[list[Release]] = None,
157
- running: Optional[list[Release]] = None,
158
- complete: Optional[list[Release]] = None,
159
- extras: Optional[DictData] = None,
160
- ):
161
- self.queue: list[Release] = queue or []
162
- self.running: list[Release] = running or []
163
- self.complete: list[Release] = complete or []
164
- self.extras: DictData = extras or {}
165
- self.lock: Lock = Lock()
166
-
167
- @classmethod
168
- def from_list(
169
- cls,
170
- queue: Optional[Union[list[datetime], list[Release]]] = None,
171
- ) -> Self:
172
- """Construct ReleaseQueue object from an input queue value that passing
173
- with list of datetime or list of Release.
174
-
175
- :param queue: A queue object for create ReleaseQueue instance.
176
-
177
- :raise TypeError: If the type of input queue does not valid.
178
-
179
- :rtype: ReleaseQueue
180
- """
181
- if queue is None:
182
- return cls()
183
-
184
- if isinstance(queue, list):
185
- if all(isinstance(q, datetime) for q in queue):
186
- return cls(queue=[Release.from_dt(q) for q in queue])
187
-
188
- if all(isinstance(q, Release) for q in queue):
189
- return cls(queue=queue)
190
-
191
- raise TypeError(
192
- "Type of the queue does not valid with ReleaseQueue "
193
- "or list of datetime or list of Release."
194
- )
195
-
196
- @property
197
- def is_queued(self) -> bool:
198
- """Return True if it has workflow release object in the queue.
199
-
200
- :rtype: bool
201
- """
202
- return len(self.queue) > 0
203
-
204
- def check_queue(self, value: Union[Release, datetime]) -> bool:
205
- """Check a Release value already exists in list of tracking
206
- queues.
207
-
208
- :param value: A Release object that want to check it already in
209
- queues.
210
-
211
- :rtype: bool
212
- """
213
- if isinstance(value, datetime):
214
- value = Release.from_dt(value)
215
-
216
- with self.lock:
217
- return (
218
- (value in self.queue)
219
- or (value in self.running)
220
- or (value in self.complete)
221
- )
222
-
223
- def mark_complete(self, value: Release) -> Self:
224
- """Push Release to the complete queue. After push the release, it will
225
- delete old release base on the `CORE_MAX_QUEUE_COMPLETE_HIST` value.
226
-
227
- :param value: (Release) A Release value that want to push to the
228
- complete field.
229
-
230
- :rtype: Self
231
- """
232
- with self.lock:
233
- if value in self.running:
234
- self.running.remove(value)
235
-
236
- heappush(self.complete, value)
237
-
238
- # NOTE: Remove complete queue on workflow that keep more than the
239
- # maximum config value.
240
- num_complete_delete: int = len(self.complete) - dynamic(
241
- "max_queue_complete_hist", extras=self.extras
242
- )
243
-
244
- if num_complete_delete > 0:
245
- for _ in range(num_complete_delete):
246
- heappop(self.complete)
247
-
248
- return self
249
-
250
- def gen(
251
- self,
252
- end_date: datetime,
253
- audit: type[Audit],
254
- runner: CronRunner,
255
- name: str,
256
- *,
257
- force_run: bool = False,
258
- extras: Optional[DictData] = None,
259
- ) -> Self:
260
- """Generate a Release model to the queue field with an input CronRunner.
261
-
262
- Steps:
263
- - Create Release object from the current date that not reach the end
264
- date.
265
- - Check this release do not store on the release queue object.
266
- Generate the next date if it exists.
267
- - Push this release to the release queue
268
-
269
- :param end_date: (datetime) An end datetime object.
270
- :param audit: (type[Audit]) An audit class that want to make audit
271
- instance.
272
- :param runner: (CronRunner) A `CronRunner` object.
273
- :param name: (str) A target name that want to check at pointer of audit.
274
- :param force_run: (bool) A flag that allow to release workflow if the
275
- audit with that release was pointed. (Default is False).
276
- :param extras: (DictDatA) An extra parameter that want to override core
277
- config values.
278
-
279
- :rtype: ReleaseQueue
280
-
281
- """
282
- if clear_tz(runner.date) > clear_tz(end_date):
283
- return self
284
-
285
- release = Release(
286
- date=clear_tz(runner.date),
287
- type=(ReleaseType.FORCE if force_run else ReleaseType.POKING),
288
- )
289
-
290
- while self.check_queue(release) or (
291
- audit.is_pointed(name=name, release=release.date, extras=extras)
292
- and not force_run
293
- ):
294
- release = Release(
295
- date=clear_tz(runner.next),
296
- type=(ReleaseType.FORCE if force_run else ReleaseType.POKING),
297
- )
298
-
299
- if clear_tz(runner.date) > clear_tz(end_date):
300
- return self
301
-
302
- with self.lock:
303
- heappush(self.queue, release)
304
-
305
- return self
68
+ NORMAL = ReleaseType.NORMAL
69
+ RERUN = ReleaseType.RERUN
70
+ EVENT = ReleaseType.EVENT
71
+ FORCE = ReleaseType.FORCE
306
72
 
307
73
 
308
74
  class Workflow(BaseModel):
@@ -484,8 +250,9 @@ class Workflow(BaseModel):
484
250
  def __validate_jobs_need__(self) -> Self:
485
251
  """Validate each need job in any jobs should exist.
486
252
 
487
- :raise WorkflowException: If it has not exists need value in this
253
+ :raise WorkflowError: If it has not exists need value in this
488
254
  workflow job.
255
+ :raise ValueError: If the workflow name has template value.
489
256
 
490
257
  :rtype: Self
491
258
  """
@@ -493,11 +260,12 @@ class Workflow(BaseModel):
493
260
  if not_exist := [
494
261
  need for need in self.jobs[job].needs if need not in self.jobs
495
262
  ]:
496
- raise WorkflowException(
263
+ raise WorkflowError(
497
264
  f"The needed jobs: {not_exist} do not found in "
498
265
  f"{self.name!r}."
499
266
  )
500
267
 
268
+ # NOTE: Set job ID to the job model.
501
269
  self.jobs[job].id = job
502
270
 
503
271
  # VALIDATE: Validate workflow name should not dynamic with params
@@ -547,7 +315,7 @@ class Workflow(BaseModel):
547
315
  :param params: (DictData) A parameter data that receive from workflow
548
316
  execute method.
549
317
 
550
- :raise WorkflowException: If parameter value that want to validate does
318
+ :raise WorkflowError: If parameter value that want to validate does
551
319
  not include the necessary parameter that had required flag.
552
320
 
553
321
  :rtype: DictData
@@ -561,7 +329,7 @@ class Workflow(BaseModel):
561
329
  if (k not in params and self.params[k].required)
562
330
  ]
563
331
  if check_key:
564
- raise WorkflowException(
332
+ raise WorkflowError(
565
333
  f"Required Param on this workflow setting does not set: "
566
334
  f"{', '.join(check_key)}."
567
335
  )
@@ -579,18 +347,39 @@ class Workflow(BaseModel):
579
347
  "jobs": {},
580
348
  }
581
349
 
350
+ def validate_release(self, dt: datetime) -> datetime:
351
+ """Validate the release datetime that should was replaced second and
352
+ millisecond to 0 and replaced timezone to None before checking it match
353
+ with the set `on` field.
354
+
355
+ :param dt: (datetime) A datetime object that want to validate.
356
+
357
+ :rtype: datetime
358
+ """
359
+ release: datetime = replace_sec(dt.replace(tzinfo=None))
360
+ if not self.on:
361
+ return release
362
+
363
+ for on in self.on:
364
+ if release == on.cronjob.schedule(release).next:
365
+ return release
366
+ raise WorkflowError(
367
+ "Release datetime does not support for this workflow"
368
+ )
369
+
582
370
  def release(
583
371
  self,
584
- release: Union[Release, datetime],
372
+ release: datetime,
585
373
  params: DictData,
586
374
  *,
375
+ release_type: ReleaseType = NORMAL,
587
376
  run_id: Optional[str] = None,
588
377
  parent_run_id: Optional[str] = None,
589
378
  audit: type[Audit] = None,
590
- queue: Optional[ReleaseQueue] = None,
591
379
  override_log_name: Optional[str] = None,
592
380
  result: Optional[Result] = None,
593
381
  timeout: int = 600,
382
+ excluded: Optional[list[str]] = None,
594
383
  ) -> Result:
595
384
  """Release the workflow which is executes workflow with writing audit
596
385
  log tracking. The method is overriding parameter with the release
@@ -605,23 +394,20 @@ class Workflow(BaseModel):
605
394
  - Create release data for pass to parameter templating function.
606
395
  - Execute this workflow with mapping release data to its parameters.
607
396
  - Writing result audit
608
- - Remove this release on the running queue
609
- - Push this release to complete queue
610
397
 
611
- :param release: A release datetime or Release object.
398
+ :param release: (datetime) A release datetime.
612
399
  :param params: A workflow parameter that pass to execute method.
400
+ :param release_type:
613
401
  :param run_id: (str) A workflow running ID.
614
402
  :param parent_run_id: (str) A parent workflow running ID.
615
403
  :param audit: An audit class that want to save the execution result.
616
- :param queue: (ReleaseQueue) A ReleaseQueue object.
617
404
  :param override_log_name: (str) An override logging name that use
618
405
  instead the workflow name.
619
406
  :param result: (Result) A result object for keeping context and status
620
407
  data.
621
408
  :param timeout: (int) A workflow execution time out in second unit.
622
-
623
- :raise TypeError: If a queue parameter does not match with ReleaseQueue
624
- type.
409
+ :param excluded: (list[str]) A list of key that want to exclude from
410
+ audit data.
625
411
 
626
412
  :rtype: Result
627
413
  """
@@ -634,26 +420,16 @@ class Workflow(BaseModel):
634
420
  id_logic=name,
635
421
  extras=self.extras,
636
422
  )
637
-
638
- # VALIDATE: check type of queue that valid with ReleaseQueue.
639
- if queue is not None and not isinstance(queue, ReleaseQueue):
640
- raise TypeError(
641
- "The queue argument should be ReleaseQueue object only."
642
- )
643
-
644
- # VALIDATE: Change release value to Release object.
645
- if isinstance(release, datetime):
646
- release: Release = Release.from_dt(release)
647
-
423
+ release: datetime = self.validate_release(dt=release)
648
424
  result.trace.info(
649
- f"[RELEASE]: Start {name!r} : {release.date:%Y-%m-%d %H:%M:%S}"
425
+ f"[RELEASE]: Start {name!r} : {release:%Y-%m-%d %H:%M:%S}"
650
426
  )
651
427
  tz: ZoneInfo = dynamic("tz", extras=self.extras)
652
428
  values: DictData = param2template(
653
429
  params,
654
430
  params={
655
431
  "release": {
656
- "logical_date": release.date,
432
+ "logical_date": release,
657
433
  "execute_date": datetime.now(tz=tz),
658
434
  "run_id": result.run_id,
659
435
  }
@@ -662,37 +438,33 @@ class Workflow(BaseModel):
662
438
  )
663
439
  rs: Result = self.execute(
664
440
  params=values,
665
- result=result,
666
441
  parent_run_id=result.run_id,
667
442
  timeout=timeout,
668
443
  )
444
+ result.catch(status=rs.status, context=rs.context)
669
445
  result.trace.info(
670
- f"[RELEASE]: End {name!r} : {release.date:%Y-%m-%d %H:%M:%S}"
446
+ f"[RELEASE]: End {name!r} : {release:%Y-%m-%d %H:%M:%S}"
671
447
  )
672
448
  result.trace.debug(f"[RELEASE]: Writing audit: {name!r}.")
673
449
  (
674
450
  audit(
675
451
  name=name,
676
- release=release.date,
677
- type=release.type,
452
+ release=release,
453
+ type=release_type,
678
454
  context=result.context,
679
455
  parent_run_id=result.parent_run_id,
680
456
  run_id=result.run_id,
681
457
  execution_time=result.alive_time(),
682
458
  extras=self.extras,
683
- ).save(excluded=None)
459
+ ).save(excluded=excluded)
684
460
  )
685
-
686
- if queue:
687
- queue.mark_complete(release)
688
-
689
461
  return result.catch(
690
462
  status=rs.status,
691
463
  context={
692
464
  "params": params,
693
465
  "release": {
694
- "type": release.type,
695
- "logical_date": release.date,
466
+ "type": release_type,
467
+ "logical_date": release,
696
468
  },
697
469
  **{"jobs": result.context.pop("jobs", {})},
698
470
  **(
@@ -710,7 +482,7 @@ class Workflow(BaseModel):
710
482
  *,
711
483
  result: Optional[Result] = None,
712
484
  event: Optional[Event] = None,
713
- ) -> Result:
485
+ ) -> tuple[Status, Result]:
714
486
  """Job execution with passing dynamic parameters from the main workflow
715
487
  execution to the target job object via job's ID.
716
488
 
@@ -718,7 +490,8 @@ class Workflow(BaseModel):
718
490
  model. It different with `self.execute` because this method run only
719
491
  one job and return with context of this job data.
720
492
 
721
- :raise WorkflowException: If the job execution raise JobException.
493
+ This method do not raise any error, and it will handle all exception
494
+ from the job execution.
722
495
 
723
496
  :param job: (Job) A job model that want to execute.
724
497
  :param params: (DictData) A parameter data.
@@ -726,22 +499,19 @@ class Workflow(BaseModel):
726
499
  :param event: (Event) An Event manager instance that use to cancel this
727
500
  execution if it forces stopped by parent execution.
728
501
 
729
- :rtype: Result
502
+ :rtype: tuple[Status, Result]
730
503
  """
731
504
  result: Result = result or Result(run_id=gen_id(self.name, unique=True))
732
505
 
733
- if job.is_skipped(params=params):
734
- result.trace.info(f"[WORKFLOW]: Skip Job: {job.id!r}")
735
- job.set_outputs(output={"skipped": True}, to=params)
736
- return result.catch(status=SKIP, context=params)
737
-
738
506
  if event and event.is_set():
739
- return result.catch(
507
+ error_msg: str = (
508
+ "Job execution was canceled because the event was set "
509
+ "before start job execution."
510
+ )
511
+ return CANCEL, result.catch(
740
512
  status=CANCEL,
741
513
  context={
742
- "errors": WorkflowException(
743
- "Workflow job was canceled because event was set."
744
- ).to_dict(),
514
+ "errors": WorkflowCancelError(error_msg).to_dict(),
745
515
  },
746
516
  )
747
517
 
@@ -753,18 +523,31 @@ class Workflow(BaseModel):
753
523
  event=event,
754
524
  )
755
525
  job.set_outputs(rs.context, to=params)
756
- if rs.status in (FAILED, CANCEL):
526
+
527
+ if rs.status == FAILED:
528
+ error_msg: str = f"Job execution, {job.id!r}, was failed."
529
+ return FAILED, result.catch(
530
+ status=FAILED,
531
+ context={
532
+ "errors": WorkflowError(error_msg).to_dict(),
533
+ **params,
534
+ },
535
+ )
536
+
537
+ elif rs.status == CANCEL:
757
538
  error_msg: str = (
758
- f"Job, {job.id!r}, return `{rs.status.name}` status."
539
+ f"Job execution, {job.id!r}, was canceled from the event after "
540
+ f"end job execution."
759
541
  )
760
- return result.catch(
761
- status=rs.status,
542
+ return CANCEL, result.catch(
543
+ status=CANCEL,
762
544
  context={
763
- "errors": WorkflowException(error_msg).to_dict(),
545
+ "errors": WorkflowCancelError(error_msg).to_dict(),
764
546
  **params,
765
547
  },
766
548
  )
767
- return result.catch(status=SUCCESS, context=params)
549
+
550
+ return rs.status, result.catch(status=rs.status, context=params)
768
551
 
769
552
  def execute(
770
553
  self,
@@ -772,7 +555,6 @@ class Workflow(BaseModel):
772
555
  *,
773
556
  run_id: Optional[str] = None,
774
557
  parent_run_id: Optional[str] = None,
775
- result: Optional[Result] = None,
776
558
  event: Optional[Event] = None,
777
559
  timeout: float = 3600,
778
560
  max_job_parallel: int = 2,
@@ -799,10 +581,30 @@ class Workflow(BaseModel):
799
581
  the job execution. It will warp that error and keep it in the key `errors`
800
582
  at the result context.
801
583
 
584
+
585
+ Execution --> Ok --> Result
586
+ |-status: CANCEL
587
+ ╰-context:
588
+ ╰-errors:
589
+ |-name: ...
590
+ ╰-message: ...
591
+
592
+ --> Ok --> Result
593
+ |-status: FAILED
594
+ ╰-context:
595
+ ╰-errors:
596
+ |-name: ...
597
+ ╰-message: ...
598
+
599
+ --> Ok --> Result
600
+ ╰-status: SKIP
601
+
602
+ --> Ok --> Result
603
+ ╰-status: SUCCESS
604
+
802
605
  :param params: A parameter data that will parameterize before execution.
803
606
  :param run_id: (Optional[str]) A workflow running ID.
804
607
  :param parent_run_id: (Optional[str]) A parent workflow running ID.
805
- :param result: (Result) A Result instance for return context and status.
806
608
  :param event: (Event) An Event manager instance that use to cancel this
807
609
  execution if it forces stopped by parent execution.
808
610
  :param timeout: (float) A workflow execution time out in second unit
@@ -810,13 +612,12 @@ class Workflow(BaseModel):
810
612
  This value does not force stop the task that still running more than
811
613
  this limit time. (Default: 60 * 60 seconds)
812
614
  :param max_job_parallel: (int) The maximum workers that use for job
813
- execution in `PoolThreadExecutor` object. (Default: 2 workers)
615
+ execution in `ThreadPoolExecutor` object. (Default: 2 workers)
814
616
 
815
617
  :rtype: Result
816
618
  """
817
619
  ts: float = time.monotonic()
818
620
  result: Result = Result.construct_with_rs_or_id(
819
- result,
820
621
  run_id=run_id,
821
622
  parent_run_id=parent_run_id,
822
623
  id_logic=self.name,
@@ -840,13 +641,26 @@ class Workflow(BaseModel):
840
641
  job_queue.put(job_id)
841
642
 
842
643
  not_timeout_flag: bool = True
644
+ total_job: int = len(self.jobs)
645
+ statuses: list[Status] = [WAIT] * total_job
646
+ skip_count: int = 0
647
+ sequence_statuses: list[Status] = []
843
648
  timeout: float = dynamic(
844
649
  "max_job_exec_timeout", f=timeout, extras=self.extras
845
650
  )
651
+ result.catch(status=WAIT, context=context)
652
+ if event and event.is_set():
653
+ return result.catch(
654
+ status=CANCEL,
655
+ context={
656
+ "errors": WorkflowCancelError(
657
+ "Execution was canceled from the event was set before "
658
+ "workflow execution."
659
+ ).to_dict(),
660
+ },
661
+ )
846
662
 
847
- with ThreadPoolExecutor(
848
- max_workers=max_job_parallel, thread_name_prefix="wf_exec_"
849
- ) as executor:
663
+ with ThreadPoolExecutor(max_job_parallel, "wf") as executor:
850
664
  futures: list[Future] = []
851
665
 
852
666
  while not job_queue.empty() and (
@@ -863,16 +677,20 @@ class Workflow(BaseModel):
863
677
  return result.catch(
864
678
  status=FAILED,
865
679
  context={
866
- "errors": WorkflowException(
680
+ "status": FAILED,
681
+ "errors": WorkflowError(
867
682
  f"Validate job trigger rule was failed with "
868
683
  f"{job.trigger_rule.value!r}."
869
- ).to_dict()
684
+ ).to_dict(),
870
685
  },
871
686
  )
872
687
  elif check == SKIP: # pragma: no cov
873
- result.trace.info(f"[JOB]: Skip job: {job_id!r}")
874
- job.set_outputs(output={"skipped": True}, to=context)
688
+ result.trace.info(
689
+ f"[JOB]: Skip job: {job_id!r} from trigger rule."
690
+ )
691
+ job.set_outputs(output={"status": SKIP}, to=context)
875
692
  job_queue.task_done()
693
+ skip_count += 1
876
694
  continue
877
695
 
878
696
  if max_job_parallel > 1:
@@ -898,17 +716,22 @@ class Workflow(BaseModel):
898
716
  event=event,
899
717
  )
900
718
  )
901
- time.sleep(0.025)
902
- elif (future := futures.pop(0)).done() or future.cancelled():
719
+ elif (future := futures.pop(0)).done():
720
+ if e := future.exception():
721
+ sequence_statuses.append(get_status_from_error(e))
722
+ else:
723
+ st, _ = future.result()
724
+ sequence_statuses.append(st)
725
+ job_queue.put(job_id)
726
+ elif future.cancelled():
727
+ sequence_statuses.append(CANCEL)
903
728
  job_queue.put(job_id)
904
729
  elif future.running() or "state=pending" in str(future):
905
- time.sleep(0.075)
906
730
  futures.insert(0, future)
907
731
  job_queue.put(job_id)
908
732
  else: # pragma: no cov
909
733
  job_queue.put(job_id)
910
734
  futures.insert(0, future)
911
- time.sleep(0.025)
912
735
  result.trace.warning(
913
736
  f"[WORKFLOW]: ... Execution non-threading not "
914
737
  f"handle: {future}."
@@ -918,351 +741,41 @@ class Workflow(BaseModel):
918
741
 
919
742
  if not_timeout_flag:
920
743
  job_queue.join()
921
- for future in as_completed(futures):
922
- future.result()
744
+ for total, future in enumerate(as_completed(futures), start=0):
745
+ try:
746
+ statuses[total], _ = future.result()
747
+ except WorkflowError as e:
748
+ statuses[total] = get_status_from_error(e)
749
+
750
+ # NOTE: Update skipped status from the job trigger.
751
+ for i in range(skip_count):
752
+ statuses[total + 1 + i] = SKIP
753
+
754
+ # NOTE: Update status from none-parallel job execution.
755
+ for i, s in enumerate(sequence_statuses, start=0):
756
+ statuses[total + 1 + skip_count + i] = s
757
+
923
758
  return result.catch(
924
- status=FAILED if "errors" in result.context else SUCCESS,
925
- context=context,
759
+ status=validate_statuses(statuses), context=context
926
760
  )
927
761
 
928
- result.trace.error(f"[WORKFLOW]: {self.name!r} was timeout.")
929
762
  event.set()
930
763
  for future in futures:
931
764
  future.cancel()
932
765
 
933
- return result.catch(
934
- status=FAILED,
935
- context={
936
- "errors": WorkflowException(
937
- f"{self.name!r} was timeout."
938
- ).to_dict()
939
- },
940
- )
941
-
942
-
943
- class WorkflowPoke(Workflow):
944
- """Workflow Poke model that was implemented the poke method."""
945
-
946
- def queue(
947
- self,
948
- offset: float,
949
- end_date: datetime,
950
- queue: ReleaseQueue,
951
- audit: type[Audit],
952
- *,
953
- force_run: bool = False,
954
- ) -> ReleaseQueue:
955
- """Generate Release from all on values from the on field and store them
956
- to the ReleaseQueue object.
957
-
958
- :param offset: An offset in second unit for time travel.
959
- :param end_date: An end datetime object.
960
- :param queue: A workflow queue object.
961
- :param audit: An audit class that want to make audit object.
962
- :param force_run: A flag that allow to release workflow if the audit
963
- with that release was pointed.
964
-
965
- :rtype: ReleaseQueue
966
- """
967
- for on in self.on:
968
-
969
- queue.gen(
970
- end_date,
971
- audit,
972
- on.next(get_dt_now(offset=offset).replace(microsecond=0)),
973
- self.name,
974
- force_run=force_run,
975
- )
976
-
977
- return queue
978
-
979
- def poke(
980
- self,
981
- params: Optional[DictData] = None,
982
- start_date: Optional[datetime] = None,
983
- *,
984
- run_id: Optional[str] = None,
985
- periods: int = 1,
986
- audit: Optional[Audit] = None,
987
- force_run: bool = False,
988
- timeout: int = 1800,
989
- max_poking_pool_worker: int = 2,
990
- ) -> Result:
991
- """Poke workflow with a start datetime value that will pass to its
992
- `on` field on the threading executor pool for execute the `release`
993
- method (It run all schedules that was set on the `on` values).
994
-
995
- This method will observe its `on` field that nearing to run with the
996
- `self.release()` method.
997
-
998
- The limitation of this method is not allow run a date that gather
999
- than the current date.
1000
-
1001
- :param params: (DictData) A parameter data.
1002
- :param start_date: (datetime) A start datetime object.
1003
- :param run_id: (str) A workflow running ID for this poke.
1004
- :param periods: (int) A periods in minutes value that use to run this
1005
- poking. (Default is 1)
1006
- :param audit: (Audit) An audit object that want to use on this poking
1007
- process.
1008
- :param force_run: (bool) A flag that allow to release workflow if the
1009
- audit with that release was pointed. (Default is False)
1010
- :param timeout: (int) A second value for timeout while waiting all
1011
- futures run completely.
1012
- :param max_poking_pool_worker: (int) The maximum poking pool worker.
1013
- (Default is 2 workers)
1014
-
1015
- :raise WorkflowException: If the periods parameter less or equal than 0.
1016
-
1017
- :rtype: Result
1018
- :return: A list of all results that return from `self.release` method.
1019
- """
1020
- audit: type[Audit] = audit or get_audit(extras=self.extras)
1021
- result: Result = Result(
1022
- run_id=(run_id or gen_id(self.name, unique=True))
1023
- )
1024
-
1025
- # VALIDATE: Check the periods value should gather than 0.
1026
- if periods <= 0:
1027
- raise WorkflowException(
1028
- "The period of poking should be `int` and grater or equal "
1029
- "than 1."
766
+ result.trace.error(
767
+ f"[WORKFLOW]: {self.name!r} was timeout because it use exec "
768
+ f"time more than {timeout} seconds."
1030
769
  )
1031
770
 
1032
- if len(self.on) == 0:
1033
- result.trace.warning(
1034
- f"[POKING]: {self.name!r} not have any schedule!!!"
1035
- )
1036
- return result.catch(status=SUCCESS, context={"outputs": []})
1037
-
1038
- # NOTE: Create the current date that change microsecond to 0
1039
- current_date: datetime = datetime.now().replace(microsecond=0)
1040
-
1041
- if start_date is None:
1042
- # NOTE: Force change start date if it gathers than the current date,
1043
- # or it does not pass to this method.
1044
- start_date: datetime = current_date
1045
- offset: float = 0
1046
- elif start_date <= current_date:
1047
- start_date = start_date.replace(microsecond=0)
1048
- offset: float = (current_date - start_date).total_seconds()
1049
- else:
1050
- raise WorkflowException(
1051
- f"The start datetime should less than or equal the current "
1052
- f"datetime, {current_date:%Y-%m-%d %H:%M:%S}."
1053
- )
1054
-
1055
- # NOTE: The end date is using to stop generate queue with an input
1056
- # periods value. It will change to MM:59.
1057
- # For example:
1058
- # (input) start_date = 12:04:12, offset = 2
1059
- # (output) end_date = 12:06:59
1060
- end_date: datetime = start_date.replace(second=0) + timedelta(
1061
- minutes=periods + 1, seconds=-1
1062
- )
1063
-
1064
- result.trace.info(
1065
- f"[POKING]: Execute Poking: {self.name!r} "
1066
- f"({start_date:%Y-%m-%d %H:%M:%S} ==> {end_date:%Y-%m-%d %H:%M:%S})"
1067
- )
1068
-
1069
- params: DictData = {} if params is None else params
1070
- context: list[Result] = []
1071
- q: ReleaseQueue = ReleaseQueue()
1072
-
1073
- # NOTE: Create reusable partial function and add Release to the release
1074
- # queue object.
1075
- partial_queue = partial(
1076
- self.queue, offset, end_date, audit=audit, force_run=force_run
1077
- )
1078
- partial_queue(q)
1079
- if not q.is_queued:
1080
- result.trace.warning(
1081
- f"[POKING]: Skip {self.name!r}, not have any queue!!!"
1082
- )
1083
- return result.catch(status=SUCCESS, context={"outputs": []})
1084
-
1085
- with ThreadPoolExecutor(
1086
- max_workers=dynamic(
1087
- "max_poking_pool_worker",
1088
- f=max_poking_pool_worker,
1089
- extras=self.extras,
1090
- ),
1091
- thread_name_prefix="wf_poking_",
1092
- ) as executor:
1093
-
1094
- futures: list[Future] = []
1095
-
1096
- while q.is_queued:
1097
-
1098
- # NOTE: Pop the latest Release object from the release queue.
1099
- release: Release = heappop(q.queue)
1100
-
1101
- if reach_next_minute(release.date, offset=offset):
1102
- result.trace.debug(
1103
- f"[POKING]: Skip Release: "
1104
- f"{release.date:%Y-%m-%d %H:%M:%S}"
1105
- )
1106
- heappush(q.queue, release)
1107
- wait_until_next_minute(get_dt_now(offset=offset))
1108
-
1109
- # WARNING: I already call queue poking again because issue
1110
- # about the every minute crontab.
1111
- partial_queue(q)
1112
- continue
1113
-
1114
- heappush(q.running, release)
1115
- futures.append(
1116
- executor.submit(
1117
- self.release,
1118
- release=release,
1119
- params=params,
1120
- audit=audit,
1121
- queue=q,
1122
- parent_run_id=result.run_id,
1123
- )
1124
- )
1125
-
1126
- partial_queue(q)
1127
-
1128
- # WARNING: This poking method does not allow to use fail-fast
1129
- # logic to catching parallel execution result.
1130
- for future in as_completed(futures, timeout=timeout):
1131
- context.append(future.result())
771
+ time.sleep(0.0025)
1132
772
 
1133
773
  return result.catch(
1134
- status=SUCCESS,
1135
- context={"outputs": context},
1136
- )
1137
-
1138
-
1139
- @dataclass(config=ConfigDict(arbitrary_types_allowed=True))
1140
- class WorkflowTask:
1141
- """Workflow task Pydantic dataclass object that use to keep mapping data and
1142
- workflow model for passing to the multithreading task.
1143
-
1144
- This dataclass object is mapping 1-to-1 with workflow and cron runner
1145
- objects.
1146
-
1147
- This dataclass has the release method for itself that prepare necessary
1148
- arguments before passing to the parent release method.
1149
-
1150
- :param alias: (str) An alias name of Workflow model.
1151
- :param workflow: (Workflow) A Workflow model instance.
1152
- :param runner: (CronRunner)
1153
- :param values: A value data that want to parameterize.
1154
- :param extras: An extra parameter that use to override core config values.
1155
- """
1156
-
1157
- alias: str
1158
- workflow: Workflow
1159
- runner: CronRunner
1160
- values: DictData = field(default_factory=dict)
1161
- extras: DictData = field(default_factory=dict)
1162
-
1163
- def release(
1164
- self,
1165
- release: Optional[Union[Release, datetime]] = None,
1166
- run_id: Optional[str] = None,
1167
- audit: type[Audit] = None,
1168
- queue: Optional[ReleaseQueue] = None,
1169
- ) -> Result:
1170
- """Release the workflow task that passing an override parameter to
1171
- the parent release method with the `values` field.
1172
-
1173
- This method can handler not passing release value by default
1174
- generate step. It uses the `runner` field for generate release object.
1175
-
1176
- :param release: A release datetime or Release object.
1177
- :param run_id: A workflow running ID for this release.
1178
- :param audit: An audit class that want to save the execution result.
1179
- :param queue: A ReleaseQueue object that use to mark complete.
1180
-
1181
- :raise ValueError: If a queue parameter does not pass while release
1182
- is None.
1183
- :raise TypeError: If a queue parameter does not match with ReleaseQueue
1184
- type.
1185
-
1186
- :rtype: Result
1187
- """
1188
- audit: type[Audit] = audit or get_audit(extras=self.extras)
1189
-
1190
- if release is None:
1191
-
1192
- if queue is None:
1193
- raise ValueError(
1194
- "If pass None release value, you should to pass the queue"
1195
- "for generate this release."
1196
- )
1197
- elif not isinstance(queue, ReleaseQueue):
1198
- raise TypeError(
1199
- "The queue argument should be ReleaseQueue object only."
1200
- )
1201
-
1202
- if queue.check_queue(self.runner.date):
1203
- release = self.runner.next
1204
-
1205
- while queue.check_queue(release):
1206
- release = self.runner.next
1207
- else:
1208
- release = self.runner.date
1209
-
1210
- return self.workflow.release(
1211
- release=release,
1212
- params=self.values,
1213
- run_id=run_id,
1214
- audit=audit,
1215
- queue=queue,
1216
- override_log_name=self.alias,
1217
- )
1218
-
1219
- def queue(
1220
- self,
1221
- end_date: datetime,
1222
- queue: ReleaseQueue,
1223
- audit: type[Audit],
1224
- *,
1225
- force_run: bool = False,
1226
- ) -> ReleaseQueue:
1227
- """Generate Release from the runner field and store it to the
1228
- ReleaseQueue object.
1229
-
1230
- :param end_date: An end datetime object.
1231
- :param queue: A workflow queue object.
1232
- :param audit: An audit class that want to make audit object.
1233
- :param force_run: (bool) A flag that allow to release workflow if the
1234
- audit with that release was pointed.
1235
-
1236
- :rtype: ReleaseQueue
1237
- """
1238
- return queue.gen(
1239
- end_date,
1240
- audit,
1241
- self.runner,
1242
- self.alias,
1243
- force_run=force_run,
1244
- extras=self.extras,
1245
- )
1246
-
1247
- def __repr__(self) -> str:
1248
- """Override the `__repr__` method.
1249
-
1250
- :rtype: str
1251
- """
1252
- return (
1253
- f"{self.__class__.__name__}(alias={self.alias!r}, "
1254
- f"workflow={self.workflow.name!r}, runner={self.runner!r}, "
1255
- f"values={self.values})"
774
+ status=FAILED,
775
+ context={
776
+ "errors": WorkflowTimeoutError(
777
+ f"{self.name!r} was timeout because it use exec time more "
778
+ f"than {timeout} seconds."
779
+ ).to_dict(),
780
+ },
1256
781
  )
1257
-
1258
- def __eq__(self, other: WorkflowTask) -> bool:
1259
- """Override the equal property that will compare only the same type.
1260
-
1261
- :rtype: bool
1262
- """
1263
- if isinstance(other, WorkflowTask):
1264
- return (
1265
- self.workflow.name == other.workflow.name
1266
- and self.runner.cron == other.runner.cron
1267
- )
1268
- return NotImplemented