ddeutil-workflow 0.0.56__py3-none-any.whl → 0.0.57__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,12 +3,14 @@
3
3
  # Licensed under the MIT License. See LICENSE in the project root for
4
4
  # license information.
5
5
  # ------------------------------------------------------------------------------
6
- # [x] Use dynamic config
7
- """Workflow module is the core module of this Workflow package. It keeps
8
- Release, ReleaseQueue, and Workflow Pydantic models.
6
+ """Workflow module is the core module of this package. It keeps Release,
7
+ ReleaseQueue, and Workflow models.
9
8
 
10
9
  This package implement timeout strategy on the workflow execution layer only
11
10
  because the main propose of this package is using Workflow to be orchestrator.
11
+
12
+ ReleaseQueue is the memory storage of Release for tracking this release
13
+ already run or pending in the current session.
12
14
  """
13
15
  from __future__ import annotations
14
16
 
@@ -27,30 +29,32 @@ from heapq import heappop, heappush
27
29
  from pathlib import Path
28
30
  from queue import Queue
29
31
  from textwrap import dedent
30
- from threading import Event
32
+ from threading import Event, Lock
31
33
  from typing import Any, Optional, Union
34
+ from zoneinfo import ZoneInfo
32
35
 
33
36
  from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
34
37
  from pydantic.dataclasses import dataclass
35
38
  from pydantic.functional_validators import field_validator, model_validator
36
39
  from typing_extensions import Self
37
40
 
38
- from .__cron import CronJob, CronRunner
41
+ from .__cron import CronRunner
39
42
  from .__types import DictData, TupleStr
40
43
  from .conf import FileLoad, Loader, dynamic
41
44
  from .event import On
42
- from .exceptions import JobException, UtilException, WorkflowException
45
+ from .exceptions import WorkflowException
43
46
  from .job import Job
44
47
  from .logs import Audit, get_audit
45
48
  from .params import Param
46
- from .result import FAILED, SKIP, SUCCESS, WAIT, Result
49
+ from .result import CANCEL, FAILED, SKIP, SUCCESS, WAIT, Result
47
50
  from .reusables import has_template, param2template
48
51
  from .utils import (
49
- NEWLINE,
52
+ clear_tz,
50
53
  gen_id,
51
54
  get_dt_now,
52
55
  reach_next_minute,
53
- wait_to_next_minute,
56
+ replace_sec,
57
+ wait_until_next_minute,
54
58
  )
55
59
 
56
60
  __all__: TupleStr = (
@@ -66,46 +70,50 @@ class ReleaseType(str, Enum):
66
70
  """Release Type Enum support the type field on the Release dataclass."""
67
71
 
68
72
  DEFAULT: str = "manual"
69
- TASK: str = "task"
70
- POKE: str = "poking"
73
+ SCHEDULE: str = "schedule"
74
+ POKING: str = "poking"
75
+ FORCE: str = "force"
71
76
 
72
77
 
73
78
  @total_ordering
74
- @dataclass(
75
- config=ConfigDict(arbitrary_types_allowed=True, use_enum_values=True)
76
- )
79
+ @dataclass(config=ConfigDict(use_enum_values=True))
77
80
  class Release:
78
- """Release Pydantic dataclass object that use for represent the release data
79
- that use with the `workflow.release` method.
80
- """
81
+ """Release object that use for represent the release datetime."""
81
82
 
82
- date: datetime
83
- offset: float
84
- end_date: datetime
85
- runner: CronRunner
86
- type: ReleaseType = field(default=ReleaseType.DEFAULT)
83
+ date: datetime = Field(
84
+ description=(
85
+ "A release date that should has second and millisecond equal 0."
86
+ )
87
+ )
88
+ type: ReleaseType = Field(
89
+ default=ReleaseType.DEFAULT,
90
+ description="A type of release that create before start execution.",
91
+ )
87
92
 
88
93
  def __repr__(self) -> str:
89
- """Represent string"""
94
+ """Override __repr__ method for represent value of `date` field.
95
+
96
+ :rtype: str
97
+ """
90
98
  return repr(f"{self.date:%Y-%m-%d %H:%M:%S}")
91
99
 
92
100
  def __str__(self) -> str:
93
- """Override string value of this release object with the date field.
101
+ """Override string value of this release object with the `date` field.
94
102
 
95
103
  :rtype: str
96
104
  """
97
105
  return f"{self.date:%Y-%m-%d %H:%M:%S}"
98
106
 
99
107
  @classmethod
100
- def from_dt(
101
- cls, dt: datetime | str, *, extras: Optional[DictData] = None
102
- ) -> Self:
103
- """Construct Release via datetime object only.
108
+ def from_dt(cls, dt: datetime | str) -> Self:
109
+ """Construct Release object from `datetime` or `str` objects.
110
+
111
+ This method will replace second and millisecond value to 0 and
112
+ replace timezone to the `tz` config setting or extras overriding before
113
+ create Release object.
104
114
 
105
115
  :param dt: (datetime | str) A datetime object or string that want to
106
116
  construct to the Release object.
107
- :param extras: (DictData) An extra parameters that want to pass to
108
- override config values.
109
117
 
110
118
  :raise TypeError: If the type of the dt argument does not valid with
111
119
  datetime or str object.
@@ -116,20 +124,10 @@ class Release:
116
124
  dt: datetime = datetime.fromisoformat(dt)
117
125
  elif not isinstance(dt, datetime):
118
126
  raise TypeError(
119
- "The `from_dt` need the `dt` argument type be str or datetime "
120
- "only."
127
+ f"The `from_dt` need the `dt` parameter type be `str` or "
128
+ f"`datetime` only, not {type(dt)}."
121
129
  )
122
-
123
- return cls(
124
- date=dt,
125
- offset=0,
126
- end_date=dt + timedelta(days=1),
127
- runner=(
128
- CronJob("* * * * *").schedule(
129
- dt.replace(tzinfo=dynamic("tz", extras=extras))
130
- )
131
- ),
132
- )
130
+ return cls(date=replace_sec(dt.replace(tzinfo=None)))
133
131
 
134
132
  def __eq__(self, other: Release | datetime) -> bool:
135
133
  """Override equal property that will compare only the same type or
@@ -156,31 +154,33 @@ class Release:
156
154
  return NotImplemented
157
155
 
158
156
 
159
- @dataclass
160
157
  class ReleaseQueue:
161
- """Workflow Queue object that is management of Release objects."""
158
+ """ReleaseQueue object that is storage management of Release objects on
159
+ the memory with list object.
160
+ """
162
161
 
163
- queue: list[Release] = field(default_factory=list)
164
- running: list[Release] = field(default_factory=list)
165
- complete: list[Release] = field(default_factory=list)
166
- extras: DictData = Field(
167
- default_factory=dict,
168
- description="An extra parameters that want to override config values.",
169
- repr=False,
170
- )
162
+ def __init__(
163
+ self,
164
+ queue: Optional[list[Release]] = None,
165
+ running: Optional[list[Release]] = None,
166
+ complete: Optional[list[Release]] = None,
167
+ extras: Optional[DictData] = None,
168
+ ):
169
+ self.queue: list[Release] = queue or []
170
+ self.running: list[Release] = running or []
171
+ self.complete: list[Release] = complete or []
172
+ self.extras: DictData = extras or {}
173
+ self.lock: Lock = Lock()
171
174
 
172
175
  @classmethod
173
176
  def from_list(
174
177
  cls,
175
178
  queue: Optional[Union[list[datetime], list[Release]]] = None,
176
- extras: Optional[DictData] = None,
177
179
  ) -> Self:
178
180
  """Construct ReleaseQueue object from an input queue value that passing
179
181
  with list of datetime or list of Release.
180
182
 
181
- :param queue:
182
- :param extras: An extra parameter that want to override core config
183
- values.
183
+ :param queue: A queue object for create ReleaseQueue instance.
184
184
 
185
185
  :raise TypeError: If the type of input queue does not valid.
186
186
 
@@ -191,11 +191,7 @@ class ReleaseQueue:
191
191
 
192
192
  if isinstance(queue, list):
193
193
  if all(isinstance(q, datetime) for q in queue):
194
- return cls(
195
- queue=[
196
- Release.from_dt(q, extras=(extras or {})) for q in queue
197
- ]
198
- )
194
+ return cls(queue=[Release.from_dt(q) for q in queue])
199
195
 
200
196
  if all(isinstance(q, Release) for q in queue):
201
197
  return cls(queue=queue)
@@ -223,13 +219,14 @@ class ReleaseQueue:
223
219
  :rtype: bool
224
220
  """
225
221
  if isinstance(value, datetime):
226
- value = Release.from_dt(value, extras=self.extras)
222
+ value = Release.from_dt(value)
227
223
 
228
- return (
229
- (value in self.queue)
230
- or (value in self.running)
231
- or (value in self.complete)
232
- )
224
+ with self.lock:
225
+ return (
226
+ (value in self.queue)
227
+ or (value in self.running)
228
+ or (value in self.complete)
229
+ )
233
230
 
234
231
  def mark_complete(self, value: Release) -> Self:
235
232
  """Push Release to the complete queue. After push the release, it will
@@ -240,17 +237,21 @@ class ReleaseQueue:
240
237
 
241
238
  :rtype: Self
242
239
  """
243
- heappush(self.complete, value)
240
+ with self.lock:
241
+ if value in self.running:
242
+ self.running.remove(value)
244
243
 
245
- # NOTE: Remove complete queue on workflow that keep more than the
246
- # maximum config value.
247
- num_complete_delete: int = len(self.complete) - dynamic(
248
- "max_queue_complete_hist", extras=self.extras
249
- )
244
+ heappush(self.complete, value)
250
245
 
251
- if num_complete_delete > 0:
252
- for _ in range(num_complete_delete):
253
- heappop(self.complete)
246
+ # NOTE: Remove complete queue on workflow that keep more than the
247
+ # maximum config value.
248
+ num_complete_delete: int = len(self.complete) - dynamic(
249
+ "max_queue_complete_hist", extras=self.extras
250
+ )
251
+
252
+ if num_complete_delete > 0:
253
+ for _ in range(num_complete_delete):
254
+ heappop(self.complete)
254
255
 
255
256
  return self
256
257
 
@@ -261,7 +262,6 @@ class ReleaseQueue:
261
262
  runner: CronRunner,
262
263
  name: str,
263
264
  *,
264
- offset: float = 0,
265
265
  force_run: bool = False,
266
266
  extras: Optional[DictData] = None,
267
267
  ) -> Self:
@@ -277,9 +277,8 @@ class ReleaseQueue:
277
277
  :param end_date: (datetime) An end datetime object.
278
278
  :param audit: (type[Audit]) An audit class that want to make audit
279
279
  instance.
280
- :param runner: (CronRunner) A CronRunner object.
280
+ :param runner: (CronRunner) A `CronRunner` object.
281
281
  :param name: (str) A target name that want to check at pointer of audit.
282
- :param offset: (float) An offset in second unit for time travel.
283
282
  :param force_run: (bool) A flag that allow to release workflow if the
284
283
  audit with that release was pointed. (Default is False).
285
284
  :param extras: (DictDatA) An extra parameter that want to override core
@@ -288,15 +287,12 @@ class ReleaseQueue:
288
287
  :rtype: ReleaseQueue
289
288
 
290
289
  """
291
- if runner.date > end_date:
290
+ if clear_tz(runner.date) > clear_tz(end_date):
292
291
  return self
293
292
 
294
293
  release = Release(
295
- date=runner.date,
296
- offset=offset,
297
- end_date=end_date,
298
- runner=runner,
299
- type=ReleaseType.POKE,
294
+ date=clear_tz(runner.date),
295
+ type=(ReleaseType.FORCE if force_run else ReleaseType.POKING),
300
296
  )
301
297
 
302
298
  while self.check_queue(release) or (
@@ -304,17 +300,16 @@ class ReleaseQueue:
304
300
  and not force_run
305
301
  ):
306
302
  release = Release(
307
- date=runner.next,
308
- offset=offset,
309
- end_date=end_date,
310
- runner=runner,
311
- type=ReleaseType.POKE,
303
+ date=clear_tz(runner.next),
304
+ type=(ReleaseType.FORCE if force_run else ReleaseType.POKING),
312
305
  )
313
306
 
314
- if runner.date > end_date:
307
+ if clear_tz(runner.date) > clear_tz(end_date):
315
308
  return self
316
309
 
317
- heappush(self.queue, release)
310
+ with self.lock:
311
+ heappush(self.queue, release)
312
+
318
313
  return self
319
314
 
320
315
 
@@ -447,7 +442,7 @@ class Workflow(BaseModel):
447
442
  :param value: A description string value that want to dedent.
448
443
  :rtype: str
449
444
  """
450
- return dedent(value)
445
+ return dedent(value.lstrip("\n"))
451
446
 
452
447
  @field_validator("on", mode="after")
453
448
  def __on_no_dup_and_reach_limit__(
@@ -477,6 +472,12 @@ class Workflow(BaseModel):
477
472
  # "If it has every minute cronjob on value, it should have "
478
473
  # "only one value in the on field."
479
474
  # )
475
+ set_tz: set[str] = {on.tz for on in value}
476
+ if len(set_tz) > 1:
477
+ raise ValueError(
478
+ f"The on fields should not contain multiple timezone, "
479
+ f"{list[set_tz]}."
480
+ )
480
481
 
481
482
  extras: Optional[DictData] = info.data.get("extras")
482
483
  if len(set_ons) > (
@@ -598,9 +599,10 @@ class Workflow(BaseModel):
598
599
  result: Optional[Result] = None,
599
600
  timeout: int = 600,
600
601
  ) -> Result:
601
- """Release the workflow execution with overriding parameter with the
602
- release templating that include logical date (release date), execution
603
- date, or running id to the params.
602
+ """Release the workflow which is executes workflow with writing audit
603
+ log tracking. The method is overriding parameter with the release
604
+ templating that include logical date (release date), execution date,
605
+ or running id to the params.
604
606
 
605
607
  This method allow workflow use audit object to save the execution
606
608
  result to audit destination like file audit to the local `./logs` path.
@@ -641,33 +643,32 @@ class Workflow(BaseModel):
641
643
  )
642
644
 
643
645
  # VALIDATE: check type of queue that valid with ReleaseQueue.
644
- if queue and not isinstance(queue, ReleaseQueue):
646
+ if queue is not None and not isinstance(queue, ReleaseQueue):
645
647
  raise TypeError(
646
648
  "The queue argument should be ReleaseQueue object only."
647
649
  )
648
650
 
649
651
  # VALIDATE: Change release value to Release object.
650
652
  if isinstance(release, datetime):
651
- release: Release = Release.from_dt(release, extras=self.extras)
653
+ release: Release = Release.from_dt(release)
652
654
 
653
655
  result.trace.info(
654
656
  f"[RELEASE]: Start {name!r} : {release.date:%Y-%m-%d %H:%M:%S}"
655
657
  )
658
+ tz: ZoneInfo = dynamic("tz", extras=self.extras)
659
+ values: DictData = param2template(
660
+ params,
661
+ params={
662
+ "release": {
663
+ "logical_date": release.date,
664
+ "execute_date": datetime.now(tz=tz),
665
+ "run_id": result.run_id,
666
+ }
667
+ },
668
+ extras=self.extras,
669
+ )
656
670
  rs: Result = self.execute(
657
- params=param2template(
658
- params,
659
- params={
660
- "release": {
661
- "logical_date": release.date,
662
- "execute_date": datetime.now(
663
- tz=dynamic("tz", extras=self.extras)
664
- ),
665
- "run_id": result.run_id,
666
- "timezone": dynamic("tz", extras=self.extras),
667
- }
668
- },
669
- extras=self.extras,
670
- ),
671
+ params=values,
671
672
  result=result,
672
673
  parent_run_id=result.parent_run_id,
673
674
  timeout=timeout,
@@ -675,10 +676,7 @@ class Workflow(BaseModel):
675
676
  result.trace.info(
676
677
  f"[RELEASE]: End {name!r} : {release.date:%Y-%m-%d %H:%M:%S}"
677
678
  )
678
-
679
- # NOTE: Saving execution result to destination of the input audit
680
- # object.
681
- result.trace.debug(f"[LOG]: Writing audit: {name!r}.")
679
+ result.trace.debug(f"[RELEASE]: Writing audit: {name!r}.")
682
680
  (
683
681
  audit(
684
682
  name=name,
@@ -693,8 +691,6 @@ class Workflow(BaseModel):
693
691
  )
694
692
 
695
693
  if queue:
696
- if release in queue.running:
697
- queue.running.remove(release)
698
694
  queue.mark_complete(release)
699
695
 
700
696
  return result.catch(
@@ -704,9 +700,13 @@ class Workflow(BaseModel):
704
700
  "release": {
705
701
  "type": release.type,
706
702
  "logical_date": release.date,
707
- "release": release,
708
703
  },
709
- "outputs": {"jobs": result.context.pop("jobs", {})},
704
+ **{"jobs": result.context.pop("jobs", {})},
705
+ **(
706
+ result.context["errors"]
707
+ if "errors" in result.context
708
+ else {}
709
+ ),
710
710
  },
711
711
  )
712
712
 
@@ -736,13 +736,8 @@ class Workflow(BaseModel):
736
736
  queue.gen(
737
737
  end_date,
738
738
  audit,
739
- on.next(
740
- get_dt_now(
741
- tz=dynamic("tz", extras=self.extras), offset=offset
742
- ).replace(microsecond=0)
743
- ),
739
+ on.next(get_dt_now(offset=offset).replace(microsecond=0)),
744
740
  self.name,
745
- offset=offset,
746
741
  force_run=force_run,
747
742
  )
748
743
 
@@ -750,36 +745,41 @@ class Workflow(BaseModel):
750
745
 
751
746
  def poke(
752
747
  self,
753
- start_date: datetime | None = None,
754
748
  params: DictData | None = None,
749
+ start_date: datetime | None = None,
755
750
  *,
756
751
  run_id: str | None = None,
757
752
  periods: int = 1,
758
753
  audit: Audit | None = None,
759
754
  force_run: bool = False,
760
755
  timeout: int = 1800,
761
- max_poking_pool_worker: int = 4,
756
+ max_poking_pool_worker: int = 2,
762
757
  ) -> Result:
763
- """Poke function with a start datetime value that will pass to its
758
+ """Poke workflow with a start datetime value that will pass to its
764
759
  `on` field on the threading executor pool for execute the `release`
765
760
  method (It run all schedules that was set on the `on` values).
766
761
 
767
- This method will observe its schedule that nearing to run with the
762
+ This method will observe its `on` field that nearing to run with the
768
763
  `self.release()` method.
769
764
 
770
- The limitation of this method is not allow run a date that less
765
+ The limitation of this method is not allow run a date that gather
771
766
  than the current date.
772
767
 
773
- :param start_date: A start datetime object.
774
- :param params: A parameters that want to pass to the release method.
775
- :param run_id: A workflow running ID for this poke.
776
- :param periods: A periods in minutes value that use to run this poking.
777
- :param audit: An audit object that want to use on this poking process.
778
- :param force_run: A flag that allow to release workflow if the audit with
779
- that release was pointed.
780
- :param timeout: A second value for timeout while waiting all futures
781
- run completely.
782
- :param max_poking_pool_worker: The maximum poking pool worker.
768
+ :param params: (DictData) A parameter data.
769
+ :param start_date: (datetime) A start datetime object.
770
+ :param run_id: (str) A workflow running ID for this poke.
771
+ :param periods: (int) A periods in minutes value that use to run this
772
+ poking. (Default is 1)
773
+ :param audit: (Audit) An audit object that want to use on this poking
774
+ process.
775
+ :param force_run: (bool) A flag that allow to release workflow if the
776
+ audit with that release was pointed. (Default is False)
777
+ :param timeout: (int) A second value for timeout while waiting all
778
+ futures run completely.
779
+ :param max_poking_pool_worker: (int) The maximum poking pool worker.
780
+ (Default is 2 workers)
781
+
782
+ :raise WorkflowException: If the periods parameter less or equal than 0.
783
783
 
784
784
  :rtype: Result
785
785
  :return: A list of all results that return from `self.release` method.
@@ -792,45 +792,49 @@ class Workflow(BaseModel):
792
792
  # VALIDATE: Check the periods value should gather than 0.
793
793
  if periods <= 0:
794
794
  raise WorkflowException(
795
- "The period of poking should be int and grater or equal than 1."
795
+ "The period of poking should be `int` and grater or equal "
796
+ "than 1."
796
797
  )
797
798
 
798
799
  if len(self.on) == 0:
799
- result.trace.info(
800
- f"[POKING]: {self.name!r} does not have any schedule to run."
800
+ result.trace.warning(
801
+ f"[POKING]: {self.name!r} not have any schedule!!!"
801
802
  )
802
803
  return result.catch(status=SUCCESS, context={"outputs": []})
803
804
 
804
805
  # NOTE: Create the current date that change microsecond to 0
805
- current_date: datetime = datetime.now(
806
- tz=dynamic("tz", extras=self.extras)
807
- ).replace(microsecond=0)
808
-
809
- # NOTE: Create start_date and offset variables.
810
- if start_date and start_date <= current_date:
811
- start_date = start_date.replace(
812
- tzinfo=dynamic("tz", extras=self.extras)
813
- ).replace(microsecond=0)
814
- offset: float = (current_date - start_date).total_seconds()
815
- else:
806
+ current_date: datetime = datetime.now().replace(microsecond=0)
807
+
808
+ if start_date is None:
816
809
  # NOTE: Force change start date if it gathers than the current date,
817
810
  # or it does not pass to this method.
818
811
  start_date: datetime = current_date
819
812
  offset: float = 0
813
+ elif start_date <= current_date:
814
+ start_date = start_date.replace(microsecond=0)
815
+ offset: float = (current_date - start_date).total_seconds()
816
+ else:
817
+ raise WorkflowException(
818
+ f"The start datetime should less than or equal the current "
819
+ f"datetime, {current_date:%Y-%m-%d %H:%M:%S}."
820
+ )
820
821
 
821
822
  # NOTE: The end date is using to stop generate queue with an input
822
- # periods value.
823
- end_date: datetime = start_date + timedelta(minutes=periods)
823
+ # periods value. It will change to MM:59.
824
+ # For example:
825
+ # (input) start_date = 12:04:12, offset = 2
826
+ # (output) end_date = 12:06:59
827
+ end_date: datetime = start_date.replace(second=0) + timedelta(
828
+ minutes=periods + 1, seconds=-1
829
+ )
824
830
 
825
831
  result.trace.info(
826
- f"[POKING]: Start Poking: {self.name!r} from "
827
- f"{start_date:%Y-%m-%d %H:%M:%S} to {end_date:%Y-%m-%d %H:%M:%S}"
832
+ f"[POKING]: Execute Poking: {self.name!r} ("
833
+ f"{start_date:%Y-%m-%d %H:%M:%S} ==> {end_date:%Y-%m-%d %H:%M:%S})"
828
834
  )
829
835
 
830
836
  params: DictData = {} if params is None else params
831
837
  context: list[Result] = []
832
-
833
- # NOTE: Create empty ReleaseQueue object.
834
838
  q: ReleaseQueue = ReleaseQueue()
835
839
 
836
840
  # NOTE: Create reusable partial function and add Release to the release
@@ -839,16 +843,12 @@ class Workflow(BaseModel):
839
843
  self.queue, offset, end_date, audit=audit, force_run=force_run
840
844
  )
841
845
  partial_queue(q)
842
-
843
- # NOTE: Return the empty result if it does not have any Release.
844
846
  if not q.is_queued:
845
- result.trace.info(
846
- f"[POKING]: {self.name!r} does not have any queue."
847
+ result.trace.warning(
848
+ f"[POKING]: Skip {self.name!r}, not have any queue!!!"
847
849
  )
848
850
  return result.catch(status=SUCCESS, context={"outputs": []})
849
851
 
850
- # NOTE: Start create the thread pool executor for running this poke
851
- # process.
852
852
  with ThreadPoolExecutor(
853
853
  max_workers=dynamic(
854
854
  "max_poking_pool_worker",
@@ -867,16 +867,11 @@ class Workflow(BaseModel):
867
867
 
868
868
  if reach_next_minute(release.date, offset=offset):
869
869
  result.trace.debug(
870
- f"[POKING]: Latest Release, "
871
- f"{release.date:%Y-%m-%d %H:%M:%S}, can not run on "
872
- f"this time"
870
+ f"[POKING]: Skip Release: "
871
+ f"{release.date:%Y-%m-%d %H:%M:%S}"
873
872
  )
874
873
  heappush(q.queue, release)
875
- wait_to_next_minute(
876
- get_dt_now(
877
- tz=dynamic("tz", extras=self.extras), offset=offset
878
- )
879
- )
874
+ wait_until_next_minute(get_dt_now(offset=offset))
880
875
 
881
876
  # WARNING: I already call queue poking again because issue
882
877
  # about the every minute crontab.
@@ -909,7 +904,7 @@ class Workflow(BaseModel):
909
904
 
910
905
  def execute_job(
911
906
  self,
912
- job_id: str,
907
+ job: Job,
913
908
  params: DictData,
914
909
  *,
915
910
  result: Result | None = None,
@@ -922,10 +917,9 @@ class Workflow(BaseModel):
922
917
  model. It different with `self.execute` because this method run only
923
918
  one job and return with context of this job data.
924
919
 
925
- :raise WorkflowException: If execute with not exist job's ID.
926
920
  :raise WorkflowException: If the job execution raise JobException.
927
921
 
928
- :param job_id: A job ID.
922
+ :param job: (Job) A job model that want to execute.
929
923
  :param params: (DictData) A parameter data.
930
924
  :param result: (Result) A Result instance for return context and status.
931
925
  :param event: (Event) An Event manager instance that use to cancel this
@@ -933,48 +927,37 @@ class Workflow(BaseModel):
933
927
 
934
928
  :rtype: Result
935
929
  """
936
- if result is None: # pragma: no cov
937
- result: Result = Result(run_id=gen_id(self.name, unique=True))
938
-
939
- # VALIDATE: check a job ID that exists in this workflow or not.
940
- if job_id not in self.jobs:
941
- raise WorkflowException(
942
- f"The job: {job_id!r} does not exists in {self.name!r} "
943
- f"workflow."
944
- )
930
+ result: Result = result or Result(run_id=gen_id(self.name, unique=True))
945
931
 
946
- job: Job = self.job(name=job_id)
947
932
  if job.is_skipped(params=params):
948
- result.trace.info(f"[WORKFLOW]: Skip job: {job_id!r}")
933
+ result.trace.info(f"[WORKFLOW]: Skip Job: {job.id!r}")
949
934
  job.set_outputs(output={"skipped": True}, to=params)
950
935
  return result.catch(status=SKIP, context=params)
951
936
 
952
- if event and event.is_set(): # pragma: no cov
953
- raise WorkflowException(
954
- "Workflow job was canceled from event that had set before "
955
- "job execution."
937
+ if event and event.is_set():
938
+ return result.catch(
939
+ status=CANCEL,
940
+ context={
941
+ "errors": WorkflowException(
942
+ "Workflow job was canceled because event was set."
943
+ ).to_dict(),
944
+ },
956
945
  )
957
946
 
958
- try:
959
- result.trace.info(f"[WORKFLOW]: Execute Job: {job_id!r}")
960
- rs: Result = job.execute(
961
- params=params,
962
- run_id=result.run_id,
963
- parent_run_id=result.parent_run_id,
964
- event=event,
947
+ result.trace.info(f"[WORKFLOW]: Execute Job: {job.id!r}")
948
+ rs: Result = job.execute(
949
+ params=params,
950
+ run_id=result.run_id,
951
+ parent_run_id=result.parent_run_id,
952
+ event=event,
953
+ )
954
+ job.set_outputs(rs.context, to=params)
955
+ if rs.status in (FAILED, CANCEL):
956
+ error_msg: str = (
957
+ f"Job, {job.id!r}, return `{rs.status.name}` status."
965
958
  )
966
- job.set_outputs(rs.context, to=params)
967
- except (JobException, UtilException) as e:
968
- result.trace.error(f"[WORKFLOW]: {e.__class__.__name__}: {e}")
969
- raise WorkflowException(
970
- f"Job {job_id!r} raise {e.__class__.__name__}: {e}"
971
- ) from None
972
-
973
- if rs.status == FAILED:
974
- error_msg: str = f"Workflow job, {job.id!r}, return FAILED status."
975
- result.trace.warning(f"[WORKFLOW]: {error_msg}")
976
959
  return result.catch(
977
- status=FAILED,
960
+ status=rs.status,
978
961
  context={
979
962
  "errors": WorkflowException(error_msg).to_dict(),
980
963
  **params,
@@ -1039,7 +1022,7 @@ class Workflow(BaseModel):
1039
1022
  extras=self.extras,
1040
1023
  )
1041
1024
  context: DictData = self.parameterize(params)
1042
- result.trace.info(f"[WORKFLOW]: Execute: {self.name!r} ...")
1025
+ result.trace.info(f"[WORKFLOW]: Execute: {self.name!r}")
1043
1026
  if not self.jobs:
1044
1027
  result.trace.warning(f"[WORKFLOW]: {self.name!r} does not set jobs")
1045
1028
  return result.catch(status=SUCCESS, context=context)
@@ -1053,7 +1036,9 @@ class Workflow(BaseModel):
1053
1036
  "max_job_exec_timeout", f=timeout, extras=self.extras
1054
1037
  )
1055
1038
  event: Event = event or Event()
1056
- result.trace.debug(f"... Run {self.name!r} with non-threading.")
1039
+ result.trace.debug(
1040
+ f"[WORKFLOW]: ... Run {self.name!r} with non-threading."
1041
+ )
1057
1042
  max_job_parallel: int = dynamic(
1058
1043
  "max_job_parallel", f=max_job_parallel, extras=self.extras
1059
1044
  )
@@ -1093,7 +1078,7 @@ class Workflow(BaseModel):
1093
1078
  futures.append(
1094
1079
  executor.submit(
1095
1080
  self.execute_job,
1096
- job_id=job_id,
1081
+ job=job,
1097
1082
  params=context,
1098
1083
  result=result,
1099
1084
  event=event,
@@ -1106,7 +1091,7 @@ class Workflow(BaseModel):
1106
1091
  futures.append(
1107
1092
  executor.submit(
1108
1093
  self.execute_job,
1109
- job_id=job_id,
1094
+ job=job,
1110
1095
  params=context,
1111
1096
  result=result,
1112
1097
  event=event,
@@ -1114,9 +1099,6 @@ class Workflow(BaseModel):
1114
1099
  )
1115
1100
  time.sleep(0.025)
1116
1101
  elif (future := futures.pop(0)).done() or future.cancelled():
1117
- if e := future.exception():
1118
- result.trace.error(f"[WORKFLOW]: {e}")
1119
- raise WorkflowException(str(e))
1120
1102
  job_queue.put(job_id)
1121
1103
  elif future.running() or "state=pending" in str(future):
1122
1104
  time.sleep(0.075)
@@ -1125,8 +1107,10 @@ class Workflow(BaseModel):
1125
1107
  else: # pragma: no cov
1126
1108
  job_queue.put(job_id)
1127
1109
  futures.insert(0, future)
1110
+ time.sleep(0.025)
1128
1111
  result.trace.warning(
1129
- f"... Execution non-threading not handle: {future}."
1112
+ f"[WORKFLOW]: ... Execution non-threading not "
1113
+ f"handle: {future}."
1130
1114
  )
1131
1115
 
1132
1116
  job_queue.task_done()
@@ -1134,16 +1118,7 @@ class Workflow(BaseModel):
1134
1118
  if not_timeout_flag:
1135
1119
  job_queue.join()
1136
1120
  for future in as_completed(futures):
1137
- try:
1138
- future.result()
1139
- except WorkflowException as e:
1140
- result.trace.error(f"[WORKFLOW]: Handler:{NEWLINE}{e}")
1141
- return result.catch(
1142
- status=FAILED,
1143
- context={
1144
- "errors": WorkflowException(str(e)).to_dict()
1145
- },
1146
- )
1121
+ future.result()
1147
1122
  return result.catch(
1148
1123
  status=FAILED if "errors" in result.context else SUCCESS,
1149
1124
  context=context,