ddeutil-workflow 0.0.55__py3-none-any.whl → 0.0.57__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,12 +3,14 @@
3
3
  # Licensed under the MIT License. See LICENSE in the project root for
4
4
  # license information.
5
5
  # ------------------------------------------------------------------------------
6
- # [x] Use dynamic config
7
- """Workflow module is the core module of this Workflow package. It keeps
8
- Release, ReleaseQueue, and Workflow Pydantic models.
6
+ """Workflow module is the core module of this package. It keeps Release,
7
+ ReleaseQueue, and Workflow models.
9
8
 
10
9
  This package implement timeout strategy on the workflow execution layer only
11
10
  because the main propose of this package is using Workflow to be orchestrator.
11
+
12
+ ReleaseQueue is the memory storage of Release for tracking this release
13
+ already run or pending in the current session.
12
14
  """
13
15
  from __future__ import annotations
14
16
 
@@ -27,30 +29,32 @@ from heapq import heappop, heappush
27
29
  from pathlib import Path
28
30
  from queue import Queue
29
31
  from textwrap import dedent
30
- from threading import Event
31
- from typing import Optional
32
+ from threading import Event, Lock
33
+ from typing import Any, Optional, Union
34
+ from zoneinfo import ZoneInfo
32
35
 
33
36
  from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
34
37
  from pydantic.dataclasses import dataclass
35
38
  from pydantic.functional_validators import field_validator, model_validator
36
39
  from typing_extensions import Self
37
40
 
38
- from .__cron import CronJob, CronRunner
41
+ from .__cron import CronRunner
39
42
  from .__types import DictData, TupleStr
40
- from .conf import Loader, SimLoad, dynamic
41
- from .cron import On
42
- from .exceptions import JobException, UtilException, WorkflowException
43
+ from .conf import FileLoad, Loader, dynamic
44
+ from .event import On
45
+ from .exceptions import WorkflowException
43
46
  from .job import Job
44
47
  from .logs import Audit, get_audit
45
48
  from .params import Param
46
- from .result import FAILED, SKIP, SUCCESS, WAIT, Result
49
+ from .result import CANCEL, FAILED, SKIP, SUCCESS, WAIT, Result
47
50
  from .reusables import has_template, param2template
48
51
  from .utils import (
49
- NEWLINE,
52
+ clear_tz,
50
53
  gen_id,
51
54
  get_dt_now,
52
55
  reach_next_minute,
53
- wait_to_next_minute,
56
+ replace_sec,
57
+ wait_until_next_minute,
54
58
  )
55
59
 
56
60
  __all__: TupleStr = (
@@ -66,46 +70,50 @@ class ReleaseType(str, Enum):
66
70
  """Release Type Enum support the type field on the Release dataclass."""
67
71
 
68
72
  DEFAULT: str = "manual"
69
- TASK: str = "task"
70
- POKE: str = "poking"
73
+ SCHEDULE: str = "schedule"
74
+ POKING: str = "poking"
75
+ FORCE: str = "force"
71
76
 
72
77
 
73
78
  @total_ordering
74
- @dataclass(
75
- config=ConfigDict(arbitrary_types_allowed=True, use_enum_values=True)
76
- )
79
+ @dataclass(config=ConfigDict(use_enum_values=True))
77
80
  class Release:
78
- """Release Pydantic dataclass object that use for represent the release data
79
- that use with the `workflow.release` method.
80
- """
81
+ """Release object that use for represent the release datetime."""
81
82
 
82
- date: datetime
83
- offset: float
84
- end_date: datetime
85
- runner: CronRunner
86
- type: ReleaseType = field(default=ReleaseType.DEFAULT)
83
+ date: datetime = Field(
84
+ description=(
85
+ "A release date that should has second and millisecond equal 0."
86
+ )
87
+ )
88
+ type: ReleaseType = Field(
89
+ default=ReleaseType.DEFAULT,
90
+ description="A type of release that create before start execution.",
91
+ )
87
92
 
88
93
  def __repr__(self) -> str:
89
- """Represent string"""
94
+ """Override __repr__ method for represent value of `date` field.
95
+
96
+ :rtype: str
97
+ """
90
98
  return repr(f"{self.date:%Y-%m-%d %H:%M:%S}")
91
99
 
92
100
  def __str__(self) -> str:
93
- """Override string value of this release object with the date field.
101
+ """Override string value of this release object with the `date` field.
94
102
 
95
103
  :rtype: str
96
104
  """
97
105
  return f"{self.date:%Y-%m-%d %H:%M:%S}"
98
106
 
99
107
  @classmethod
100
- def from_dt(
101
- cls, dt: datetime | str, *, extras: Optional[DictData] = None
102
- ) -> Self:
103
- """Construct Release via datetime object only.
108
+ def from_dt(cls, dt: datetime | str) -> Self:
109
+ """Construct Release object from `datetime` or `str` objects.
110
+
111
+ This method will replace second and millisecond value to 0 and
112
+ replace timezone to the `tz` config setting or extras overriding before
113
+ create Release object.
104
114
 
105
115
  :param dt: (datetime | str) A datetime object or string that want to
106
116
  construct to the Release object.
107
- :param extras: (DictData) An extra parameters that want to pass to
108
- override config values.
109
117
 
110
118
  :raise TypeError: If the type of the dt argument does not valid with
111
119
  datetime or str object.
@@ -116,20 +124,10 @@ class Release:
116
124
  dt: datetime = datetime.fromisoformat(dt)
117
125
  elif not isinstance(dt, datetime):
118
126
  raise TypeError(
119
- "The `from_dt` need the `dt` argument type be str or datetime "
120
- "only."
127
+ f"The `from_dt` need the `dt` parameter type be `str` or "
128
+ f"`datetime` only, not {type(dt)}."
121
129
  )
122
-
123
- return cls(
124
- date=dt,
125
- offset=0,
126
- end_date=dt + timedelta(days=1),
127
- runner=(
128
- CronJob("* * * * *").schedule(
129
- dt.replace(tzinfo=dynamic("tz", extras=extras))
130
- )
131
- ),
132
- )
130
+ return cls(date=replace_sec(dt.replace(tzinfo=None)))
133
131
 
134
132
  def __eq__(self, other: Release | datetime) -> bool:
135
133
  """Override equal property that will compare only the same type or
@@ -144,7 +142,7 @@ class Release:
144
142
  return NotImplemented
145
143
 
146
144
  def __lt__(self, other: Release | datetime) -> bool:
147
- """Override equal property that will compare only the same type or
145
+ """Override less-than property that will compare only the same type or
148
146
  datetime.
149
147
 
150
148
  :rtype: bool
@@ -156,31 +154,33 @@ class Release:
156
154
  return NotImplemented
157
155
 
158
156
 
159
- @dataclass
160
157
  class ReleaseQueue:
161
- """Workflow Queue object that is management of Release objects."""
158
+ """ReleaseQueue object that is storage management of Release objects on
159
+ the memory with list object.
160
+ """
162
161
 
163
- queue: list[Release] = field(default_factory=list)
164
- running: list[Release] = field(default_factory=list)
165
- complete: list[Release] = field(default_factory=list)
166
- extras: DictData = Field(
167
- default_factory=dict,
168
- description="An extra parameters that want to override config values.",
169
- repr=False,
170
- )
162
+ def __init__(
163
+ self,
164
+ queue: Optional[list[Release]] = None,
165
+ running: Optional[list[Release]] = None,
166
+ complete: Optional[list[Release]] = None,
167
+ extras: Optional[DictData] = None,
168
+ ):
169
+ self.queue: list[Release] = queue or []
170
+ self.running: list[Release] = running or []
171
+ self.complete: list[Release] = complete or []
172
+ self.extras: DictData = extras or {}
173
+ self.lock: Lock = Lock()
171
174
 
172
175
  @classmethod
173
176
  def from_list(
174
177
  cls,
175
- queue: list[datetime] | list[Release] | None = None,
176
- extras: Optional[DictData] = None,
178
+ queue: Optional[Union[list[datetime], list[Release]]] = None,
177
179
  ) -> Self:
178
180
  """Construct ReleaseQueue object from an input queue value that passing
179
181
  with list of datetime or list of Release.
180
182
 
181
- :param queue:
182
- :param extras: An extra parameter that want to override core config
183
- values.
183
+ :param queue: A queue object for create ReleaseQueue instance.
184
184
 
185
185
  :raise TypeError: If the type of input queue does not valid.
186
186
 
@@ -190,13 +190,8 @@ class ReleaseQueue:
190
190
  return cls()
191
191
 
192
192
  if isinstance(queue, list):
193
-
194
193
  if all(isinstance(q, datetime) for q in queue):
195
- return cls(
196
- queue=[
197
- Release.from_dt(q, extras=(extras or {})) for q in queue
198
- ]
199
- )
194
+ return cls(queue=[Release.from_dt(q) for q in queue])
200
195
 
201
196
  if all(isinstance(q, Release) for q in queue):
202
197
  return cls(queue=queue)
@@ -224,30 +219,39 @@ class ReleaseQueue:
224
219
  :rtype: bool
225
220
  """
226
221
  if isinstance(value, datetime):
227
- value = Release.from_dt(value, extras=self.extras)
222
+ value = Release.from_dt(value)
228
223
 
229
- return (
230
- (value in self.queue)
231
- or (value in self.running)
232
- or (value in self.complete)
233
- )
224
+ with self.lock:
225
+ return (
226
+ (value in self.queue)
227
+ or (value in self.running)
228
+ or (value in self.complete)
229
+ )
234
230
 
235
231
  def mark_complete(self, value: Release) -> Self:
236
- """Push Release to the complete queue.
232
+ """Push Release to the complete queue. After push the release, it will
233
+ delete old release base on the `CORE_MAX_QUEUE_COMPLETE_HIST` value.
234
+
235
+ :param value: (Release) A Release value that want to push to the
236
+ complete field.
237
237
 
238
238
  :rtype: Self
239
239
  """
240
- heappush(self.complete, value)
240
+ with self.lock:
241
+ if value in self.running:
242
+ self.running.remove(value)
241
243
 
242
- # NOTE: Remove complete queue on workflow that keep more than the
243
- # maximum config value.
244
- num_complete_delete: int = len(self.complete) - dynamic(
245
- "max_queue_complete_hist", extras=self.extras
246
- )
244
+ heappush(self.complete, value)
245
+
246
+ # NOTE: Remove complete queue on workflow that keep more than the
247
+ # maximum config value.
248
+ num_complete_delete: int = len(self.complete) - dynamic(
249
+ "max_queue_complete_hist", extras=self.extras
250
+ )
247
251
 
248
- if num_complete_delete > 0:
249
- for _ in range(num_complete_delete):
250
- heappop(self.complete)
252
+ if num_complete_delete > 0:
253
+ for _ in range(num_complete_delete):
254
+ heappop(self.complete)
251
255
 
252
256
  return self
253
257
 
@@ -258,11 +262,10 @@ class ReleaseQueue:
258
262
  runner: CronRunner,
259
263
  name: str,
260
264
  *,
261
- offset: float = 0,
262
265
  force_run: bool = False,
263
266
  extras: Optional[DictData] = None,
264
267
  ) -> Self:
265
- """Generate Release model to queue.
268
+ """Generate a Release model to the queue field with an input CronRunner.
266
269
 
267
270
  Steps:
268
271
  - Create Release object from the current date that not reach the end
@@ -274,50 +277,44 @@ class ReleaseQueue:
274
277
  :param end_date: (datetime) An end datetime object.
275
278
  :param audit: (type[Audit]) An audit class that want to make audit
276
279
  instance.
277
- :param runner: (CronRunner) A CronRunner object.
280
+ :param runner: (CronRunner) A `CronRunner` object.
278
281
  :param name: (str) A target name that want to check at pointer of audit.
279
- :param offset: (float) An offset in second unit for time travel.
280
- :param force_run: A flag that allow to release workflow if the audit
281
- with that release was pointed.
282
- :param extras: An extra parameter that want to override core config.
282
+ :param force_run: (bool) A flag that allow to release workflow if the
283
+ audit with that release was pointed. (Default is False).
284
+ :param extras: (DictDatA) An extra parameter that want to override core
285
+ config values.
283
286
 
284
287
  :rtype: ReleaseQueue
285
288
 
286
289
  """
287
- if runner.date > end_date:
290
+ if clear_tz(runner.date) > clear_tz(end_date):
288
291
  return self
289
292
 
290
- workflow_release = Release(
291
- date=runner.date,
292
- offset=offset,
293
- end_date=end_date,
294
- runner=runner,
295
- type=ReleaseType.POKE,
293
+ release = Release(
294
+ date=clear_tz(runner.date),
295
+ type=(ReleaseType.FORCE if force_run else ReleaseType.POKING),
296
296
  )
297
297
 
298
- while self.check_queue(workflow_release) or (
299
- audit.is_pointed(
300
- name=name, release=workflow_release.date, extras=extras
301
- )
298
+ while self.check_queue(release) or (
299
+ audit.is_pointed(name=name, release=release.date, extras=extras)
302
300
  and not force_run
303
301
  ):
304
- workflow_release = Release(
305
- date=runner.next,
306
- offset=offset,
307
- end_date=end_date,
308
- runner=runner,
309
- type=ReleaseType.POKE,
302
+ release = Release(
303
+ date=clear_tz(runner.next),
304
+ type=(ReleaseType.FORCE if force_run else ReleaseType.POKING),
310
305
  )
311
306
 
312
- if runner.date > end_date:
307
+ if clear_tz(runner.date) > clear_tz(end_date):
313
308
  return self
314
309
 
315
- heappush(self.queue, workflow_release)
310
+ with self.lock:
311
+ heappush(self.queue, release)
312
+
316
313
  return self
317
314
 
318
315
 
319
316
  class Workflow(BaseModel):
320
- """Workflow Pydantic model.
317
+ """Workflow model that use to keep the `Job` and `On` models.
321
318
 
322
319
  This is the main future of this project because it uses to be workflow
323
320
  data for running everywhere that you want or using it to scheduler task in
@@ -355,6 +352,7 @@ class Workflow(BaseModel):
355
352
  cls,
356
353
  name: str,
357
354
  *,
355
+ path: Optional[Path] = None,
358
356
  extras: DictData | None = None,
359
357
  loader: type[Loader] = None,
360
358
  ) -> Self:
@@ -362,45 +360,8 @@ class Workflow(BaseModel):
362
360
  an input workflow name. The loader object will use this workflow name to
363
361
  searching configuration data of this workflow model in conf path.
364
362
 
365
- :param name: A workflow name that want to pass to Loader object.
366
- :param extras: An extra parameters that want to pass to Loader
367
- object.
368
- :param loader: A loader class for override default loader object.
369
-
370
- :raise ValueError: If the type does not match with current object.
371
-
372
- :rtype: Self
373
- """
374
- loader: Loader = (loader or Loader)(name, externals=(extras or {}))
375
-
376
- # NOTE: Validate the config type match with current connection model
377
- if loader.type != cls.__name__:
378
- raise ValueError(f"Type {loader.type} does not match with {cls}")
379
-
380
- loader_data: DictData = copy.deepcopy(loader.data)
381
- loader_data["name"] = name.replace(" ", "_")
382
-
383
- if extras:
384
- loader_data["extras"] = extras
385
-
386
- cls.__bypass_on__(loader_data, path=loader.conf_path, extras=extras)
387
- return cls.model_validate(obj=loader_data)
388
-
389
- @classmethod
390
- def from_path(
391
- cls,
392
- name: str,
393
- path: Path,
394
- *,
395
- extras: DictData | None = None,
396
- loader: type[Loader] = None,
397
- ) -> Self:
398
- """Create Workflow instance from the specific path. The loader object
399
- will use this workflow name and path to searching configuration data of
400
- this workflow model.
401
-
402
363
  :param name: (str) A workflow name that want to pass to Loader object.
403
- :param path: (Path) A config path that want to search.
364
+ :param path: (Path) An override config path.
404
365
  :param extras: (DictData) An extra parameters that want to override core
405
366
  config values.
406
367
  :param loader: A loader class for override default loader object.
@@ -409,21 +370,21 @@ class Workflow(BaseModel):
409
370
 
410
371
  :rtype: Self
411
372
  """
412
- loader: SimLoad = (loader or SimLoad)(
413
- name, conf_path=path, externals=(extras or {})
414
- )
373
+ loader: type[Loader] = loader or FileLoad
374
+ load: Loader = loader(name, path=path, extras=extras)
375
+
415
376
  # NOTE: Validate the config type match with current connection model
416
- if loader.type != cls.__name__:
417
- raise ValueError(f"Type {loader.type} does not match with {cls}")
377
+ if load.type != cls.__name__:
378
+ raise ValueError(f"Type {load.type} does not match with {cls}")
418
379
 
419
- loader_data: DictData = copy.deepcopy(loader.data)
420
- loader_data["name"] = name.replace(" ", "_")
380
+ data: DictData = copy.deepcopy(load.data)
381
+ data["name"] = name
421
382
 
422
383
  if extras:
423
- loader_data["extras"] = extras
384
+ data["extras"] = extras
424
385
 
425
- cls.__bypass_on__(loader_data, path=path, extras=extras)
426
- return cls.model_validate(obj=loader_data)
386
+ cls.__bypass_on__(data, path=load.path, extras=extras, loader=loader)
387
+ return cls.model_validate(obj=data)
427
388
 
428
389
  @classmethod
429
390
  def __bypass_on__(
@@ -431,6 +392,7 @@ class Workflow(BaseModel):
431
392
  data: DictData,
432
393
  path: Path,
433
394
  extras: DictData | None = None,
395
+ loader: type[Loader] = None,
434
396
  ) -> DictData:
435
397
  """Bypass the on data to loaded config data.
436
398
 
@@ -451,7 +413,7 @@ class Workflow(BaseModel):
451
413
  # field.
452
414
  data["on"] = [
453
415
  (
454
- SimLoad(n, conf_path=path, externals=(extras or {})).data
416
+ (loader or FileLoad)(n, path=path, extras=extras).data
455
417
  if isinstance(n, str)
456
418
  else n
457
419
  )
@@ -460,11 +422,10 @@ class Workflow(BaseModel):
460
422
  return data
461
423
 
462
424
  @model_validator(mode="before")
463
- def __prepare_model_before__(cls, values: DictData) -> DictData:
425
+ def __prepare_model_before__(cls, data: Any) -> Any:
464
426
  """Prepare the params key in the data model before validating."""
465
- # NOTE: Prepare params type if it is passing with only type value.
466
- if params := values.pop("params", {}):
467
- values["params"] = {
427
+ if isinstance(data, dict) and (params := data.pop("params", {})):
428
+ data["params"] = {
468
429
  p: (
469
430
  {"type": params[p]}
470
431
  if isinstance(params[p], str)
@@ -472,7 +433,7 @@ class Workflow(BaseModel):
472
433
  )
473
434
  for p in params
474
435
  }
475
- return values
436
+ return data
476
437
 
477
438
  @field_validator("desc", mode="after")
478
439
  def __dedent_desc__(cls, value: str) -> str:
@@ -481,7 +442,7 @@ class Workflow(BaseModel):
481
442
  :param value: A description string value that want to dedent.
482
443
  :rtype: str
483
444
  """
484
- return dedent(value)
445
+ return dedent(value.lstrip("\n"))
485
446
 
486
447
  @field_validator("on", mode="after")
487
448
  def __on_no_dup_and_reach_limit__(
@@ -511,6 +472,12 @@ class Workflow(BaseModel):
511
472
  # "If it has every minute cronjob on value, it should have "
512
473
  # "only one value in the on field."
513
474
  # )
475
+ set_tz: set[str] = {on.tz for on in value}
476
+ if len(set_tz) > 1:
477
+ raise ValueError(
478
+ f"The on fields should not contain multiple timezone, "
479
+ f"{list[set_tz]}."
480
+ )
514
481
 
515
482
  extras: Optional[DictData] = info.data.get("extras")
516
483
  if len(set_ons) > (
@@ -632,9 +599,10 @@ class Workflow(BaseModel):
632
599
  result: Optional[Result] = None,
633
600
  timeout: int = 600,
634
601
  ) -> Result:
635
- """Release the workflow execution with overriding parameter with the
636
- release templating that include logical date (release date), execution
637
- date, or running id to the params.
602
+ """Release the workflow which is executes workflow with writing audit
603
+ log tracking. The method is overriding parameter with the release
604
+ templating that include logical date (release date), execution date,
605
+ or running id to the params.
638
606
 
639
607
  This method allow workflow use audit object to save the execution
640
608
  result to audit destination like file audit to the local `./logs` path.
@@ -675,33 +643,32 @@ class Workflow(BaseModel):
675
643
  )
676
644
 
677
645
  # VALIDATE: check type of queue that valid with ReleaseQueue.
678
- if queue and not isinstance(queue, ReleaseQueue):
646
+ if queue is not None and not isinstance(queue, ReleaseQueue):
679
647
  raise TypeError(
680
648
  "The queue argument should be ReleaseQueue object only."
681
649
  )
682
650
 
683
651
  # VALIDATE: Change release value to Release object.
684
652
  if isinstance(release, datetime):
685
- release: Release = Release.from_dt(release, extras=self.extras)
653
+ release: Release = Release.from_dt(release)
686
654
 
687
655
  result.trace.info(
688
656
  f"[RELEASE]: Start {name!r} : {release.date:%Y-%m-%d %H:%M:%S}"
689
657
  )
658
+ tz: ZoneInfo = dynamic("tz", extras=self.extras)
659
+ values: DictData = param2template(
660
+ params,
661
+ params={
662
+ "release": {
663
+ "logical_date": release.date,
664
+ "execute_date": datetime.now(tz=tz),
665
+ "run_id": result.run_id,
666
+ }
667
+ },
668
+ extras=self.extras,
669
+ )
690
670
  rs: Result = self.execute(
691
- params=param2template(
692
- params,
693
- params={
694
- "release": {
695
- "logical_date": release.date,
696
- "execute_date": datetime.now(
697
- tz=dynamic("tz", extras=self.extras)
698
- ),
699
- "run_id": result.run_id,
700
- "timezone": dynamic("tz", extras=self.extras),
701
- }
702
- },
703
- extras=self.extras,
704
- ),
671
+ params=values,
705
672
  result=result,
706
673
  parent_run_id=result.parent_run_id,
707
674
  timeout=timeout,
@@ -709,10 +676,7 @@ class Workflow(BaseModel):
709
676
  result.trace.info(
710
677
  f"[RELEASE]: End {name!r} : {release.date:%Y-%m-%d %H:%M:%S}"
711
678
  )
712
-
713
- # NOTE: Saving execution result to destination of the input audit
714
- # object.
715
- result.trace.debug(f"[LOG]: Writing audit: {name!r}.")
679
+ result.trace.debug(f"[RELEASE]: Writing audit: {name!r}.")
716
680
  (
717
681
  audit(
718
682
  name=name,
@@ -727,8 +691,6 @@ class Workflow(BaseModel):
727
691
  )
728
692
 
729
693
  if queue:
730
- if release in queue.running:
731
- queue.running.remove(release)
732
694
  queue.mark_complete(release)
733
695
 
734
696
  return result.catch(
@@ -738,9 +700,13 @@ class Workflow(BaseModel):
738
700
  "release": {
739
701
  "type": release.type,
740
702
  "logical_date": release.date,
741
- "release": release,
742
703
  },
743
- "outputs": {"jobs": result.context.pop("jobs", {})},
704
+ **{"jobs": result.context.pop("jobs", {})},
705
+ **(
706
+ result.context["errors"]
707
+ if "errors" in result.context
708
+ else {}
709
+ ),
744
710
  },
745
711
  )
746
712
 
@@ -770,13 +736,8 @@ class Workflow(BaseModel):
770
736
  queue.gen(
771
737
  end_date,
772
738
  audit,
773
- on.next(
774
- get_dt_now(
775
- tz=dynamic("tz", extras=self.extras), offset=offset
776
- ).replace(microsecond=0)
777
- ),
739
+ on.next(get_dt_now(offset=offset).replace(microsecond=0)),
778
740
  self.name,
779
- offset=offset,
780
741
  force_run=force_run,
781
742
  )
782
743
 
@@ -784,36 +745,41 @@ class Workflow(BaseModel):
784
745
 
785
746
  def poke(
786
747
  self,
787
- start_date: datetime | None = None,
788
748
  params: DictData | None = None,
749
+ start_date: datetime | None = None,
789
750
  *,
790
751
  run_id: str | None = None,
791
752
  periods: int = 1,
792
753
  audit: Audit | None = None,
793
754
  force_run: bool = False,
794
755
  timeout: int = 1800,
795
- max_poking_pool_worker: int = 4,
756
+ max_poking_pool_worker: int = 2,
796
757
  ) -> Result:
797
- """Poke function with a start datetime value that will pass to its
758
+ """Poke workflow with a start datetime value that will pass to its
798
759
  `on` field on the threading executor pool for execute the `release`
799
760
  method (It run all schedules that was set on the `on` values).
800
761
 
801
- This method will observe its schedule that nearing to run with the
762
+ This method will observe its `on` field that nearing to run with the
802
763
  `self.release()` method.
803
764
 
804
- The limitation of this method is not allow run a date that less
765
+ The limitation of this method is not allow run a date that gather
805
766
  than the current date.
806
767
 
807
- :param start_date: A start datetime object.
808
- :param params: A parameters that want to pass to the release method.
809
- :param run_id: A workflow running ID for this poke.
810
- :param periods: A periods in minutes value that use to run this poking.
811
- :param audit: An audit object that want to use on this poking process.
812
- :param force_run: A flag that allow to release workflow if the audit with
813
- that release was pointed.
814
- :param timeout: A second value for timeout while waiting all futures
815
- run completely.
816
- :param max_poking_pool_worker: The maximum poking pool worker.
768
+ :param params: (DictData) A parameter data.
769
+ :param start_date: (datetime) A start datetime object.
770
+ :param run_id: (str) A workflow running ID for this poke.
771
+ :param periods: (int) A periods in minutes value that use to run this
772
+ poking. (Default is 1)
773
+ :param audit: (Audit) An audit object that want to use on this poking
774
+ process.
775
+ :param force_run: (bool) A flag that allow to release workflow if the
776
+ audit with that release was pointed. (Default is False)
777
+ :param timeout: (int) A second value for timeout while waiting all
778
+ futures run completely.
779
+ :param max_poking_pool_worker: (int) The maximum poking pool worker.
780
+ (Default is 2 workers)
781
+
782
+ :raise WorkflowException: If the periods parameter less or equal than 0.
817
783
 
818
784
  :rtype: Result
819
785
  :return: A list of all results that return from `self.release` method.
@@ -826,45 +792,49 @@ class Workflow(BaseModel):
826
792
  # VALIDATE: Check the periods value should gather than 0.
827
793
  if periods <= 0:
828
794
  raise WorkflowException(
829
- "The period of poking should be int and grater or equal than 1."
795
+ "The period of poking should be `int` and grater or equal "
796
+ "than 1."
830
797
  )
831
798
 
832
799
  if len(self.on) == 0:
833
- result.trace.info(
834
- f"[POKING]: {self.name!r} does not have any schedule to run."
800
+ result.trace.warning(
801
+ f"[POKING]: {self.name!r} not have any schedule!!!"
835
802
  )
836
803
  return result.catch(status=SUCCESS, context={"outputs": []})
837
804
 
838
805
  # NOTE: Create the current date that change microsecond to 0
839
- current_date: datetime = datetime.now(
840
- tz=dynamic("tz", extras=self.extras)
841
- ).replace(microsecond=0)
842
-
843
- # NOTE: Create start_date and offset variables.
844
- if start_date and start_date <= current_date:
845
- start_date = start_date.replace(
846
- tzinfo=dynamic("tz", extras=self.extras)
847
- ).replace(microsecond=0)
848
- offset: float = (current_date - start_date).total_seconds()
849
- else:
806
+ current_date: datetime = datetime.now().replace(microsecond=0)
807
+
808
+ if start_date is None:
850
809
  # NOTE: Force change start date if it gathers than the current date,
851
810
  # or it does not pass to this method.
852
811
  start_date: datetime = current_date
853
812
  offset: float = 0
813
+ elif start_date <= current_date:
814
+ start_date = start_date.replace(microsecond=0)
815
+ offset: float = (current_date - start_date).total_seconds()
816
+ else:
817
+ raise WorkflowException(
818
+ f"The start datetime should less than or equal the current "
819
+ f"datetime, {current_date:%Y-%m-%d %H:%M:%S}."
820
+ )
854
821
 
855
822
  # NOTE: The end date is using to stop generate queue with an input
856
- # periods value.
857
- end_date: datetime = start_date + timedelta(minutes=periods)
823
+ # periods value. It will change to MM:59.
824
+ # For example:
825
+ # (input) start_date = 12:04:12, offset = 2
826
+ # (output) end_date = 12:06:59
827
+ end_date: datetime = start_date.replace(second=0) + timedelta(
828
+ minutes=periods + 1, seconds=-1
829
+ )
858
830
 
859
831
  result.trace.info(
860
- f"[POKING]: Start Poking: {self.name!r} from "
861
- f"{start_date:%Y-%m-%d %H:%M:%S} to {end_date:%Y-%m-%d %H:%M:%S}"
832
+ f"[POKING]: Execute Poking: {self.name!r} ("
833
+ f"{start_date:%Y-%m-%d %H:%M:%S} ==> {end_date:%Y-%m-%d %H:%M:%S})"
862
834
  )
863
835
 
864
836
  params: DictData = {} if params is None else params
865
837
  context: list[Result] = []
866
-
867
- # NOTE: Create empty ReleaseQueue object.
868
838
  q: ReleaseQueue = ReleaseQueue()
869
839
 
870
840
  # NOTE: Create reusable partial function and add Release to the release
@@ -873,16 +843,12 @@ class Workflow(BaseModel):
873
843
  self.queue, offset, end_date, audit=audit, force_run=force_run
874
844
  )
875
845
  partial_queue(q)
876
-
877
- # NOTE: Return the empty result if it does not have any Release.
878
846
  if not q.is_queued:
879
- result.trace.info(
880
- f"[POKING]: {self.name!r} does not have any queue."
847
+ result.trace.warning(
848
+ f"[POKING]: Skip {self.name!r}, not have any queue!!!"
881
849
  )
882
850
  return result.catch(status=SUCCESS, context={"outputs": []})
883
851
 
884
- # NOTE: Start create the thread pool executor for running this poke
885
- # process.
886
852
  with ThreadPoolExecutor(
887
853
  max_workers=dynamic(
888
854
  "max_poking_pool_worker",
@@ -901,16 +867,11 @@ class Workflow(BaseModel):
901
867
 
902
868
  if reach_next_minute(release.date, offset=offset):
903
869
  result.trace.debug(
904
- f"[POKING]: Latest Release, "
905
- f"{release.date:%Y-%m-%d %H:%M:%S}, can not run on "
906
- f"this time"
870
+ f"[POKING]: Skip Release: "
871
+ f"{release.date:%Y-%m-%d %H:%M:%S}"
907
872
  )
908
873
  heappush(q.queue, release)
909
- wait_to_next_minute(
910
- get_dt_now(
911
- tz=dynamic("tz", extras=self.extras), offset=offset
912
- )
913
- )
874
+ wait_until_next_minute(get_dt_now(offset=offset))
914
875
 
915
876
  # WARNING: I already call queue poking again because issue
916
877
  # about the every minute crontab.
@@ -943,7 +904,7 @@ class Workflow(BaseModel):
943
904
 
944
905
  def execute_job(
945
906
  self,
946
- job_id: str,
907
+ job: Job,
947
908
  params: DictData,
948
909
  *,
949
910
  result: Result | None = None,
@@ -956,10 +917,9 @@ class Workflow(BaseModel):
956
917
  model. It different with `self.execute` because this method run only
957
918
  one job and return with context of this job data.
958
919
 
959
- :raise WorkflowException: If execute with not exist job's ID.
960
920
  :raise WorkflowException: If the job execution raise JobException.
961
921
 
962
- :param job_id: A job ID.
922
+ :param job: (Job) A job model that want to execute.
963
923
  :param params: (DictData) A parameter data.
964
924
  :param result: (Result) A Result instance for return context and status.
965
925
  :param event: (Event) An Event manager instance that use to cancel this
@@ -967,48 +927,37 @@ class Workflow(BaseModel):
967
927
 
968
928
  :rtype: Result
969
929
  """
970
- if result is None: # pragma: no cov
971
- result: Result = Result(run_id=gen_id(self.name, unique=True))
930
+ result: Result = result or Result(run_id=gen_id(self.name, unique=True))
972
931
 
973
- # VALIDATE: check a job ID that exists in this workflow or not.
974
- if job_id not in self.jobs:
975
- raise WorkflowException(
976
- f"The job: {job_id!r} does not exists in {self.name!r} "
977
- f"workflow."
978
- )
979
-
980
- job: Job = self.job(name=job_id)
981
932
  if job.is_skipped(params=params):
982
- result.trace.info(f"[WORKFLOW]: Skip job: {job_id!r}")
933
+ result.trace.info(f"[WORKFLOW]: Skip Job: {job.id!r}")
983
934
  job.set_outputs(output={"skipped": True}, to=params)
984
935
  return result.catch(status=SKIP, context=params)
985
936
 
986
- if event and event.is_set(): # pragma: no cov
987
- raise WorkflowException(
988
- "Workflow job was canceled from event that had set before "
989
- "job execution."
937
+ if event and event.is_set():
938
+ return result.catch(
939
+ status=CANCEL,
940
+ context={
941
+ "errors": WorkflowException(
942
+ "Workflow job was canceled because event was set."
943
+ ).to_dict(),
944
+ },
990
945
  )
991
946
 
992
- try:
993
- result.trace.info(f"[WORKFLOW]: Execute Job: {job_id!r}")
994
- rs: Result = job.execute(
995
- params=params,
996
- run_id=result.run_id,
997
- parent_run_id=result.parent_run_id,
998
- event=event,
947
+ result.trace.info(f"[WORKFLOW]: Execute Job: {job.id!r}")
948
+ rs: Result = job.execute(
949
+ params=params,
950
+ run_id=result.run_id,
951
+ parent_run_id=result.parent_run_id,
952
+ event=event,
953
+ )
954
+ job.set_outputs(rs.context, to=params)
955
+ if rs.status in (FAILED, CANCEL):
956
+ error_msg: str = (
957
+ f"Job, {job.id!r}, return `{rs.status.name}` status."
999
958
  )
1000
- job.set_outputs(rs.context, to=params)
1001
- except (JobException, UtilException) as e:
1002
- result.trace.error(f"[WORKFLOW]: {e.__class__.__name__}: {e}")
1003
- raise WorkflowException(
1004
- f"Job {job_id!r} raise {e.__class__.__name__}: {e}"
1005
- ) from None
1006
-
1007
- if rs.status == FAILED:
1008
- error_msg: str = f"Workflow job, {job.id!r}, return FAILED status."
1009
- result.trace.warning(f"[WORKFLOW]: {error_msg}")
1010
959
  return result.catch(
1011
- status=FAILED,
960
+ status=rs.status,
1012
961
  context={
1013
962
  "errors": WorkflowException(error_msg).to_dict(),
1014
963
  **params,
@@ -1073,7 +1022,7 @@ class Workflow(BaseModel):
1073
1022
  extras=self.extras,
1074
1023
  )
1075
1024
  context: DictData = self.parameterize(params)
1076
- result.trace.info(f"[WORKFLOW]: Execute: {self.name!r} ...")
1025
+ result.trace.info(f"[WORKFLOW]: Execute: {self.name!r}")
1077
1026
  if not self.jobs:
1078
1027
  result.trace.warning(f"[WORKFLOW]: {self.name!r} does not set jobs")
1079
1028
  return result.catch(status=SUCCESS, context=context)
@@ -1087,7 +1036,9 @@ class Workflow(BaseModel):
1087
1036
  "max_job_exec_timeout", f=timeout, extras=self.extras
1088
1037
  )
1089
1038
  event: Event = event or Event()
1090
- result.trace.debug(f"... Run {self.name!r} with non-threading.")
1039
+ result.trace.debug(
1040
+ f"[WORKFLOW]: ... Run {self.name!r} with non-threading."
1041
+ )
1091
1042
  max_job_parallel: int = dynamic(
1092
1043
  "max_job_parallel", f=max_job_parallel, extras=self.extras
1093
1044
  )
@@ -1127,7 +1078,7 @@ class Workflow(BaseModel):
1127
1078
  futures.append(
1128
1079
  executor.submit(
1129
1080
  self.execute_job,
1130
- job_id=job_id,
1081
+ job=job,
1131
1082
  params=context,
1132
1083
  result=result,
1133
1084
  event=event,
@@ -1140,7 +1091,7 @@ class Workflow(BaseModel):
1140
1091
  futures.append(
1141
1092
  executor.submit(
1142
1093
  self.execute_job,
1143
- job_id=job_id,
1094
+ job=job,
1144
1095
  params=context,
1145
1096
  result=result,
1146
1097
  event=event,
@@ -1148,9 +1099,6 @@ class Workflow(BaseModel):
1148
1099
  )
1149
1100
  time.sleep(0.025)
1150
1101
  elif (future := futures.pop(0)).done() or future.cancelled():
1151
- if e := future.exception():
1152
- result.trace.error(f"[WORKFLOW]: {e}")
1153
- raise WorkflowException(str(e))
1154
1102
  job_queue.put(job_id)
1155
1103
  elif future.running() or "state=pending" in str(future):
1156
1104
  time.sleep(0.075)
@@ -1159,8 +1107,10 @@ class Workflow(BaseModel):
1159
1107
  else: # pragma: no cov
1160
1108
  job_queue.put(job_id)
1161
1109
  futures.insert(0, future)
1110
+ time.sleep(0.025)
1162
1111
  result.trace.warning(
1163
- f"... Execution non-threading not handle: {future}."
1112
+ f"[WORKFLOW]: ... Execution non-threading not "
1113
+ f"handle: {future}."
1164
1114
  )
1165
1115
 
1166
1116
  job_queue.task_done()
@@ -1168,16 +1118,7 @@ class Workflow(BaseModel):
1168
1118
  if not_timeout_flag:
1169
1119
  job_queue.join()
1170
1120
  for future in as_completed(futures):
1171
- try:
1172
- future.result()
1173
- except WorkflowException as e:
1174
- result.trace.error(f"[WORKFLOW]: Handler:{NEWLINE}{e}")
1175
- return result.catch(
1176
- status=FAILED,
1177
- context={
1178
- "errors": WorkflowException(str(e)).to_dict()
1179
- },
1180
- )
1121
+ future.result()
1181
1122
  return result.catch(
1182
1123
  status=FAILED if "errors" in result.context else SUCCESS,
1183
1124
  context=context,