ddeutil-workflow 0.0.41__py3-none-any.whl → 0.0.42__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,6 +3,7 @@
3
3
  # Licensed under the MIT License. See LICENSE in the project root for
4
4
  # license information.
5
5
  # ------------------------------------------------------------------------------
6
+ # [x] Use dynamic config`
6
7
  """A Workflow module that is the core model of this package."""
7
8
  from __future__ import annotations
8
9
 
@@ -24,14 +25,14 @@ from textwrap import dedent
24
25
  from threading import Event
25
26
  from typing import Optional
26
27
 
27
- from pydantic import BaseModel, ConfigDict, Field
28
+ from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
28
29
  from pydantic.dataclasses import dataclass
29
30
  from pydantic.functional_validators import field_validator, model_validator
30
31
  from typing_extensions import Self
31
32
 
32
33
  from .__cron import CronJob, CronRunner
33
34
  from .__types import DictData, TupleStr
34
- from .conf import Loader, SimLoad, config, get_logger
35
+ from .conf import Loader, SimLoad, dynamic, get_logger
35
36
  from .cron import On
36
37
  from .exceptions import JobException, WorkflowException
37
38
  from .job import Job, TriggerState
@@ -92,11 +93,15 @@ class Release:
92
93
  return f"{self.date:%Y-%m-%d %H:%M:%S}"
93
94
 
94
95
  @classmethod
95
- def from_dt(cls, dt: datetime | str) -> Self:
96
+ def from_dt(
97
+ cls, dt: datetime | str, *, externals: Optional[DictData] = None
98
+ ) -> Self:
96
99
  """Construct Release via datetime object only.
97
100
 
98
101
  :param dt: (datetime | str) A datetime object or string that want to
99
102
  construct to the Release object.
103
+ :param externals: An external parameters that want to pass to override
104
+ config.
100
105
 
101
106
  :raise TypeError: If the type of the dt argument does not valid with
102
107
  datetime or str object.
@@ -115,7 +120,9 @@ class Release:
115
120
  date=dt,
116
121
  offset=0,
117
122
  end_date=dt + timedelta(days=1),
118
- runner=CronJob("* * * * *").schedule(dt.replace(tzinfo=config.tz)),
123
+ runner=CronJob("* * * * *").schedule(
124
+ dt.replace(tzinfo=dynamic("tz", extras=externals))
125
+ ),
119
126
  )
120
127
 
121
128
  def __eq__(self, other: Release | datetime) -> bool:
@@ -150,6 +157,10 @@ class ReleaseQueue:
150
157
  queue: list[Release] = field(default_factory=list)
151
158
  running: list[Release] = field(default_factory=list)
152
159
  complete: list[Release] = field(default_factory=list)
160
+ extras: DictData = Field(
161
+ default_factory=dict,
162
+ description="An extra override config values.",
163
+ )
153
164
 
154
165
  @classmethod
155
166
  def from_list(
@@ -232,8 +243,8 @@ class ReleaseQueue:
232
243
 
233
244
  # NOTE: Remove complete queue on workflow that keep more than the
234
245
  # maximum config.
235
- num_complete_delete: int = (
236
- len(self.complete) - config.max_queue_complete_hist
246
+ num_complete_delete: int = len(self.complete) - dynamic(
247
+ "max_queue_complete_hist", extras=self.extras
237
248
  )
238
249
 
239
250
  if num_complete_delete > 0:
@@ -252,6 +263,11 @@ class Workflow(BaseModel):
252
263
  execute method on it.
253
264
  """
254
265
 
266
+ extras: DictData = Field(
267
+ default_factory=dict,
268
+ description="An extra override config values.",
269
+ )
270
+
255
271
  name: str = Field(description="A workflow name.")
256
272
  desc: Optional[str] = Field(
257
273
  default=None,
@@ -271,30 +287,26 @@ class Workflow(BaseModel):
271
287
  default_factory=dict,
272
288
  description="A mapping of job ID and job model that already loaded.",
273
289
  )
274
- extras: DictData = Field(
275
- default_factory=dict,
276
- description="An extra override values.",
277
- )
278
290
 
279
291
  @classmethod
280
- def from_loader(
292
+ def from_conf(
281
293
  cls,
282
294
  name: str,
283
- externals: DictData | None = None,
295
+ extras: DictData | None = None,
284
296
  ) -> Self:
285
297
  """Create Workflow instance from the Loader object that only receive
286
298
  an input workflow name. The loader object will use this workflow name to
287
299
  searching configuration data of this workflow model in conf path.
288
300
 
289
301
  :param name: A workflow name that want to pass to Loader object.
290
- :param externals: An external parameters that want to pass to Loader
302
+ :param extras: An extra parameters that want to pass to Loader
291
303
  object.
292
304
 
293
305
  :raise ValueError: If the type does not match with current object.
294
306
 
295
307
  :rtype: Self
296
308
  """
297
- loader: Loader = Loader(name, externals=(externals or {}))
309
+ loader: Loader = Loader(name, externals=(extras or {}))
298
310
 
299
311
  # NOTE: Validate the config type match with current connection model
300
312
  if loader.type != cls.__name__:
@@ -302,12 +314,10 @@ class Workflow(BaseModel):
302
314
 
303
315
  loader_data: DictData = copy.deepcopy(loader.data)
304
316
  loader_data["name"] = name.replace(" ", "_")
305
- if externals: # pragma: no cov
306
- loader_data["extras"] = externals
317
+ if extras: # pragma: no cov
318
+ loader_data["extras"] = extras
307
319
 
308
- cls.__bypass_on__(
309
- loader_data, path=loader.conf_path, externals=externals
310
- )
320
+ cls.__bypass_on__(loader_data, path=loader.conf_path, extras=extras)
311
321
  return cls.model_validate(obj=loader_data)
312
322
 
313
323
  @classmethod
@@ -315,7 +325,7 @@ class Workflow(BaseModel):
315
325
  cls,
316
326
  name: str,
317
327
  path: Path,
318
- externals: DictData | None = None,
328
+ extras: DictData | None = None,
319
329
  ) -> Self:
320
330
  """Create Workflow instance from the specific path. The loader object
321
331
  will use this workflow name and path to searching configuration data of
@@ -323,7 +333,7 @@ class Workflow(BaseModel):
323
333
 
324
334
  :param name: (str) A workflow name that want to pass to Loader object.
325
335
  :param path: (Path) A config path that want to search.
326
- :param externals: An external parameters that want to pass to Loader
336
+ :param extras: An extra parameters that want to pass to Loader
327
337
  object.
328
338
 
329
339
  :raise ValueError: If the type does not match with current object.
@@ -331,7 +341,7 @@ class Workflow(BaseModel):
331
341
  :rtype: Self
332
342
  """
333
343
  loader: SimLoad = SimLoad(
334
- name, conf_path=path, externals=(externals or {})
344
+ name, conf_path=path, externals=(extras or {})
335
345
  )
336
346
  # NOTE: Validate the config type match with current connection model
337
347
  if loader.type != cls.__name__:
@@ -339,10 +349,10 @@ class Workflow(BaseModel):
339
349
 
340
350
  loader_data: DictData = copy.deepcopy(loader.data)
341
351
  loader_data["name"] = name.replace(" ", "_")
342
- if externals: # pragma: no cov
343
- loader_data["extras"] = externals
352
+ if extras: # pragma: no cov
353
+ loader_data["extras"] = extras
344
354
 
345
- cls.__bypass_on__(loader_data, path=path, externals=externals)
355
+ cls.__bypass_on__(loader_data, path=path, extras=extras)
346
356
  return cls.model_validate(obj=loader_data)
347
357
 
348
358
  @classmethod
@@ -350,14 +360,15 @@ class Workflow(BaseModel):
350
360
  cls,
351
361
  data: DictData,
352
362
  path: Path,
353
- externals: DictData | None = None,
363
+ extras: DictData | None = None,
354
364
  ) -> DictData:
355
365
  """Bypass the on data to loaded config data.
356
366
 
357
367
  :param data: A data to construct to this Workflow model.
358
368
  :param path: A config path.
359
- :param externals: An external parameters that want to pass to SimLoad
369
+ :param extras: An extra parameters that want to pass to SimLoad
360
370
  object.
371
+
361
372
  :rtype: DictData
362
373
  """
363
374
  if on := data.pop("on", []):
@@ -370,7 +381,7 @@ class Workflow(BaseModel):
370
381
  # field.
371
382
  data["on"] = [
372
383
  (
373
- SimLoad(n, conf_path=path, externals=(externals or {})).data
384
+ SimLoad(n, conf_path=path, externals=(extras or {})).data
374
385
  if isinstance(n, str)
375
386
  else n
376
387
  )
@@ -403,7 +414,11 @@ class Workflow(BaseModel):
403
414
  return dedent(value)
404
415
 
405
416
  @field_validator("on", mode="after")
406
- def __on_no_dup_and_reach_limit__(cls, value: list[On]) -> list[On]:
417
+ def __on_no_dup_and_reach_limit__(
418
+ cls,
419
+ value: list[On],
420
+ info: ValidationInfo,
421
+ ) -> list[On]:
407
422
  """Validate the on fields should not contain duplicate values and if it
408
423
  contains the every minute value more than one value, it will remove to
409
424
  only one value.
@@ -427,10 +442,12 @@ class Workflow(BaseModel):
427
442
  # "only one value in the on field."
428
443
  # )
429
444
 
430
- if len(set_ons) > config.max_on_per_workflow:
445
+ extras: DictData = info.data.get("extras", {})
446
+ if len(set_ons) > (
447
+ conf := dynamic("max_on_per_workflow", extras=extras)
448
+ ):
431
449
  raise ValueError(
432
- f"The number of the on should not more than "
433
- f"{config.max_on_per_workflow} crontab."
450
+ f"The number of the on should not more than {conf} crontabs."
434
451
  )
435
452
  return value
436
453
 
@@ -604,9 +621,11 @@ class Workflow(BaseModel):
604
621
  release_params: DictData = {
605
622
  "release": {
606
623
  "logical_date": release.date,
607
- "execute_date": datetime.now(tz=config.tz),
624
+ "execute_date": datetime.now(
625
+ tz=dynamic("tz", extras=self.extras)
626
+ ),
608
627
  "run_id": result.run_id,
609
- "timezone": config.tz,
628
+ "timezone": dynamic("tz", extras=self.extras),
610
629
  }
611
630
  }
612
631
 
@@ -616,7 +635,7 @@ class Workflow(BaseModel):
616
635
  # ... {"params": ..., "jobs": ...}
617
636
  #
618
637
  self.execute(
619
- params=param2template(params, release_params),
638
+ params=param2template(params, release_params, extras=self.extras),
620
639
  result=result,
621
640
  parent_run_id=result.parent_run_id,
622
641
  )
@@ -656,7 +675,6 @@ class Workflow(BaseModel):
656
675
  return result.catch(
657
676
  status=Status.SUCCESS,
658
677
  context={
659
- # NOTE: Update the real params that pass in this method.
660
678
  "params": params,
661
679
  "release": {
662
680
  "type": release.type,
@@ -700,10 +718,11 @@ class Workflow(BaseModel):
700
718
  for on in self.on:
701
719
 
702
720
  runner: CronRunner = on.next(
703
- get_dt_now(tz=config.tz, offset=offset).replace(microsecond=0)
721
+ get_dt_now(
722
+ tz=dynamic("tz", extras=self.extras), offset=offset
723
+ ).replace(microsecond=0)
704
724
  )
705
725
 
706
- # NOTE: Skip this runner date if it more than the end date.
707
726
  if runner.date > end_date:
708
727
  continue
709
728
 
@@ -730,7 +749,6 @@ class Workflow(BaseModel):
730
749
  if runner.date > end_date:
731
750
  continue
732
751
 
733
- # NOTE: Push the Release object to queue.
734
752
  heappush(queue.queue, workflow_release)
735
753
 
736
754
  return queue
@@ -745,6 +763,7 @@ class Workflow(BaseModel):
745
763
  audit: Audit | None = None,
746
764
  force_run: bool = False,
747
765
  timeout: int = 1800,
766
+ max_poking_pool_worker: int = 4,
748
767
  ) -> Result:
749
768
  """Poke function with a start datetime value that will pass to its
750
769
  `on` field on the threading executor pool for execute the `release`
@@ -765,6 +784,7 @@ class Workflow(BaseModel):
765
784
  that release was pointed.
766
785
  :param timeout: A second value for timeout while waiting all futures
767
786
  run completely.
787
+ :param max_poking_pool_worker: The maximum poking pool worker.
768
788
 
769
789
  :rtype: Result
770
790
  :return: A list of all results that return from `self.release` method.
@@ -780,8 +800,6 @@ class Workflow(BaseModel):
780
800
  "The period of poking should be int and grater or equal than 1."
781
801
  )
782
802
 
783
- # NOTE: If this workflow does not set the on schedule, it will return
784
- # empty result.
785
803
  if len(self.on) == 0:
786
804
  result.trace.info(
787
805
  f"[POKING]: {self.name!r} does not have any schedule to run."
@@ -789,15 +807,15 @@ class Workflow(BaseModel):
789
807
  return result.catch(status=Status.SUCCESS, context={"outputs": []})
790
808
 
791
809
  # NOTE: Create the current date that change microsecond to 0
792
- current_date: datetime = datetime.now(tz=config.tz).replace(
793
- microsecond=0
794
- )
810
+ current_date: datetime = datetime.now(
811
+ tz=dynamic("tz", extras=self.extras)
812
+ ).replace(microsecond=0)
795
813
 
796
814
  # NOTE: Create start_date and offset variables.
797
815
  if start_date and start_date <= current_date:
798
- start_date = start_date.replace(tzinfo=config.tz).replace(
799
- microsecond=0
800
- )
816
+ start_date = start_date.replace(
817
+ tzinfo=dynamic("tz", extras=self.extras)
818
+ ).replace(microsecond=0)
801
819
  offset: float = (current_date - start_date).total_seconds()
802
820
  else:
803
821
  # NOTE: Force change start date if it gathers than the current date,
@@ -837,7 +855,11 @@ class Workflow(BaseModel):
837
855
  # NOTE: Start create the thread pool executor for running this poke
838
856
  # process.
839
857
  with ThreadPoolExecutor(
840
- max_workers=config.max_poking_pool_worker,
858
+ max_workers=dynamic(
859
+ "max_poking_pool_worker",
860
+ f=max_poking_pool_worker,
861
+ extras=self.extras,
862
+ ),
841
863
  thread_name_prefix="wf_poking_",
842
864
  ) as executor:
843
865
 
@@ -848,14 +870,22 @@ class Workflow(BaseModel):
848
870
  # NOTE: Pop the latest Release object from the release queue.
849
871
  release: Release = heappop(q.queue)
850
872
 
851
- if reach_next_minute(release.date, tz=config.tz, offset=offset):
873
+ if reach_next_minute(
874
+ release.date,
875
+ tz=dynamic("tz", extras=self.extras),
876
+ offset=offset,
877
+ ):
852
878
  result.trace.debug(
853
879
  f"[POKING]: The latest release, "
854
880
  f"{release.date:%Y-%m-%d %H:%M:%S}, is not able to run "
855
881
  f"on this minute"
856
882
  )
857
883
  heappush(q.queue, release)
858
- wait_to_next_minute(get_dt_now(tz=config.tz, offset=offset))
884
+ wait_to_next_minute(
885
+ get_dt_now(
886
+ tz=dynamic("tz", extras=self.extras), offset=offset
887
+ )
888
+ )
859
889
 
860
890
  # WARNING: I already call queue poking again because issue
861
891
  # about the every minute crontab.
@@ -935,12 +965,6 @@ class Workflow(BaseModel):
935
965
  "job execution."
936
966
  )
937
967
 
938
- # IMPORTANT:
939
- # This execution change all job running IDs to the current workflow
940
- # running ID, but it still trac log to the same parent running ID
941
- # (with passing `run_id` and `parent_run_id` to the job execution
942
- # arguments).
943
- #
944
968
  try:
945
969
  job: Job = self.jobs[job_id]
946
970
  if job.is_skipped(params=params):
@@ -975,8 +999,10 @@ class Workflow(BaseModel):
975
999
  *,
976
1000
  run_id: str | None = None,
977
1001
  parent_run_id: str | None = None,
978
- timeout: int = 0,
1002
+ timeout: int = 600,
979
1003
  result: Result | None = None,
1004
+ max_job_parallel: int = 2,
1005
+ event: Event | None = None,
980
1006
  ) -> Result:
981
1007
  """Execute workflow with passing a dynamic parameters to all jobs that
982
1008
  included in this workflow model with ``jobs`` field.
@@ -1001,6 +1027,9 @@ class Workflow(BaseModel):
1001
1027
  limit time. (default: 0)
1002
1028
  :param result: (Result) A result object for keeping context and status
1003
1029
  data.
1030
+ :param max_job_parallel: (int) The maximum threads of job execution.
1031
+ :param event: (Event) An event manager that pass to the
1032
+ PoolThreadExecutor.
1004
1033
 
1005
1034
  :rtype: Result
1006
1035
  """
@@ -1038,13 +1067,19 @@ class Workflow(BaseModel):
1038
1067
  context: DictData = self.parameterize(params)
1039
1068
  status: Status = Status.SUCCESS
1040
1069
  try:
1041
- if config.max_job_parallel == 1:
1070
+ if (
1071
+ dynamic(
1072
+ "max_job_parallel", f=max_job_parallel, extras=self.extras
1073
+ )
1074
+ == 1
1075
+ ):
1042
1076
  self.__exec_non_threading(
1043
1077
  result=result,
1044
1078
  context=context,
1045
1079
  ts=ts,
1046
1080
  job_queue=jq,
1047
1081
  timeout=timeout,
1082
+ event=event,
1048
1083
  )
1049
1084
  else:
1050
1085
  self.__exec_threading(
@@ -1053,6 +1088,7 @@ class Workflow(BaseModel):
1053
1088
  ts=ts,
1054
1089
  job_queue=jq,
1055
1090
  timeout=timeout,
1091
+ event=event,
1056
1092
  )
1057
1093
  except WorkflowException as err:
1058
1094
  status = Status.FAILED
@@ -1067,8 +1103,9 @@ class Workflow(BaseModel):
1067
1103
  ts: float,
1068
1104
  job_queue: Queue,
1069
1105
  *,
1070
- timeout: int = 0,
1106
+ timeout: int = 600,
1071
1107
  thread_timeout: int = 1800,
1108
+ event: Event | None = None,
1072
1109
  ) -> DictData:
1073
1110
  """Workflow execution by threading strategy that use multithreading.
1074
1111
 
@@ -1082,18 +1119,19 @@ class Workflow(BaseModel):
1082
1119
  :param job_queue: (Queue) A job queue object.
1083
1120
  :param timeout: (int) A second value unit that bounding running time.
1084
1121
  :param thread_timeout: A timeout to waiting all futures complete.
1122
+ :param event: (Event) An event manager that pass to the
1123
+ PoolThreadExecutor.
1085
1124
 
1086
1125
  :rtype: DictData
1087
1126
  """
1088
1127
  not_timeout_flag: bool = True
1089
- timeout: int = timeout or config.max_job_exec_timeout
1090
- event: Event = Event()
1128
+ timeout: int = dynamic(
1129
+ "max_job_exec_timeout", f=timeout, extras=self.extras
1130
+ )
1131
+ event: Event = event or Event()
1091
1132
  result.trace.debug(f"[WORKFLOW]: Run {self.name!r} with threading.")
1092
-
1093
- # IMPORTANT: The job execution can run parallel and waiting by
1094
- # needed.
1095
1133
  with ThreadPoolExecutor(
1096
- max_workers=config.max_job_parallel,
1134
+ max_workers=dynamic("max_job_parallel", extras=self.extras),
1097
1135
  thread_name_prefix="wf_exec_threading_",
1098
1136
  ) as executor:
1099
1137
  futures: list[Future] = []
@@ -1153,7 +1191,6 @@ class Workflow(BaseModel):
1153
1191
  result.trace.error(f"[WORKFLOW]: {err}")
1154
1192
  raise WorkflowException(str(err))
1155
1193
 
1156
- # NOTE: This getting result does not do anything.
1157
1194
  future.result()
1158
1195
 
1159
1196
  return context
@@ -1174,7 +1211,8 @@ class Workflow(BaseModel):
1174
1211
  ts: float,
1175
1212
  job_queue: Queue,
1176
1213
  *,
1177
- timeout: int = 0,
1214
+ timeout: int = 600,
1215
+ event: Event | None = None,
1178
1216
  ) -> DictData:
1179
1217
  """Workflow execution with non-threading strategy that use sequential
1180
1218
  job running and waiting previous job was run successful.
@@ -1187,12 +1225,16 @@ class Workflow(BaseModel):
1187
1225
  :param ts: (float) A start timestamp that use for checking execute time
1188
1226
  should time out.
1189
1227
  :param timeout: (int) A second value unit that bounding running time.
1228
+ :param event: (Event) An event manager that pass to the
1229
+ PoolThreadExecutor.
1190
1230
 
1191
1231
  :rtype: DictData
1192
1232
  """
1193
1233
  not_timeout_flag: bool = True
1194
- timeout: int = timeout or config.max_job_exec_timeout
1195
- event: Event = Event()
1234
+ timeout: int = dynamic(
1235
+ "max_job_exec_timeout", f=timeout, extras=self.extras
1236
+ )
1237
+ event: Event = event or Event()
1196
1238
  future: Future | None = None
1197
1239
  result.trace.debug(f"[WORKFLOW]: Run {self.name!r} with non-threading.")
1198
1240
 
@@ -1207,7 +1249,6 @@ class Workflow(BaseModel):
1207
1249
  job_id: str = job_queue.get()
1208
1250
  job: Job = self.jobs[job_id]
1209
1251
 
1210
- # NOTE: Waiting dependency job run successful before release.
1211
1252
  if (check := job.check_needs(context["jobs"])).is_waiting():
1212
1253
  job_queue.task_done()
1213
1254
  job_queue.put(job_id)
@@ -1256,13 +1297,9 @@ class Workflow(BaseModel):
1256
1297
  f"that not running."
1257
1298
  )
1258
1299
 
1259
- # NOTE: Mark this job queue done.
1260
1300
  job_queue.task_done()
1261
1301
 
1262
1302
  if not_timeout_flag:
1263
-
1264
- # NOTE: Wait for all items to finish processing by `task_done()`
1265
- # method.
1266
1303
  job_queue.join()
1267
1304
  executor.shutdown()
1268
1305
  return context
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ddeutil-workflow
3
- Version: 0.0.41
3
+ Version: 0.0.42
4
4
  Summary: Lightweight workflow orchestration
5
5
  Author-email: ddeutils <korawich.anu@gmail.com>
6
6
  License: MIT
@@ -264,7 +264,7 @@ it will use default value and do not raise any error to you.
264
264
  | Name | Component | Default | Description |
265
265
  |:-----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------|
266
266
  | **ROOT_PATH** | Core | `.` | The root path of the workflow application. |
267
- | **REGISTRY** | Core | `.` | List of importable string for the call stage. |
267
+ | **REGISTRY_CALLER** | Core | `.` | List of importable string for the call stage. |
268
268
  | **REGISTRY_FILTER** | Core | `ddeutil.workflow.templates` | List of importable string for the filter template. |
269
269
  | **CONF_PATH** | Core | `conf` | The config path that keep all template `.yaml` files. |
270
270
  | **TIMEZONE** | Core | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
@@ -272,20 +272,17 @@ it will use default value and do not raise any error to you.
272
272
  | **STAGE_RAISE_ERROR** | Core | `false` | A flag that all stage raise StageException from stage execution. |
273
273
  | **JOB_DEFAULT_ID** | Core | `false` | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
274
274
  | **JOB_RAISE_ERROR** | Core | `true` | A flag that all job raise JobException from job strategy execution. |
275
- | **MAX_NUM_POKING** | Core | `4` | . |
276
- | **MAX_JOB_PARALLEL** | Core | `2` | The maximum job number that able to run parallel in workflow executor. |
277
- | **MAX_JOB_EXEC_TIMEOUT** | Core | `600` | |
278
275
  | **MAX_CRON_PER_WORKFLOW** | Core | `5` | |
279
276
  | **MAX_QUEUE_COMPLETE_HIST** | Core | `16` | |
280
277
  | **GENERATE_ID_SIMPLE_MODE** | Core | `true` | A flog that enable generating ID with `md5` algorithm. |
281
- | **PATH** | Log | `./logs` | The log path of the workflow saving log. |
278
+ | **TRACE_PATH** | Log | `./logs` | The log path of the workflow saving log. |
282
279
  | **DEBUG_MODE** | Log | `true` | A flag that enable logging with debug level mode. |
283
280
  | **FORMAT** | Log | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | |
284
281
  | **FORMAT_FILE** | Log | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})` | |
285
282
  | **DATETIME_FORMAT** | Log | `%Y-%m-%d %H:%M:%S` | |
286
- | **ENABLE_WRITE** | Log | `false` | |
287
- | **PATH** | Audit | `./audits` | |
288
- | **ENABLE_WRITE** | Audit | `true` | A flag that enable logging object saving log to its destination. |
283
+ | **TRACE_ENABLE_WRITE** | Log | `false` | |
284
+ | **AUDIT_PATH** | Log | `./audits` | |
285
+ | **AUDIT_ENABLE_WRITE** | Log | `true` | A flag that enable logging object saving log to its destination. |
289
286
  | **MAX_PROCESS** | App | `2` | The maximum process worker number that run in scheduler app module. |
290
287
  | **MAX_SCHEDULE_PER_PROCESS** | App | `100` | A schedule per process that run parallel. |
291
288
  | **STOP_BOUNDARY_DELTA** | App | `'{"minutes": 5, "seconds": 20}'` | A time delta value that use to stop scheduler app in json string format. |
@@ -0,0 +1,30 @@
1
+ ddeutil/workflow/__about__.py,sha256=B3KiwXw9ATZmMG1vER6qdPImMLkQPPjYkRvYepuIhF4,28
2
+ ddeutil/workflow/__cron.py,sha256=h8rLeIUAAEB2SdZ4Jhch7LU1Yl3bbJ-iNNJ3tQ0eYVM,28095
3
+ ddeutil/workflow/__init__.py,sha256=cYWwG2utpsYvdwqvkFSRWi_Q6gylDgNQBcIWcF5NFs4,1861
4
+ ddeutil/workflow/__types.py,sha256=8jBdbfb3aZSetjz0mvNrpGHwwxJff7mK8_4v41cLqlc,4316
5
+ ddeutil/workflow/conf.py,sha256=lDzWiVSNlNAhTzxbNIhIbQAIF1ggbmetAp0yn2fgnsc,12385
6
+ ddeutil/workflow/cron.py,sha256=80SijzMdDOBxTWRsiF-Fmuz7Ym7leY0XT2lzRAPGdXc,8781
7
+ ddeutil/workflow/exceptions.py,sha256=fO37f9p7lOjIJgVOpKE_1X44yJTwBepyukZV9a7NNm4,1241
8
+ ddeutil/workflow/job.py,sha256=vsayKMzwKDpjchgYQnbshZHnp-vuM9CobpFWhUJETRU,30315
9
+ ddeutil/workflow/logs.py,sha256=RkM5o_JPoWhFY7NrbYAARZQWjLC62YB_FYzTTcyDp8U,19816
10
+ ddeutil/workflow/params.py,sha256=Mv-D2DY5inm1ug0lsgCPDkO5wT_AUhc5XEF5jxgDx6U,8036
11
+ ddeutil/workflow/result.py,sha256=ynZB0g_vEEXn24034J-hatjNWDBmRAj38S8SqGRM-8I,4029
12
+ ddeutil/workflow/reusables.py,sha256=AtZO83HDFu1uK_azUinv5d8jsA36f2i3n_tqMrolbvc,17529
13
+ ddeutil/workflow/scheduler.py,sha256=wFEgcnxtgF-8y5otv8RqT1MuBttZl7mu-bBu5ffwV_Y,27534
14
+ ddeutil/workflow/stages.py,sha256=prw1-za1zwYehbrjeAnoJ79GxpfTqdKLsI2PY0OuSlY,48417
15
+ ddeutil/workflow/utils.py,sha256=sblje9qOtejCHVt8EVrbC0KY98vKqvxccaR5HIkRiTA,7363
16
+ ddeutil/workflow/workflow.py,sha256=Y1D5arh2KSobkIZGJ1fWSTe15heURi9OhhdfIr0jHyo,50591
17
+ ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
18
+ ddeutil/workflow/api/api.py,sha256=b-bMg0aRsEqt8Qb2hNUtamEt2Fq2CgNotF2oXSAdDu8,5226
19
+ ddeutil/workflow/api/log.py,sha256=NMTnOnsBrDB5129329xF2myLdrb-z9k1MQrmrP7qXJw,1818
20
+ ddeutil/workflow/api/repeat.py,sha256=cycd1-91j-4v6uY1SkrZHd9l95e-YgVC4UCSNNFuGJ8,5277
21
+ ddeutil/workflow/api/routes/__init__.py,sha256=qoGtOMyVgQ5nTUc8J8wH27A8isaxl3IFCX8qoyibeCY,484
22
+ ddeutil/workflow/api/routes/job.py,sha256=YVta083i8vU8-o4WdKFwDpfdC9vN1dZ6goZSmNlQXHA,1954
23
+ ddeutil/workflow/api/routes/logs.py,sha256=TeRDrEelbKS2Hu_EovgLh0bOdmSv9mfnrIZsrE7uPD4,5353
24
+ ddeutil/workflow/api/routes/schedules.py,sha256=rUWBm5RgLS1PNBHSWwWXJ0l-c5mYWfl9os0BA9_OTEw,4810
25
+ ddeutil/workflow/api/routes/workflows.py,sha256=ctgQGxXfpIV6bHFDM9IQ1_qaQHT6n5-HjJ1-D4GKWpc,4527
26
+ ddeutil_workflow-0.0.42.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
27
+ ddeutil_workflow-0.0.42.dist-info/METADATA,sha256=TJp1M40eLXYOInkTpl_XhFOWGHLd0hIHktXQXiFsmEw,18853
28
+ ddeutil_workflow-0.0.42.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
29
+ ddeutil_workflow-0.0.42.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
30
+ ddeutil_workflow-0.0.42.dist-info/RECORD,,
@@ -1,61 +0,0 @@
1
- from __future__ import annotations
2
-
3
- from typing import Optional, Union
4
-
5
- from pydantic import BaseModel, ConfigDict, Field
6
-
7
- from .__types import DictData
8
-
9
-
10
- class ErrorContext(BaseModel): # pragma: no cov
11
- model_config = ConfigDict(arbitrary_types_allowed=True)
12
-
13
- obj: Exception = Field(alias="class")
14
- name: str = Field(description="A name of exception class.")
15
- message: str = Field(description="A exception message.")
16
-
17
-
18
- class OutputContext(BaseModel): # pragma: no cov
19
- outputs: DictData = Field(default_factory=dict)
20
- errors: Optional[ErrorContext] = Field(default=None)
21
- skipped: bool = Field(default=False)
22
-
23
- def is_exception(self) -> bool:
24
- return self.errors is not None
25
-
26
-
27
- class StageContext(BaseModel): # pragma: no cov
28
- stages: dict[str, OutputContext]
29
- errors: Optional[ErrorContext] = Field(default=None)
30
-
31
- def is_exception(self) -> bool:
32
- return self.errors is not None
33
-
34
-
35
- class MatrixContext(StageContext): # pragma: no cov
36
- matrix: DictData = Field(default_factory=dict)
37
-
38
-
39
- MatrixStageContext = dict[
40
- str, Union[MatrixContext, StageContext]
41
- ] # pragma: no cov
42
-
43
-
44
- class StrategyContext(BaseModel): # pragma: no cov
45
- strategies: MatrixStageContext
46
- errors: Optional[ErrorContext] = Field(default=None)
47
-
48
- def is_exception(self) -> bool:
49
- return self.errors is not None
50
-
51
-
52
- StrategyMatrixContext = Union[
53
- StrategyContext, MatrixStageContext
54
- ] # pragma: no cov
55
-
56
-
57
- class JobContext(BaseModel): # pragma: no cov
58
- params: DictData = Field(description="A parameterize value")
59
- jobs: dict[str, StrategyMatrixContext]
60
- errors: Optional[ErrorContext] = Field(default=None)
61
- skipped: bool = Field(default=False)