ddeutil-workflow 0.0.47__py3-none-any.whl → 0.0.48__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +0 -1
- ddeutil/workflow/api/api.py +2 -1
- ddeutil/workflow/api/repeat.py +2 -1
- ddeutil/workflow/api/routes/job.py +1 -1
- ddeutil/workflow/api/routes/schedules.py +2 -1
- ddeutil/workflow/api/routes/workflows.py +2 -2
- ddeutil/workflow/conf.py +34 -39
- ddeutil/workflow/job.py +3 -0
- ddeutil/workflow/logs.py +169 -56
- ddeutil/workflow/result.py +15 -6
- ddeutil/workflow/reusables.py +1 -2
- ddeutil/workflow/scheduler.py +19 -9
- ddeutil/workflow/stages.py +10 -5
- ddeutil/workflow/workflow.py +30 -18
- {ddeutil_workflow-0.0.47.dist-info → ddeutil_workflow-0.0.48.dist-info}/METADATA +26 -26
- ddeutil_workflow-0.0.48.dist-info/RECORD +31 -0
- ddeutil_workflow-0.0.47.dist-info/RECORD +0 -31
- {ddeutil_workflow-0.0.47.dist-info → ddeutil_workflow-0.0.48.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.47.dist-info → ddeutil_workflow-0.0.48.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.47.dist-info → ddeutil_workflow-0.0.48.dist-info}/top_level.txt +0 -0
ddeutil/workflow/__about__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__: str = "0.0.
|
1
|
+
__version__: str = "0.0.48"
|
ddeutil/workflow/__init__.py
CHANGED
ddeutil/workflow/api/api.py
CHANGED
@@ -20,7 +20,8 @@ from fastapi.middleware.gzip import GZipMiddleware
|
|
20
20
|
from fastapi.responses import UJSONResponse
|
21
21
|
|
22
22
|
from ..__about__ import __version__
|
23
|
-
from ..conf import api_config, config
|
23
|
+
from ..conf import api_config, config
|
24
|
+
from ..logs import get_logger
|
24
25
|
from ..scheduler import ReleaseThread, ReleaseThreads
|
25
26
|
from ..workflow import ReleaseQueue, WorkflowTask
|
26
27
|
from .repeat import repeat_at
|
ddeutil/workflow/api/repeat.py
CHANGED
@@ -13,7 +13,8 @@ from functools import wraps
|
|
13
13
|
from starlette.concurrency import run_in_threadpool
|
14
14
|
|
15
15
|
from ..__cron import CronJob
|
16
|
-
from ..conf import config
|
16
|
+
from ..conf import config
|
17
|
+
from ..logs import get_logger
|
17
18
|
|
18
19
|
logger = get_logger("uvicorn.error")
|
19
20
|
|
@@ -12,9 +12,9 @@ from fastapi.responses import UJSONResponse
|
|
12
12
|
from pydantic import BaseModel
|
13
13
|
|
14
14
|
from ...__types import DictData
|
15
|
-
from ...conf import get_logger
|
16
15
|
from ...exceptions import JobException
|
17
16
|
from ...job import Job
|
17
|
+
from ...logs import get_logger
|
18
18
|
from ...result import Result
|
19
19
|
|
20
20
|
logger = get_logger("uvicorn.error")
|
@@ -12,7 +12,8 @@ from fastapi import APIRouter, HTTPException, Request
|
|
12
12
|
from fastapi import status as st
|
13
13
|
from fastapi.responses import UJSONResponse
|
14
14
|
|
15
|
-
from ...conf import config
|
15
|
+
from ...conf import config
|
16
|
+
from ...logs import get_logger
|
16
17
|
from ...scheduler import Schedule
|
17
18
|
|
18
19
|
logger = get_logger("uvicorn.error")
|
@@ -15,8 +15,8 @@ from fastapi.responses import UJSONResponse
|
|
15
15
|
from pydantic import BaseModel
|
16
16
|
|
17
17
|
from ...__types import DictData
|
18
|
-
from ...conf import Loader
|
19
|
-
from ...logs import Audit, get_audit
|
18
|
+
from ...conf import Loader
|
19
|
+
from ...logs import Audit, get_audit, get_logger
|
20
20
|
from ...result import Result
|
21
21
|
from ...workflow import Workflow
|
22
22
|
|
ddeutil/workflow/conf.py
CHANGED
@@ -6,11 +6,10 @@
|
|
6
6
|
from __future__ import annotations
|
7
7
|
|
8
8
|
import json
|
9
|
-
import logging
|
10
9
|
import os
|
11
10
|
from collections.abc import Iterator
|
12
11
|
from datetime import timedelta
|
13
|
-
from functools import cached_property
|
12
|
+
from functools import cached_property
|
14
13
|
from pathlib import Path
|
15
14
|
from typing import Optional, TypeVar
|
16
15
|
from zoneinfo import ZoneInfo
|
@@ -36,7 +35,6 @@ def env(var: str, default: str | None = None) -> str | None: # pragma: no cov
|
|
36
35
|
__all__: TupleStr = (
|
37
36
|
"api_config",
|
38
37
|
"env",
|
39
|
-
"get_logger",
|
40
38
|
"Config",
|
41
39
|
"SimLoad",
|
42
40
|
"Loader",
|
@@ -55,7 +53,8 @@ class Config: # pragma: no cov
|
|
55
53
|
# NOTE: Core
|
56
54
|
@property
|
57
55
|
def root_path(self) -> Path:
|
58
|
-
"""Root path or the project path
|
56
|
+
"""Root path or the project path for this workflow engine that use for
|
57
|
+
combine with `conf_path` value.
|
59
58
|
|
60
59
|
:rtype: Path
|
61
60
|
"""
|
@@ -63,7 +62,7 @@ class Config: # pragma: no cov
|
|
63
62
|
|
64
63
|
@property
|
65
64
|
def conf_path(self) -> Path:
|
66
|
-
"""Config path that
|
65
|
+
"""Config path that keep all workflow template YAML files.
|
67
66
|
|
68
67
|
:rtype: Path
|
69
68
|
"""
|
@@ -71,6 +70,11 @@ class Config: # pragma: no cov
|
|
71
70
|
|
72
71
|
@property
|
73
72
|
def tz(self) -> ZoneInfo:
|
73
|
+
"""Timezone value that return with the `ZoneInfo` object and use for all
|
74
|
+
datetime object in this workflow engine.
|
75
|
+
|
76
|
+
:rtype: ZoneInfo
|
77
|
+
"""
|
74
78
|
return ZoneInfo(env("CORE_TIMEZONE", "UTC"))
|
75
79
|
|
76
80
|
@property
|
@@ -80,7 +84,8 @@ class Config: # pragma: no cov
|
|
80
84
|
# NOTE: Register
|
81
85
|
@property
|
82
86
|
def regis_call(self) -> list[str]:
|
83
|
-
"""Register Caller
|
87
|
+
"""Register Caller that is a list of importable string for the call
|
88
|
+
stage model can get.
|
84
89
|
|
85
90
|
:rtype: list[str]
|
86
91
|
"""
|
@@ -89,7 +94,8 @@ class Config: # pragma: no cov
|
|
89
94
|
|
90
95
|
@property
|
91
96
|
def regis_filter(self) -> list[str]:
|
92
|
-
"""Register Filter
|
97
|
+
"""Register Filter that is a list of importable string for the filter
|
98
|
+
template.
|
93
99
|
|
94
100
|
:rtype: list[str]
|
95
101
|
"""
|
@@ -222,15 +228,13 @@ class SimLoad:
|
|
222
228
|
:param externals: An external parameters
|
223
229
|
|
224
230
|
Noted:
|
225
|
-
|
226
231
|
The config data should have ``type`` key for modeling validation that
|
227
232
|
make this loader know what is config should to do pass to.
|
228
233
|
|
229
234
|
... <identity-key>:
|
230
235
|
... type: <importable-object>
|
231
|
-
... <key-data>: <value-data>
|
232
|
-
...
|
233
|
-
|
236
|
+
... <key-data-1>: <value-data-1>
|
237
|
+
... <key-data-2>: <value-data-2>
|
234
238
|
"""
|
235
239
|
|
236
240
|
def __init__(
|
@@ -253,7 +257,10 @@ class SimLoad:
|
|
253
257
|
|
254
258
|
# VALIDATE: check the data that reading should not empty.
|
255
259
|
if not self.data:
|
256
|
-
raise ValueError(
|
260
|
+
raise ValueError(
|
261
|
+
f"Config {name!r} does not found on conf path: "
|
262
|
+
f"{self.conf_path}."
|
263
|
+
)
|
257
264
|
|
258
265
|
self.data.update(self.externals)
|
259
266
|
|
@@ -299,10 +306,25 @@ class SimLoad:
|
|
299
306
|
|
300
307
|
@classmethod
|
301
308
|
def is_ignore(cls, file: Path, conf_path: Path) -> bool:
|
309
|
+
"""Check this file was ignored.
|
310
|
+
|
311
|
+
:param file: (Path) A file path that want to check.
|
312
|
+
:param conf_path: (Path) A config path that want to read the config
|
313
|
+
ignore file.
|
314
|
+
|
315
|
+
:rtype: bool
|
316
|
+
"""
|
302
317
|
return is_ignored(file, read_ignore(conf_path / ".confignore"))
|
303
318
|
|
304
319
|
@classmethod
|
305
320
|
def filter_yaml(cls, file: Path, name: str | None = None) -> DictData:
|
321
|
+
"""Read a YAML file context from an input file path and specific name.
|
322
|
+
|
323
|
+
:param file: (Path)
|
324
|
+
:param name: (str)
|
325
|
+
|
326
|
+
:rtype: DictData
|
327
|
+
"""
|
306
328
|
if any(file.suffix.endswith(s) for s in (".yml", ".yaml")):
|
307
329
|
values: DictData = YamlFlResolve(file).read()
|
308
330
|
if values is not None:
|
@@ -388,30 +410,3 @@ class Loader(SimLoad):
|
|
388
410
|
conf_path=dynamic("conf_path", extras=externals),
|
389
411
|
externals=externals,
|
390
412
|
)
|
391
|
-
|
392
|
-
|
393
|
-
@lru_cache
|
394
|
-
def get_logger(name: str):
|
395
|
-
"""Return logger object with an input module name.
|
396
|
-
|
397
|
-
:param name: A module name that want to log.
|
398
|
-
"""
|
399
|
-
logger = logging.getLogger(name)
|
400
|
-
|
401
|
-
# NOTE: Developers using this package can then disable all logging just for
|
402
|
-
# this package by;
|
403
|
-
#
|
404
|
-
# `logging.getLogger('ddeutil.workflow').propagate = False`
|
405
|
-
#
|
406
|
-
logger.addHandler(logging.NullHandler())
|
407
|
-
|
408
|
-
formatter = logging.Formatter(
|
409
|
-
fmt=config.log_format,
|
410
|
-
datefmt=config.log_datetime_format,
|
411
|
-
)
|
412
|
-
stream = logging.StreamHandler()
|
413
|
-
stream.setFormatter(formatter)
|
414
|
-
logger.addHandler(stream)
|
415
|
-
|
416
|
-
logger.setLevel(logging.DEBUG if config.debug else logging.INFO)
|
417
|
-
return logger
|
ddeutil/workflow/job.py
CHANGED
@@ -570,6 +570,7 @@ class Job(BaseModel):
|
|
570
570
|
run_id=run_id,
|
571
571
|
parent_run_id=parent_run_id,
|
572
572
|
id_logic=(self.id or "not-set"),
|
573
|
+
extras=self.extras,
|
573
574
|
)
|
574
575
|
|
575
576
|
if self.runs_on.type == RunsOnType.LOCAL:
|
@@ -756,6 +757,7 @@ def local_execute(
|
|
756
757
|
run_id=run_id,
|
757
758
|
parent_run_id=parent_run_id,
|
758
759
|
id_logic=(job.id or "not-set"),
|
760
|
+
extras=job.extras,
|
759
761
|
)
|
760
762
|
event: Event = Event() if event is None else event
|
761
763
|
|
@@ -891,6 +893,7 @@ def self_hosted_execute(
|
|
891
893
|
run_id=run_id,
|
892
894
|
parent_run_id=parent_run_id,
|
893
895
|
id_logic=(job.id or "not-set"),
|
896
|
+
extras=job.extras,
|
894
897
|
)
|
895
898
|
|
896
899
|
if event and event.is_set():
|
ddeutil/workflow/logs.py
CHANGED
@@ -9,33 +9,65 @@
|
|
9
9
|
from __future__ import annotations
|
10
10
|
|
11
11
|
import json
|
12
|
+
import logging
|
12
13
|
import os
|
13
14
|
from abc import ABC, abstractmethod
|
14
15
|
from collections.abc import Iterator
|
16
|
+
from dataclasses import field
|
15
17
|
from datetime import datetime
|
18
|
+
from functools import lru_cache
|
16
19
|
from inspect import Traceback, currentframe, getframeinfo
|
17
20
|
from pathlib import Path
|
18
21
|
from threading import get_ident
|
19
22
|
from typing import ClassVar, Literal, Optional, Union
|
20
23
|
|
21
|
-
from pydantic import BaseModel, Field
|
24
|
+
from pydantic import BaseModel, Field, ValidationInfo
|
22
25
|
from pydantic.dataclasses import dataclass
|
23
26
|
from pydantic.functional_validators import model_validator
|
24
27
|
from typing_extensions import Self
|
25
28
|
|
26
29
|
from .__types import DictData, DictStr, TupleStr
|
27
|
-
from .conf import config,
|
30
|
+
from .conf import config, dynamic
|
28
31
|
from .utils import cut_id, get_dt_now
|
29
32
|
|
33
|
+
|
34
|
+
@lru_cache
|
35
|
+
def get_logger(name: str):
|
36
|
+
"""Return logger object with an input module name.
|
37
|
+
|
38
|
+
:param name: A module name that want to log.
|
39
|
+
"""
|
40
|
+
lg = logging.getLogger(name)
|
41
|
+
|
42
|
+
# NOTE: Developers using this package can then disable all logging just for
|
43
|
+
# this package by;
|
44
|
+
#
|
45
|
+
# `logging.getLogger('ddeutil.workflow').propagate = False`
|
46
|
+
#
|
47
|
+
lg.addHandler(logging.NullHandler())
|
48
|
+
|
49
|
+
formatter = logging.Formatter(
|
50
|
+
fmt=config.log_format,
|
51
|
+
datefmt=config.log_datetime_format,
|
52
|
+
)
|
53
|
+
stream = logging.StreamHandler()
|
54
|
+
stream.setFormatter(formatter)
|
55
|
+
lg.addHandler(stream)
|
56
|
+
|
57
|
+
lg.setLevel(logging.DEBUG if config.debug else logging.INFO)
|
58
|
+
return lg
|
59
|
+
|
60
|
+
|
30
61
|
logger = get_logger("ddeutil.workflow")
|
31
62
|
|
32
63
|
__all__: TupleStr = (
|
33
64
|
"FileTraceLog",
|
34
65
|
"SQLiteTraceLog",
|
35
66
|
"TraceData",
|
36
|
-
"
|
67
|
+
"TraceMeta",
|
37
68
|
"TraceLog",
|
38
69
|
"get_dt_tznow",
|
70
|
+
"get_logger",
|
39
71
|
"get_trace",
|
40
72
|
"get_trace_obj",
|
41
73
|
"get_audit",
|
@@ -53,7 +85,9 @@ def get_dt_tznow() -> datetime: # pragma: no cov
|
|
53
85
|
return get_dt_now(tz=config.tz)
|
54
86
|
|
55
87
|
|
56
|
-
class
|
88
|
+
class TraceMeta(BaseModel): # pragma: no cov
|
89
|
+
"""Trace Meta model."""
|
90
|
+
|
57
91
|
mode: Literal["stdout", "stderr"]
|
58
92
|
datetime: str
|
59
93
|
process: int
|
@@ -63,14 +97,28 @@ class TraceMeda(BaseModel): # pragma: no cov
|
|
63
97
|
lineno: int
|
64
98
|
|
65
99
|
@classmethod
|
66
|
-
def make(
|
67
|
-
|
100
|
+
def make(
|
101
|
+
cls,
|
102
|
+
mode: Literal["stdout", "stderr"],
|
103
|
+
message: str,
|
104
|
+
*,
|
105
|
+
extras: Optional[DictData] = None,
|
106
|
+
) -> Self:
|
107
|
+
"""Make the current TraceMeta instance that catching local state.
|
108
|
+
|
109
|
+
:rtype: Self
|
110
|
+
"""
|
68
111
|
frame_info: Traceback = getframeinfo(
|
69
112
|
currentframe().f_back.f_back.f_back
|
70
113
|
)
|
114
|
+
extras: DictData = extras or {}
|
71
115
|
return cls(
|
72
116
|
mode=mode,
|
73
|
-
datetime=
|
117
|
+
datetime=(
|
118
|
+
get_dt_now(tz=dynamic("tz", extras=extras)).strftime(
|
119
|
+
dynamic("log_datetime_format", extras=extras)
|
120
|
+
)
|
121
|
+
),
|
74
122
|
process=os.getpid(),
|
75
123
|
thread=get_ident(),
|
76
124
|
message=message,
|
@@ -84,7 +132,7 @@ class TraceData(BaseModel): # pragma: no cov
|
|
84
132
|
|
85
133
|
stdout: str = Field(description="A standard output trace data.")
|
86
134
|
stderr: str = Field(description="A standard error trace data.")
|
87
|
-
meta: list[
|
135
|
+
meta: list[TraceMeta] = Field(
|
88
136
|
default_factory=list,
|
89
137
|
description=(
|
90
138
|
"A metadata mapping of this output and error before making it to "
|
@@ -126,7 +174,8 @@ class BaseTraceLog(ABC): # pragma: no cov
|
|
126
174
|
"""Base Trace Log dataclass object."""
|
127
175
|
|
128
176
|
run_id: str
|
129
|
-
parent_run_id: Optional[str] = None
|
177
|
+
parent_run_id: Optional[str] = field(default=None)
|
178
|
+
extras: DictData = field(default_factory=dict, compare=False, repr=False)
|
130
179
|
|
131
180
|
@abstractmethod
|
132
181
|
def writer(self, message: str, is_err: bool = False) -> None:
|
@@ -148,6 +197,9 @@ class BaseTraceLog(ABC): # pragma: no cov
|
|
148
197
|
:param message:
|
149
198
|
:param is_err:
|
150
199
|
"""
|
200
|
+
raise NotImplementedError(
|
201
|
+
"Create async writer logic for this trace object before using."
|
202
|
+
)
|
151
203
|
|
152
204
|
@abstractmethod
|
153
205
|
def make_message(self, message: str) -> str:
|
@@ -169,7 +221,7 @@ class BaseTraceLog(ABC): # pragma: no cov
|
|
169
221
|
"""
|
170
222
|
msg: str = self.make_message(message)
|
171
223
|
|
172
|
-
if
|
224
|
+
if dynamic("debug", extras=self.extras):
|
173
225
|
self.writer(msg)
|
174
226
|
|
175
227
|
logger.debug(msg, stacklevel=2)
|
@@ -221,7 +273,7 @@ class BaseTraceLog(ABC): # pragma: no cov
|
|
221
273
|
:param message: (str) A message that want to log.
|
222
274
|
"""
|
223
275
|
msg: str = self.make_message(message)
|
224
|
-
if
|
276
|
+
if dynamic("debug", extras=self.extras):
|
225
277
|
await self.awriter(msg)
|
226
278
|
logger.info(msg, stacklevel=2)
|
227
279
|
|
@@ -271,21 +323,38 @@ class FileTraceLog(BaseTraceLog): # pragma: no cov
|
|
271
323
|
|
272
324
|
@classmethod
|
273
325
|
def find_logs(
|
274
|
-
cls,
|
326
|
+
cls,
|
327
|
+
path: Path | None = None,
|
328
|
+
extras: Optional[DictData] = None,
|
275
329
|
) -> Iterator[TraceData]: # pragma: no cov
|
276
|
-
"""Find trace logs.
|
330
|
+
"""Find trace logs.
|
331
|
+
|
332
|
+
:param path: (Path)
|
333
|
+
:param extras: An extra parameter that want to override core config.
|
334
|
+
"""
|
277
335
|
for file in sorted(
|
278
|
-
(path or
|
336
|
+
(path or dynamic("log_path", extras=extras)).glob("./run_id=*"),
|
279
337
|
key=lambda f: f.lstat().st_mtime,
|
280
338
|
):
|
281
339
|
yield TraceData.from_path(file)
|
282
340
|
|
283
341
|
@classmethod
|
284
342
|
def find_log_with_id(
|
285
|
-
cls,
|
343
|
+
cls,
|
344
|
+
run_id: str,
|
345
|
+
force_raise: bool = True,
|
346
|
+
*,
|
347
|
+
path: Path | None = None,
|
348
|
+
extras: Optional[DictData] = None,
|
286
349
|
) -> TraceData:
|
287
|
-
"""Find trace log with an input specific run ID.
|
288
|
-
|
350
|
+
"""Find trace log with an input specific run ID.
|
351
|
+
|
352
|
+
:param run_id: A running ID of trace log.
|
353
|
+
:param force_raise:
|
354
|
+
:param path:
|
355
|
+
:param extras: An extra parameter that want to override core config.
|
356
|
+
"""
|
357
|
+
base_path: Path = path or dynamic("log_path", extras=extras)
|
289
358
|
file: Path = base_path / f"run_id={run_id}"
|
290
359
|
if file.exists():
|
291
360
|
return TraceData.from_path(file)
|
@@ -299,7 +368,8 @@ class FileTraceLog(BaseTraceLog): # pragma: no cov
|
|
299
368
|
@property
|
300
369
|
def pointer(self) -> Path:
|
301
370
|
log_file: Path = (
|
302
|
-
|
371
|
+
dynamic("log_path", extras=self.extras)
|
372
|
+
/ f"run_id={self.parent_run_id or self.run_id}"
|
303
373
|
)
|
304
374
|
if not log_file.exists():
|
305
375
|
log_file.mkdir(parents=True)
|
@@ -340,18 +410,17 @@ class FileTraceLog(BaseTraceLog): # pragma: no cov
|
|
340
410
|
:param message: A message after making.
|
341
411
|
:param is_err: A flag for writing with an error trace or not.
|
342
412
|
"""
|
343
|
-
if not
|
413
|
+
if not dynamic("enable_write_log", extras=self.extras):
|
344
414
|
return
|
345
415
|
|
346
416
|
write_file: str = "stderr" if is_err else "stdout"
|
347
|
-
trace_meta:
|
417
|
+
trace_meta: TraceMeta = TraceMeta.make(mode=write_file, message=message)
|
348
418
|
|
349
419
|
with (self.pointer / f"{write_file}.txt").open(
|
350
420
|
mode="at", encoding="utf-8"
|
351
421
|
) as f:
|
352
|
-
|
353
|
-
|
354
|
-
)
|
422
|
+
fmt: str = dynamic("log_format_file", extras=self.extras)
|
423
|
+
f.write(f"{fmt}\n".format(**trace_meta.model_dump()))
|
355
424
|
|
356
425
|
with (self.pointer / "metadata.json").open(
|
357
426
|
mode="at", encoding="utf-8"
|
@@ -361,7 +430,7 @@ class FileTraceLog(BaseTraceLog): # pragma: no cov
|
|
361
430
|
async def awriter(
|
362
431
|
self, message: str, is_err: bool = False
|
363
432
|
) -> None: # pragma: no cov
|
364
|
-
if not
|
433
|
+
if not dynamic("enable_write_log", extras=self.extras):
|
365
434
|
return
|
366
435
|
|
367
436
|
try:
|
@@ -370,14 +439,13 @@ class FileTraceLog(BaseTraceLog): # pragma: no cov
|
|
370
439
|
raise ImportError("Async mode need aiofiles package") from e
|
371
440
|
|
372
441
|
write_file: str = "stderr" if is_err else "stdout"
|
373
|
-
trace_meta:
|
442
|
+
trace_meta: TraceMeta = TraceMeta.make(mode=write_file, message=message)
|
374
443
|
|
375
444
|
async with aiofiles.open(
|
376
445
|
self.pointer / f"{write_file}.txt", mode="at", encoding="utf-8"
|
377
446
|
) as f:
|
378
|
-
|
379
|
-
|
380
|
-
)
|
447
|
+
fmt: str = dynamic("log_format_file", extras=self.extras)
|
448
|
+
await f.write(f"{fmt}\n".format(**trace_meta.model_dump()))
|
381
449
|
|
382
450
|
async with aiofiles.open(
|
383
451
|
self.pointer / "metadata.json", mode="at", encoding="utf-8"
|
@@ -419,16 +487,38 @@ TraceLog = Union[
|
|
419
487
|
|
420
488
|
|
421
489
|
def get_trace(
|
422
|
-
run_id: str,
|
490
|
+
run_id: str,
|
491
|
+
parent_run_id: str | None = None,
|
492
|
+
*,
|
493
|
+
extras: Optional[DictData] = None,
|
423
494
|
) -> TraceLog: # pragma: no cov
|
424
|
-
"""Get dynamic TraceLog object from the setting config.
|
425
|
-
|
426
|
-
|
427
|
-
|
495
|
+
"""Get dynamic TraceLog object from the setting config.
|
496
|
+
|
497
|
+
:param run_id: A running ID.
|
498
|
+
:param parent_run_id: A parent running ID.
|
499
|
+
:param extras: An extra parameter that want to override the core config.
|
500
|
+
|
501
|
+
:rtype: TraceLog
|
502
|
+
"""
|
503
|
+
if dynamic("log_path", extras=extras).is_file():
|
504
|
+
return SQLiteTraceLog(
|
505
|
+
run_id, parent_run_id=parent_run_id, extras=(extras or {})
|
506
|
+
)
|
507
|
+
return FileTraceLog(
|
508
|
+
run_id, parent_run_id=parent_run_id, extras=(extras or {})
|
509
|
+
)
|
510
|
+
|
511
|
+
|
512
|
+
def get_trace_obj(
|
513
|
+
extras: Optional[DictData] = None,
|
514
|
+
) -> type[TraceLog]: # pragma: no cov
|
515
|
+
"""Get trace object.
|
428
516
|
|
517
|
+
:param extras: An extra parameter that want to override the core config.
|
429
518
|
|
430
|
-
|
431
|
-
|
519
|
+
:rtype: type[TraceLog]
|
520
|
+
"""
|
521
|
+
if dynamic("log_path", extras=extras).is_file():
|
432
522
|
return SQLiteTraceLog
|
433
523
|
return FileTraceLog
|
434
524
|
|
@@ -439,6 +529,10 @@ class BaseAudit(BaseModel, ABC):
|
|
439
529
|
subclass like file, sqlite, etc.
|
440
530
|
"""
|
441
531
|
|
532
|
+
extras: DictData = Field(
|
533
|
+
default_factory=dict,
|
534
|
+
description="An extras parameter that want to override core config",
|
535
|
+
)
|
442
536
|
name: str = Field(description="A workflow name.")
|
443
537
|
release: datetime = Field(description="A release datetime.")
|
444
538
|
type: str = Field(description="A running type before logging.")
|
@@ -454,12 +548,12 @@ class BaseAudit(BaseModel, ABC):
|
|
454
548
|
execution_time: float = Field(default=0, description="An execution time.")
|
455
549
|
|
456
550
|
@model_validator(mode="after")
|
457
|
-
def __model_action(self) -> Self:
|
551
|
+
def __model_action(self, info: ValidationInfo) -> Self:
|
458
552
|
"""Do before the Audit action with WORKFLOW_AUDIT_ENABLE_WRITE env variable.
|
459
553
|
|
460
554
|
:rtype: Self
|
461
555
|
"""
|
462
|
-
if
|
556
|
+
if dynamic("enable_write_audit", extras=info.data.get("extras")):
|
463
557
|
self.do_before()
|
464
558
|
return self
|
465
559
|
|
@@ -487,15 +581,20 @@ class FileAudit(BaseAudit):
|
|
487
581
|
self.pointer().mkdir(parents=True, exist_ok=True)
|
488
582
|
|
489
583
|
@classmethod
|
490
|
-
def find_audits(
|
584
|
+
def find_audits(
|
585
|
+
cls, name: str, *, extras: Optional[DictData] = None
|
586
|
+
) -> Iterator[Self]:
|
491
587
|
"""Generate the audit data that found from logs path with specific a
|
492
588
|
workflow name.
|
493
589
|
|
494
590
|
:param name: A workflow name that want to search release logging data.
|
591
|
+
:param extras: An extra parameter that want to override core config.
|
495
592
|
|
496
593
|
:rtype: Iterator[Self]
|
497
594
|
"""
|
498
|
-
pointer: Path =
|
595
|
+
pointer: Path = (
|
596
|
+
dynamic("audit_path", extras=extras) / f"workflow={name}"
|
597
|
+
)
|
499
598
|
if not pointer.exists():
|
500
599
|
raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
|
501
600
|
|
@@ -508,13 +607,16 @@ class FileAudit(BaseAudit):
|
|
508
607
|
cls,
|
509
608
|
name: str,
|
510
609
|
release: datetime | None = None,
|
610
|
+
*,
|
611
|
+
extras: Optional[DictData] = None,
|
511
612
|
) -> Self:
|
512
613
|
"""Return the audit data that found from logs path with specific
|
513
614
|
workflow name and release values. If a release does not pass to an input
|
514
615
|
argument, it will return the latest release from the current log path.
|
515
616
|
|
516
|
-
:param name: A workflow name that want to search log.
|
517
|
-
:param release: A release datetime that want to search log.
|
617
|
+
:param name: (str) A workflow name that want to search log.
|
618
|
+
:param release: (datetime) A release datetime that want to search log.
|
619
|
+
:param extras: An extra parameter that want to override core config.
|
518
620
|
|
519
621
|
:raise FileNotFoundError:
|
520
622
|
:raise NotImplementedError: If an input release does not pass to this
|
@@ -526,7 +628,7 @@ class FileAudit(BaseAudit):
|
|
526
628
|
raise NotImplementedError("Find latest log does not implement yet.")
|
527
629
|
|
528
630
|
pointer: Path = (
|
529
|
-
|
631
|
+
dynamic("audit_path", extras=extras)
|
530
632
|
/ f"workflow={name}/release={release:%Y%m%d%H%M%S}"
|
531
633
|
)
|
532
634
|
if not pointer.exists():
|
@@ -540,24 +642,31 @@ class FileAudit(BaseAudit):
|
|
540
642
|
return cls.model_validate(obj=json.load(f))
|
541
643
|
|
542
644
|
@classmethod
|
543
|
-
def is_pointed(
|
645
|
+
def is_pointed(
|
646
|
+
cls,
|
647
|
+
name: str,
|
648
|
+
release: datetime,
|
649
|
+
*,
|
650
|
+
extras: Optional[DictData] = None,
|
651
|
+
) -> bool:
|
544
652
|
"""Check the release log already pointed or created at the destination
|
545
653
|
log path.
|
546
654
|
|
547
|
-
:param name: A workflow name.
|
548
|
-
:param release: A release datetime.
|
655
|
+
:param name: (str) A workflow name.
|
656
|
+
:param release: (datetime) A release datetime.
|
657
|
+
:param extras: An extra parameter that want to override core config.
|
549
658
|
|
550
659
|
:rtype: bool
|
551
660
|
:return: Return False if the release log was not pointed or created.
|
552
661
|
"""
|
553
662
|
# NOTE: Return False if enable writing log flag does not set.
|
554
|
-
if not
|
663
|
+
if not dynamic("enable_write_audit", extras=extras):
|
555
664
|
return False
|
556
665
|
|
557
666
|
# NOTE: create pointer path that use the same logic of pointer method.
|
558
|
-
pointer: Path =
|
559
|
-
|
560
|
-
)
|
667
|
+
pointer: Path = dynamic(
|
668
|
+
"audit_path", extras=extras
|
669
|
+
) / cls.filename_fmt.format(name=name, release=release)
|
561
670
|
|
562
671
|
return pointer.exists()
|
563
672
|
|
@@ -566,9 +675,9 @@ class FileAudit(BaseAudit):
|
|
566
675
|
|
567
676
|
:rtype: Path
|
568
677
|
"""
|
569
|
-
return
|
570
|
-
|
571
|
-
)
|
678
|
+
return dynamic(
|
679
|
+
"audit_path", extras=self.extras
|
680
|
+
) / self.filename_fmt.format(name=self.name, release=self.release)
|
572
681
|
|
573
682
|
def save(self, excluded: list[str] | None) -> Self:
|
574
683
|
"""Save logging data that receive a context data from a workflow
|
@@ -582,7 +691,7 @@ class FileAudit(BaseAudit):
|
|
582
691
|
trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
|
583
692
|
|
584
693
|
# NOTE: Check environ variable was set for real writing.
|
585
|
-
if not
|
694
|
+
if not dynamic("enable_write_audit", extras=self.extras):
|
586
695
|
trace.debug("[LOG]: Skip writing log cause config was set")
|
587
696
|
return self
|
588
697
|
|
@@ -624,7 +733,7 @@ class SQLiteAudit(BaseAudit): # pragma: no cov
|
|
624
733
|
trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
|
625
734
|
|
626
735
|
# NOTE: Check environ variable was set for real writing.
|
627
|
-
if not
|
736
|
+
if not dynamic("enable_write_audit", extras=self.extras):
|
628
737
|
trace.debug("[LOG]: Skip writing log cause config was set")
|
629
738
|
return self
|
630
739
|
|
@@ -649,11 +758,15 @@ Audit = Union[
|
|
649
758
|
]
|
650
759
|
|
651
760
|
|
652
|
-
def get_audit(
|
761
|
+
def get_audit(
|
762
|
+
extras: Optional[DictData] = None,
|
763
|
+
) -> type[Audit]: # pragma: no cov
|
653
764
|
"""Get an audit class that dynamic base on the config audit path value.
|
654
765
|
|
766
|
+
:param extras: An extra parameter that want to override the core config.
|
767
|
+
|
655
768
|
:rtype: type[Audit]
|
656
769
|
"""
|
657
|
-
if
|
770
|
+
if dynamic("audit_path", extras=extras).is_file():
|
658
771
|
return SQLiteAudit
|
659
772
|
return FileAudit
|
ddeutil/workflow/result.py
CHANGED
@@ -3,6 +3,7 @@
|
|
3
3
|
# Licensed under the MIT License. See LICENSE in the project root for
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
|
+
# [x] Use dynamic config
|
6
7
|
"""Result module. It is the data context transfer objects that use by all object
|
7
8
|
in this package. This module provide Status enum object and Result dataclass.
|
8
9
|
"""
|
@@ -19,8 +20,9 @@ from pydantic.functional_validators import model_validator
|
|
19
20
|
from typing_extensions import Self
|
20
21
|
|
21
22
|
from .__types import DictData
|
23
|
+
from .conf import dynamic
|
22
24
|
from .logs import TraceLog, get_dt_tznow, get_trace
|
23
|
-
from .utils import default_gen_id, gen_id
|
25
|
+
from .utils import default_gen_id, gen_id, get_dt_now
|
24
26
|
|
25
27
|
|
26
28
|
class Status(IntEnum):
|
@@ -67,7 +69,7 @@ class Result:
|
|
67
69
|
ts: datetime = field(default_factory=get_dt_tznow, compare=False)
|
68
70
|
|
69
71
|
trace: Optional[TraceLog] = field(default=None, compare=False, repr=False)
|
70
|
-
extras: DictData = field(default_factory=dict)
|
72
|
+
extras: DictData = field(default_factory=dict, compare=False, repr=False)
|
71
73
|
|
72
74
|
@classmethod
|
73
75
|
def construct_with_rs_or_id(
|
@@ -111,17 +113,22 @@ class Result:
|
|
111
113
|
:rtype: Self
|
112
114
|
"""
|
113
115
|
if self.trace is None: # pragma: no cov
|
114
|
-
self.trace: TraceLog = get_trace(
|
116
|
+
self.trace: TraceLog = get_trace(
|
117
|
+
self.run_id, self.parent_run_id, extras=self.extras
|
118
|
+
)
|
115
119
|
return self
|
116
120
|
|
117
121
|
def set_parent_run_id(self, running_id: str) -> Self:
|
118
122
|
"""Set a parent running ID.
|
119
123
|
|
120
|
-
:param running_id: A running ID that want to update on this model.
|
124
|
+
:param running_id: (str) A running ID that want to update on this model.
|
125
|
+
|
121
126
|
:rtype: Self
|
122
127
|
"""
|
123
128
|
self.parent_run_id: str = running_id
|
124
|
-
self.trace: TraceLog = get_trace(
|
129
|
+
self.trace: TraceLog = get_trace(
|
130
|
+
self.run_id, running_id, extras=self.extras
|
131
|
+
)
|
125
132
|
return self
|
126
133
|
|
127
134
|
def catch(
|
@@ -152,4 +159,6 @@ class Result:
|
|
152
159
|
|
153
160
|
:rtype: float
|
154
161
|
"""
|
155
|
-
return (
|
162
|
+
return (
|
163
|
+
get_dt_now(tz=dynamic("tz", extras=self.extras)) - self.ts
|
164
|
+
).total_seconds()
|
ddeutil/workflow/reusables.py
CHANGED
@@ -534,8 +534,7 @@ def extract_call(
|
|
534
534
|
|
535
535
|
call: CallSearchData = CallSearchData(**found.groupdict())
|
536
536
|
rgt: dict[str, Registry] = make_registry(
|
537
|
-
submodule=f"{call.path}",
|
538
|
-
registries=registries,
|
537
|
+
submodule=f"{call.path}", registries=registries
|
539
538
|
)
|
540
539
|
|
541
540
|
if call.func not in rgt:
|
ddeutil/workflow/scheduler.py
CHANGED
@@ -52,7 +52,7 @@ except ImportError: # pragma: no cov
|
|
52
52
|
|
53
53
|
from .__cron import CronRunner
|
54
54
|
from .__types import DictData, TupleStr
|
55
|
-
from .conf import Loader, SimLoad, config
|
55
|
+
from .conf import Loader, SimLoad, config
|
56
56
|
from .cron import On
|
57
57
|
from .exceptions import ScheduleException, WorkflowException
|
58
58
|
from .logs import Audit, get_audit
|
@@ -62,7 +62,6 @@ from .workflow import Release, ReleaseQueue, Workflow, WorkflowTask
|
|
62
62
|
|
63
63
|
P = ParamSpec("P")
|
64
64
|
|
65
|
-
logger = get_logger("ddeutil.workflow")
|
66
65
|
logging.getLogger("schedule").setLevel(logging.INFO)
|
67
66
|
|
68
67
|
|
@@ -231,6 +230,11 @@ class Schedule(BaseModel):
|
|
231
230
|
enhance the workflow object by adding the alias and values fields.
|
232
231
|
"""
|
233
232
|
|
233
|
+
extras: DictData = Field(
|
234
|
+
default_factory=dict,
|
235
|
+
description="An extra override config values.",
|
236
|
+
)
|
237
|
+
|
234
238
|
desc: Optional[str] = Field(
|
235
239
|
default=None,
|
236
240
|
description=(
|
@@ -281,6 +285,9 @@ class Schedule(BaseModel):
|
|
281
285
|
# NOTE: Add name to loader data
|
282
286
|
loader_data["name"] = name.replace(" ", "_")
|
283
287
|
|
288
|
+
if extras:
|
289
|
+
loader_data["extras"] = extras
|
290
|
+
|
284
291
|
return cls.model_validate(obj=loader_data)
|
285
292
|
|
286
293
|
@classmethod
|
@@ -288,7 +295,7 @@ class Schedule(BaseModel):
|
|
288
295
|
cls,
|
289
296
|
name: str,
|
290
297
|
path: Path,
|
291
|
-
|
298
|
+
extras: DictData | None = None,
|
292
299
|
) -> Self:
|
293
300
|
"""Create Schedule instance from the SimLoad object that receive an
|
294
301
|
input schedule name and conf path. The loader object will use this
|
@@ -297,7 +304,7 @@ class Schedule(BaseModel):
|
|
297
304
|
|
298
305
|
:param name: (str) A schedule name that want to pass to Loader object.
|
299
306
|
:param path: (Path) A config path that want to search.
|
300
|
-
:param
|
307
|
+
:param extras: An external parameters that want to pass to Loader
|
301
308
|
object.
|
302
309
|
|
303
310
|
:raise ValueError: If the type does not match with current object.
|
@@ -305,7 +312,7 @@ class Schedule(BaseModel):
|
|
305
312
|
:rtype: Self
|
306
313
|
"""
|
307
314
|
loader: SimLoad = SimLoad(
|
308
|
-
name, conf_path=path, externals=(
|
315
|
+
name, conf_path=path, externals=(extras or {})
|
309
316
|
)
|
310
317
|
|
311
318
|
# NOTE: Validate the config type match with current connection model
|
@@ -317,6 +324,9 @@ class Schedule(BaseModel):
|
|
317
324
|
# NOTE: Add name to loader data
|
318
325
|
loader_data["name"] = name.replace(" ", "_")
|
319
326
|
|
327
|
+
if extras:
|
328
|
+
loader_data["extras"] = extras
|
329
|
+
|
320
330
|
return cls.model_validate(obj=loader_data)
|
321
331
|
|
322
332
|
def tasks(
|
@@ -715,7 +725,7 @@ def schedule_control(
|
|
715
725
|
|
716
726
|
:param schedules: A list of workflow names that want to schedule running.
|
717
727
|
:param stop: A datetime value that use to stop running schedule.
|
718
|
-
:param extras: An extra parameters that
|
728
|
+
:param extras: An extra parameters that want to override core config.
|
719
729
|
:param audit: An audit class that use on the workflow task release for
|
720
730
|
writing its release audit context.
|
721
731
|
:param parent_run_id: A parent workflow running ID for this release.
|
@@ -761,7 +771,7 @@ def schedule_control(
|
|
761
771
|
|
762
772
|
def schedule_runner(
|
763
773
|
stop: datetime | None = None,
|
764
|
-
|
774
|
+
extras: DictData | None = None,
|
765
775
|
excluded: list[str] | None = None,
|
766
776
|
) -> Result: # pragma: no cov
|
767
777
|
"""Schedule runner function it the multiprocess controller function for
|
@@ -770,7 +780,7 @@ def schedule_runner(
|
|
770
780
|
path by `WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS` value.
|
771
781
|
|
772
782
|
:param stop: A stop datetime object that force stop running scheduler.
|
773
|
-
:param
|
783
|
+
:param extras: An extra parameter that want to override core config.
|
774
784
|
:param excluded: A list of schedule name that want to exclude from finding.
|
775
785
|
|
776
786
|
This function will get all workflows that include on value that was
|
@@ -801,7 +811,7 @@ def schedule_runner(
|
|
801
811
|
schedule_control,
|
802
812
|
schedules=[load[0] for load in loader],
|
803
813
|
stop=stop,
|
804
|
-
|
814
|
+
extras=(extras or {}),
|
805
815
|
parent_run_id=result.parent_run_id,
|
806
816
|
)
|
807
817
|
for loader in batch(
|
ddeutil/workflow/stages.py
CHANGED
@@ -38,7 +38,6 @@ from concurrent.futures import (
|
|
38
38
|
ThreadPoolExecutor,
|
39
39
|
as_completed,
|
40
40
|
)
|
41
|
-
from dataclasses import is_dataclass
|
42
41
|
from inspect import Parameter
|
43
42
|
from pathlib import Path
|
44
43
|
from subprocess import CompletedProcess
|
@@ -206,6 +205,7 @@ class BaseStage(BaseModel, ABC):
|
|
206
205
|
run_id=run_id,
|
207
206
|
parent_run_id=parent_run_id,
|
208
207
|
id_logic=self.iden,
|
208
|
+
extras=self.extras,
|
209
209
|
)
|
210
210
|
|
211
211
|
try:
|
@@ -389,14 +389,15 @@ class BaseAsyncStage(BaseStage):
|
|
389
389
|
run_id=run_id,
|
390
390
|
parent_run_id=parent_run_id,
|
391
391
|
id_logic=self.iden,
|
392
|
+
extras=self.extras,
|
392
393
|
)
|
393
394
|
|
394
395
|
try:
|
395
396
|
rs: Result = await self.axecute(params, result=result, event=event)
|
396
|
-
if to is not None:
|
397
|
+
if to is not None: # pragma: no cov
|
397
398
|
return self.set_outputs(rs.context, to=to)
|
398
399
|
return rs
|
399
|
-
except Exception as e:
|
400
|
+
except Exception as e: # pragma: no cov
|
400
401
|
await result.trace.aerror(f"[STAGE]: {e.__class__.__name__}: {e}")
|
401
402
|
|
402
403
|
if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
|
@@ -856,6 +857,7 @@ class CallStage(BaseStage):
|
|
856
857
|
)
|
857
858
|
|
858
859
|
args = self.parse_model_args(call_func, args, result)
|
860
|
+
|
859
861
|
if inspect.iscoroutinefunction(call_func):
|
860
862
|
loop = asyncio.get_event_loop()
|
861
863
|
rs: DictData = loop.run_until_complete(
|
@@ -904,8 +906,11 @@ class CallStage(BaseStage):
|
|
904
906
|
args[arg] = args.pop(arg.removeprefix("_"))
|
905
907
|
|
906
908
|
t: Any = type_hints[arg]
|
907
|
-
|
908
|
-
|
909
|
+
|
910
|
+
# NOTE: Check Result argument was passed to this caller function.
|
911
|
+
#
|
912
|
+
# if is_dataclass(t) and t.__name__ == "Result" and arg not in args:
|
913
|
+
# args[arg] = result
|
909
914
|
|
910
915
|
if issubclass(t, BaseModel) and arg in args:
|
911
916
|
args[arg] = t.model_validate(obj=args[arg])
|
ddeutil/workflow/workflow.py
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
# Licensed under the MIT License. See LICENSE in the project root for
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
|
-
# [x] Use dynamic config
|
6
|
+
# [x] Use dynamic config
|
7
7
|
"""A Workflow module that is the core model of this package."""
|
8
8
|
from __future__ import annotations
|
9
9
|
|
@@ -32,7 +32,7 @@ from typing_extensions import Self
|
|
32
32
|
|
33
33
|
from .__cron import CronJob, CronRunner
|
34
34
|
from .__types import DictData, TupleStr
|
35
|
-
from .conf import Loader, SimLoad, dynamic
|
35
|
+
from .conf import Loader, SimLoad, dynamic
|
36
36
|
from .cron import On
|
37
37
|
from .exceptions import JobException, WorkflowException
|
38
38
|
from .job import Job
|
@@ -47,8 +47,6 @@ from .utils import (
|
|
47
47
|
wait_to_next_minute,
|
48
48
|
)
|
49
49
|
|
50
|
-
logger = get_logger("ddeutil.workflow")
|
51
|
-
|
52
50
|
__all__: TupleStr = (
|
53
51
|
"Release",
|
54
52
|
"ReleaseQueue",
|
@@ -75,10 +73,10 @@ class Release:
|
|
75
73
|
that use with the `workflow.release` method.
|
76
74
|
"""
|
77
75
|
|
78
|
-
date: datetime
|
79
|
-
offset: float
|
80
|
-
end_date: datetime
|
81
|
-
runner: CronRunner
|
76
|
+
date: datetime
|
77
|
+
offset: float
|
78
|
+
end_date: datetime
|
79
|
+
runner: CronRunner
|
82
80
|
type: ReleaseType = field(default=ReleaseType.DEFAULT)
|
83
81
|
|
84
82
|
def __repr__(self) -> str:
|
@@ -94,13 +92,13 @@ class Release:
|
|
94
92
|
|
95
93
|
@classmethod
|
96
94
|
def from_dt(
|
97
|
-
cls, dt: datetime | str, *,
|
95
|
+
cls, dt: datetime | str, *, extras: Optional[DictData] = None
|
98
96
|
) -> Self:
|
99
97
|
"""Construct Release via datetime object only.
|
100
98
|
|
101
99
|
:param dt: (datetime | str) A datetime object or string that want to
|
102
100
|
construct to the Release object.
|
103
|
-
:param
|
101
|
+
:param extras: An extra parameters that want to pass to override
|
104
102
|
config.
|
105
103
|
|
106
104
|
:raise TypeError: If the type of the dt argument does not valid with
|
@@ -120,8 +118,10 @@ class Release:
|
|
120
118
|
date=dt,
|
121
119
|
offset=0,
|
122
120
|
end_date=dt + timedelta(days=1),
|
123
|
-
runner=
|
124
|
-
|
121
|
+
runner=(
|
122
|
+
CronJob("* * * * *").schedule(
|
123
|
+
dt.replace(tzinfo=dynamic("tz", extras=extras))
|
124
|
+
)
|
125
125
|
),
|
126
126
|
)
|
127
127
|
|
@@ -164,11 +164,16 @@ class ReleaseQueue:
|
|
164
164
|
|
165
165
|
@classmethod
|
166
166
|
def from_list(
|
167
|
-
cls,
|
167
|
+
cls,
|
168
|
+
queue: list[datetime] | list[Release] | None = None,
|
169
|
+
extras: Optional[DictData] = None,
|
168
170
|
) -> Self:
|
169
171
|
"""Construct ReleaseQueue object from an input queue value that passing
|
170
172
|
with list of datetime or list of Release.
|
171
173
|
|
174
|
+
:param queue:
|
175
|
+
:param extras: An extra parameter that want to override core config.
|
176
|
+
|
172
177
|
:raise TypeError: If the type of input queue does not valid.
|
173
178
|
|
174
179
|
:rtype: ReleaseQueue
|
@@ -179,7 +184,11 @@ class ReleaseQueue:
|
|
179
184
|
if isinstance(queue, list):
|
180
185
|
|
181
186
|
if all(isinstance(q, datetime) for q in queue):
|
182
|
-
return cls(
|
187
|
+
return cls(
|
188
|
+
queue=[
|
189
|
+
Release.from_dt(q, extras=(extras or {})) for q in queue
|
190
|
+
]
|
191
|
+
)
|
183
192
|
|
184
193
|
if all(isinstance(q, Release) for q in queue):
|
185
194
|
return cls(queue=queue)
|
@@ -216,7 +225,7 @@ class ReleaseQueue:
|
|
216
225
|
:rtype: bool
|
217
226
|
"""
|
218
227
|
if isinstance(value, datetime):
|
219
|
-
value = Release.from_dt(value)
|
228
|
+
value = Release.from_dt(value, extras=self.extras)
|
220
229
|
|
221
230
|
return (
|
222
231
|
(value in self.queue)
|
@@ -445,7 +454,7 @@ class Workflow(BaseModel):
|
|
445
454
|
# "only one value in the on field."
|
446
455
|
# )
|
447
456
|
|
448
|
-
extras: DictData = info.data.get("extras"
|
457
|
+
extras: Optional[DictData] = info.data.get("extras")
|
449
458
|
if len(set_ons) > (
|
450
459
|
conf := dynamic("max_on_per_workflow", extras=extras)
|
451
460
|
):
|
@@ -596,13 +605,14 @@ class Workflow(BaseModel):
|
|
596
605
|
|
597
606
|
:rtype: Result
|
598
607
|
"""
|
599
|
-
audit: type[Audit] = audit or get_audit()
|
608
|
+
audit: type[Audit] = audit or get_audit(extras=self.extras)
|
600
609
|
name: str = override_log_name or self.name
|
601
610
|
result: Result = Result.construct_with_rs_or_id(
|
602
611
|
result,
|
603
612
|
run_id=run_id,
|
604
613
|
parent_run_id=parent_run_id,
|
605
614
|
id_logic=name,
|
615
|
+
extras=self.extras,
|
606
616
|
)
|
607
617
|
|
608
618
|
if queue is not None and not isinstance(queue, ReleaseQueue):
|
@@ -612,7 +622,7 @@ class Workflow(BaseModel):
|
|
612
622
|
|
613
623
|
# VALIDATE: Change release value to Release object.
|
614
624
|
if isinstance(release, datetime):
|
615
|
-
release: Release = Release.from_dt(release)
|
625
|
+
release: Release = Release.from_dt(release, extras=self.extras)
|
616
626
|
|
617
627
|
result.trace.debug(
|
618
628
|
f"[RELEASE]: Start release - {name!r} : "
|
@@ -659,6 +669,7 @@ class Workflow(BaseModel):
|
|
659
669
|
parent_run_id=result.parent_run_id,
|
660
670
|
run_id=result.run_id,
|
661
671
|
execution_time=result.alive_time(),
|
672
|
+
extras=self.extras,
|
662
673
|
).save(excluded=None)
|
663
674
|
)
|
664
675
|
|
@@ -1042,6 +1053,7 @@ class Workflow(BaseModel):
|
|
1042
1053
|
run_id=run_id,
|
1043
1054
|
parent_run_id=parent_run_id,
|
1044
1055
|
id_logic=self.name,
|
1056
|
+
extras=self.extras,
|
1045
1057
|
)
|
1046
1058
|
|
1047
1059
|
result.trace.info(f"[WORKFLOW]: Start Execute: {self.name!r} ...")
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: ddeutil-workflow
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.48
|
4
4
|
Summary: Lightweight workflow orchestration
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
6
6
|
License: MIT
|
@@ -260,31 +260,31 @@ it will use default value and do not raise any error to you.
|
|
260
260
|
> The config value that you will set on the environment should combine with
|
261
261
|
> prefix, component, and name which is `WORKFLOW_{component}_{name}` (Upper case).
|
262
262
|
|
263
|
-
| Name | Component | Default |
|
264
|
-
|
265
|
-
| **ROOT_PATH** | Core | `.` |
|
266
|
-
| **REGISTRY_CALLER** | Core | `.` |
|
267
|
-
| **REGISTRY_FILTER** | Core | `ddeutil.workflow.templates` |
|
268
|
-
| **CONF_PATH** | Core | `conf` |
|
269
|
-
| **TIMEZONE** | Core | `Asia/Bangkok` |
|
270
|
-
| **STAGE_DEFAULT_ID** | Core | `true` |
|
271
|
-
| **STAGE_RAISE_ERROR** | Core | `false` |
|
272
|
-
| **JOB_DEFAULT_ID** | Core | `false` |
|
273
|
-
| **JOB_RAISE_ERROR** | Core | `true` |
|
274
|
-
| **MAX_CRON_PER_WORKFLOW** | Core | `5` |
|
275
|
-
| **MAX_QUEUE_COMPLETE_HIST** | Core | `16` |
|
276
|
-
| **GENERATE_ID_SIMPLE_MODE** | Core | `true` |
|
277
|
-
| **DEBUG_MODE** | Log | `true` |
|
278
|
-
| **FORMAT** | Log | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` |
|
279
|
-
| **FORMAT_FILE** | Log | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})` |
|
280
|
-
| **DATETIME_FORMAT** | Log | `%Y-%m-%d %H:%M:%S` |
|
281
|
-
| **TRACE_PATH** | Log | `./logs` |
|
282
|
-
| **TRACE_ENABLE_WRITE** | Log | `false` |
|
283
|
-
| **AUDIT_PATH** | Log | `./audits` |
|
284
|
-
| **AUDIT_ENABLE_WRITE** | Log | `true` |
|
285
|
-
| **MAX_PROCESS** | App | `2` |
|
286
|
-
| **MAX_SCHEDULE_PER_PROCESS** | App | `100` |
|
287
|
-
| **STOP_BOUNDARY_DELTA** | App | `'{"minutes": 5, "seconds": 20}'` |
|
263
|
+
| Name | Component | Default | Description |
|
264
|
+
|:-----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------|
|
265
|
+
| **ROOT_PATH** | Core | `.` | Root path or the project path for this workflow engine. |
|
266
|
+
| **REGISTRY_CALLER** | Core | `.` | List of importable string for the call stage. |
|
267
|
+
| **REGISTRY_FILTER** | Core | `ddeutil.workflow.templates` | List of importable string for the filter template. |
|
268
|
+
| **CONF_PATH** | Core | `conf` | The config path that keep all template `.yaml` files. |
|
269
|
+
| **TIMEZONE** | Core | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
270
|
+
| **STAGE_DEFAULT_ID** | Core | `true` | A flag that enable default stage ID that use for catch an execution output. |
|
271
|
+
| **STAGE_RAISE_ERROR** | Core | `false` | A flag that all stage raise StageException from stage execution. |
|
272
|
+
| **JOB_DEFAULT_ID** | Core | `false` | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
|
273
|
+
| **JOB_RAISE_ERROR** | Core | `true` | A flag that all job raise JobException from job strategy execution. |
|
274
|
+
| **MAX_CRON_PER_WORKFLOW** | Core | `5` | |
|
275
|
+
| **MAX_QUEUE_COMPLETE_HIST** | Core | `16` | |
|
276
|
+
| **GENERATE_ID_SIMPLE_MODE** | Core | `true` | A flog that enable generating ID with `md5` algorithm. |
|
277
|
+
| **DEBUG_MODE** | Log | `true` | A flag that enable logging with debug level mode. |
|
278
|
+
| **FORMAT** | Log | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | |
|
279
|
+
| **FORMAT_FILE** | Log | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})` | |
|
280
|
+
| **DATETIME_FORMAT** | Log | `%Y-%m-%d %H:%M:%S` | |
|
281
|
+
| **TRACE_PATH** | Log | `./logs` | The log path of the workflow saving log. |
|
282
|
+
| **TRACE_ENABLE_WRITE** | Log | `false` | |
|
283
|
+
| **AUDIT_PATH** | Log | `./audits` | |
|
284
|
+
| **AUDIT_ENABLE_WRITE** | Log | `true` | A flag that enable logging object saving log to its destination. |
|
285
|
+
| **MAX_PROCESS** | App | `2` | The maximum process worker number that run in scheduler app module. |
|
286
|
+
| **MAX_SCHEDULE_PER_PROCESS** | App | `100` | A schedule per process that run parallel. |
|
287
|
+
| **STOP_BOUNDARY_DELTA** | App | `'{"minutes": 5, "seconds": 20}'` | A time delta value that use to stop scheduler app in json string format. |
|
288
288
|
|
289
289
|
**API Application**:
|
290
290
|
|
@@ -0,0 +1,31 @@
|
|
1
|
+
ddeutil/workflow/__about__.py,sha256=OFynARvYDKZ4fFNVea1bykjJJKDpblDyUNtdv9rywxE,28
|
2
|
+
ddeutil/workflow/__cron.py,sha256=h8rLeIUAAEB2SdZ4Jhch7LU1Yl3bbJ-iNNJ3tQ0eYVM,28095
|
3
|
+
ddeutil/workflow/__init__.py,sha256=t7AaJ3gY7E8i2WeL3_8dYz-F5mzskUxsSAx7-Ny4Fhw,1927
|
4
|
+
ddeutil/workflow/__main__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
5
|
+
ddeutil/workflow/__types.py,sha256=8jBdbfb3aZSetjz0mvNrpGHwwxJff7mK8_4v41cLqlc,4316
|
6
|
+
ddeutil/workflow/conf.py,sha256=sXN7epudr90I1gUAjwJqvWTQV39mhO6cJhuuOuoYCw0,12153
|
7
|
+
ddeutil/workflow/cron.py,sha256=80SijzMdDOBxTWRsiF-Fmuz7Ym7leY0XT2lzRAPGdXc,8781
|
8
|
+
ddeutil/workflow/exceptions.py,sha256=uLNxzav3HRcr4vaZnvbUIF_eTR6UXXZNaxroMWFOUL4,1418
|
9
|
+
ddeutil/workflow/job.py,sha256=nvcSH1vxQrq8tGDrOs-8wHifaOLrfZdTAUk1vD-QtRA,30762
|
10
|
+
ddeutil/workflow/logs.py,sha256=o_EziK1MgP-7fJIl6bwE58BZHt8FBTrsmrEBhu1XGTo,24670
|
11
|
+
ddeutil/workflow/params.py,sha256=xCtFEh0-G-G-f8y_SXxyf31bU6Ox5p5Z-WbBFXrjy8M,9960
|
12
|
+
ddeutil/workflow/result.py,sha256=6yqWXFE__xMr8VY8xchBhBd3lyU-XX1nHOpx_2V5VGU,5390
|
13
|
+
ddeutil/workflow/reusables.py,sha256=7uamdx0nnBnDHcc0xXqwucItFYHUXI4_O-SHdFHIZCo,17528
|
14
|
+
ddeutil/workflow/scheduler.py,sha256=jyTLML8ppwdCrcuVw9ZMcZ1JwJ1SW6wDrJg5soHDFAw,27681
|
15
|
+
ddeutil/workflow/stages.py,sha256=mnP07SLvGRfggOV1i9bZ7_j5K_ksHlorCYb2crU1pus,49170
|
16
|
+
ddeutil/workflow/utils.py,sha256=sblje9qOtejCHVt8EVrbC0KY98vKqvxccaR5HIkRiTA,7363
|
17
|
+
ddeutil/workflow/workflow.py,sha256=V8uJw16gtjTd8T5aCpvSUr9z_oGQrV-ycybvUjA8NHI,50073
|
18
|
+
ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
|
19
|
+
ddeutil/workflow/api/api.py,sha256=CWtPLgOv2Jus9E7nzG5mG2Z32ZEkUK3JWQ2htZyMRpA,5244
|
20
|
+
ddeutil/workflow/api/log.py,sha256=NMTnOnsBrDB5129329xF2myLdrb-z9k1MQrmrP7qXJw,1818
|
21
|
+
ddeutil/workflow/api/repeat.py,sha256=uTtUFVLpiYYahXvCVx8sueRQ03K2Xw1id_gW3IMmX1U,5295
|
22
|
+
ddeutil/workflow/api/routes/__init__.py,sha256=qoGtOMyVgQ5nTUc8J8wH27A8isaxl3IFCX8qoyibeCY,484
|
23
|
+
ddeutil/workflow/api/routes/job.py,sha256=oPwBVP0Mxwxv-bGPlfmxQQ9PcVl0ev9HoPzndpYDCCQ,1954
|
24
|
+
ddeutil/workflow/api/routes/logs.py,sha256=TeRDrEelbKS2Hu_EovgLh0bOdmSv9mfnrIZsrE7uPD4,5353
|
25
|
+
ddeutil/workflow/api/routes/schedules.py,sha256=EgUjyRGhsm6UNaMj5luh6TcY6l571sCHcla-BL1iOfY,4829
|
26
|
+
ddeutil/workflow/api/routes/workflows.py,sha256=JcDOrn1deK8ztFRcMTNATQejG6KMA7JxZLVc4QeBsP4,4527
|
27
|
+
ddeutil_workflow-0.0.48.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
28
|
+
ddeutil_workflow-0.0.48.dist-info/METADATA,sha256=wv_dQPbCSS_1TuKmZ2jl1cWTRu0sM2eGtV3pBHHdWmQ,18841
|
29
|
+
ddeutil_workflow-0.0.48.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
|
30
|
+
ddeutil_workflow-0.0.48.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
31
|
+
ddeutil_workflow-0.0.48.dist-info/RECORD,,
|
@@ -1,31 +0,0 @@
|
|
1
|
-
ddeutil/workflow/__about__.py,sha256=6qZfpuPCbzNW572qdp7t-HFsC1JN75TQi_50aMab9kk,28
|
2
|
-
ddeutil/workflow/__cron.py,sha256=h8rLeIUAAEB2SdZ4Jhch7LU1Yl3bbJ-iNNJ3tQ0eYVM,28095
|
3
|
-
ddeutil/workflow/__init__.py,sha256=m7ZTCuUOarcTKJuXOyuaXd5WTIO7NTkqCeCrNX3d5i8,1943
|
4
|
-
ddeutil/workflow/__main__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
5
|
-
ddeutil/workflow/__types.py,sha256=8jBdbfb3aZSetjz0mvNrpGHwwxJff7mK8_4v41cLqlc,4316
|
6
|
-
ddeutil/workflow/conf.py,sha256=ZlaaLeZuBzqcnS-gfHQV58dJVMwQaRTjWxWZGCzX42s,12068
|
7
|
-
ddeutil/workflow/cron.py,sha256=80SijzMdDOBxTWRsiF-Fmuz7Ym7leY0XT2lzRAPGdXc,8781
|
8
|
-
ddeutil/workflow/exceptions.py,sha256=uLNxzav3HRcr4vaZnvbUIF_eTR6UXXZNaxroMWFOUL4,1418
|
9
|
-
ddeutil/workflow/job.py,sha256=uDT_lxAmtWDk6OYm6E4_rz_ngMdS5S03YF4D3WZMP8k,30676
|
10
|
-
ddeutil/workflow/logs.py,sha256=Ki1t6HkThwimzAe1OSxPPc7OQ4r-kXAc1kB63x2DsOg,21160
|
11
|
-
ddeutil/workflow/params.py,sha256=xCtFEh0-G-G-f8y_SXxyf31bU6Ox5p5Z-WbBFXrjy8M,9960
|
12
|
-
ddeutil/workflow/result.py,sha256=bysM6A194LPcivzmCrNNZsT8MphzhYAvqhjEYD6zryo,5145
|
13
|
-
ddeutil/workflow/reusables.py,sha256=vUsbh1dw5Cpojv98QTZarwBLHjvTMvR05H7XV76zUKQ,17537
|
14
|
-
ddeutil/workflow/scheduler.py,sha256=_MDsEHbBVOeF-381U8DfIMDyca_nG3XNXmgX4229_EU,27437
|
15
|
-
ddeutil/workflow/stages.py,sha256=oYEGWD-3kodtKSZ6JPAqAT5_za-sZI3GAKMJxeYUD8o,49009
|
16
|
-
ddeutil/workflow/utils.py,sha256=sblje9qOtejCHVt8EVrbC0KY98vKqvxccaR5HIkRiTA,7363
|
17
|
-
ddeutil/workflow/workflow.py,sha256=kEbPr2Wi9n5fDaCi5R26f4SHw7083_TdcIkZw-w7cEA,49716
|
18
|
-
ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
|
19
|
-
ddeutil/workflow/api/api.py,sha256=b-bMg0aRsEqt8Qb2hNUtamEt2Fq2CgNotF2oXSAdDu8,5226
|
20
|
-
ddeutil/workflow/api/log.py,sha256=NMTnOnsBrDB5129329xF2myLdrb-z9k1MQrmrP7qXJw,1818
|
21
|
-
ddeutil/workflow/api/repeat.py,sha256=cycd1-91j-4v6uY1SkrZHd9l95e-YgVC4UCSNNFuGJ8,5277
|
22
|
-
ddeutil/workflow/api/routes/__init__.py,sha256=qoGtOMyVgQ5nTUc8J8wH27A8isaxl3IFCX8qoyibeCY,484
|
23
|
-
ddeutil/workflow/api/routes/job.py,sha256=YVta083i8vU8-o4WdKFwDpfdC9vN1dZ6goZSmNlQXHA,1954
|
24
|
-
ddeutil/workflow/api/routes/logs.py,sha256=TeRDrEelbKS2Hu_EovgLh0bOdmSv9mfnrIZsrE7uPD4,5353
|
25
|
-
ddeutil/workflow/api/routes/schedules.py,sha256=rUWBm5RgLS1PNBHSWwWXJ0l-c5mYWfl9os0BA9_OTEw,4810
|
26
|
-
ddeutil/workflow/api/routes/workflows.py,sha256=ctgQGxXfpIV6bHFDM9IQ1_qaQHT6n5-HjJ1-D4GKWpc,4527
|
27
|
-
ddeutil_workflow-0.0.47.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
28
|
-
ddeutil_workflow-0.0.47.dist-info/METADATA,sha256=EngsQlJUAaJf_8irHXLk04p6F71UjQnBifd84g1GhNw,19116
|
29
|
-
ddeutil_workflow-0.0.47.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
|
30
|
-
ddeutil_workflow-0.0.47.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
31
|
-
ddeutil_workflow-0.0.47.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|