ddeutil-workflow 0.0.26__py3-none-any.whl → 0.0.26.post0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/api/route.py +40 -3
- ddeutil/workflow/conf.py +21 -22
- {ddeutil_workflow-0.0.26.dist-info → ddeutil_workflow-0.0.26.post0.dist-info}/METADATA +30 -29
- {ddeutil_workflow-0.0.26.dist-info → ddeutil_workflow-0.0.26.post0.dist-info}/RECORD +8 -8
- {ddeutil_workflow-0.0.26.dist-info → ddeutil_workflow-0.0.26.post0.dist-info}/LICENSE +0 -0
- {ddeutil_workflow-0.0.26.dist-info → ddeutil_workflow-0.0.26.post0.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.26.dist-info → ddeutil_workflow-0.0.26.post0.dist-info}/top_level.txt +0 -0
ddeutil/workflow/__about__.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__: str = "0.0.26"
|
|
1
|
+
__version__: str = "0.0.26.post0"
|
ddeutil/workflow/api/route.py
CHANGED
|
@@ -16,7 +16,7 @@ from fastapi.responses import UJSONResponse
|
|
|
16
16
|
from pydantic import BaseModel
|
|
17
17
|
|
|
18
18
|
from ..__types import DictData
|
|
19
|
-
from ..conf import Loader, config, get_logger
|
|
19
|
+
from ..conf import FileLog, Loader, config, get_logger
|
|
20
20
|
from ..result import Result
|
|
21
21
|
from ..scheduler import Schedule
|
|
22
22
|
from ..workflow import Workflow
|
|
@@ -99,12 +99,49 @@ async def execute_workflow(name: str, payload: ExecutePayload) -> DictData:
|
|
|
99
99
|
|
|
100
100
|
@workflow_route.get(path="/{name}/logs")
|
|
101
101
|
async def get_workflow_logs(name: str):
|
|
102
|
-
|
|
102
|
+
try:
|
|
103
|
+
return {
|
|
104
|
+
"message": f"Getting workflow {name!r} logs",
|
|
105
|
+
"logs": [
|
|
106
|
+
log.model_dump(
|
|
107
|
+
by_alias=True,
|
|
108
|
+
exclude_none=True,
|
|
109
|
+
exclude_unset=True,
|
|
110
|
+
exclude_defaults=True,
|
|
111
|
+
)
|
|
112
|
+
for log in FileLog.find_logs(name=name)
|
|
113
|
+
],
|
|
114
|
+
}
|
|
115
|
+
except FileNotFoundError:
|
|
116
|
+
raise HTTPException(
|
|
117
|
+
status_code=st.HTTP_404_NOT_FOUND,
|
|
118
|
+
detail=f"Does not found log for workflow {name!r}",
|
|
119
|
+
) from None
|
|
103
120
|
|
|
104
121
|
|
|
105
122
|
@workflow_route.get(path="/{name}/logs/{release}")
|
|
106
123
|
async def get_workflow_release_log(name: str, release: str):
|
|
107
|
-
|
|
124
|
+
try:
|
|
125
|
+
log: FileLog = FileLog.find_log_with_release(
|
|
126
|
+
name=name, release=datetime.strptime(release, "%Y%m%d%H%M%S")
|
|
127
|
+
)
|
|
128
|
+
except FileNotFoundError:
|
|
129
|
+
raise HTTPException(
|
|
130
|
+
status_code=st.HTTP_404_NOT_FOUND,
|
|
131
|
+
detail=(
|
|
132
|
+
f"Does not found log for workflow {name!r} "
|
|
133
|
+
f"with release {release!r}"
|
|
134
|
+
),
|
|
135
|
+
) from None
|
|
136
|
+
return {
|
|
137
|
+
"message": f"Getting workflow {name!r} log in release {release}",
|
|
138
|
+
"log": log.model_dump(
|
|
139
|
+
by_alias=True,
|
|
140
|
+
exclude_none=True,
|
|
141
|
+
exclude_unset=True,
|
|
142
|
+
exclude_defaults=True,
|
|
143
|
+
),
|
|
144
|
+
}
|
|
108
145
|
|
|
109
146
|
|
|
110
147
|
@workflow_route.delete(
|
ddeutil/workflow/conf.py
CHANGED
|
@@ -17,7 +17,7 @@ from typing import ClassVar, Optional, TypeVar, Union
|
|
|
17
17
|
from zoneinfo import ZoneInfo
|
|
18
18
|
|
|
19
19
|
from ddeutil.core import str2bool
|
|
20
|
-
from ddeutil.io import
|
|
20
|
+
from ddeutil.io import YamlFlResolve
|
|
21
21
|
from dotenv import load_dotenv
|
|
22
22
|
from pydantic import BaseModel, Field
|
|
23
23
|
from pydantic.functional_validators import model_validator
|
|
@@ -93,6 +93,7 @@ class Config:
|
|
|
93
93
|
enable_write_log: bool = str2bool(
|
|
94
94
|
os.getenv("WORKFLOW_LOG_ENABLE_WRITE", "false")
|
|
95
95
|
)
|
|
96
|
+
log_path: Path = Path(os.getenv("WORKFLOW_LOG_PATH", "./logs"))
|
|
96
97
|
|
|
97
98
|
# NOTE: Stage
|
|
98
99
|
stage_raise_error: bool = str2bool(
|
|
@@ -161,14 +162,6 @@ class Config:
|
|
|
161
162
|
f"timedelta with {self.stop_boundary_delta_str}."
|
|
162
163
|
) from err
|
|
163
164
|
|
|
164
|
-
def refresh_dotenv(self) -> Self:
|
|
165
|
-
"""Reload environment variables from the current stage."""
|
|
166
|
-
self.tz: ZoneInfo = ZoneInfo(env("WORKFLOW_CORE_TIMEZONE", "UTC"))
|
|
167
|
-
self.stage_raise_error: bool = str2bool(
|
|
168
|
-
env("WORKFLOW_CORE_STAGE_RAISE_ERROR", "false")
|
|
169
|
-
)
|
|
170
|
-
return self
|
|
171
|
-
|
|
172
165
|
@property
|
|
173
166
|
def conf_path(self) -> Path:
|
|
174
167
|
"""Config path that use root_path class argument for this construction.
|
|
@@ -213,8 +206,14 @@ class SimLoad:
|
|
|
213
206
|
externals: DictData | None = None,
|
|
214
207
|
) -> None:
|
|
215
208
|
self.data: DictData = {}
|
|
216
|
-
for file in
|
|
217
|
-
if
|
|
209
|
+
for file in conf.conf_path.rglob("*"):
|
|
210
|
+
if not file.is_file():
|
|
211
|
+
continue
|
|
212
|
+
|
|
213
|
+
if data := self.filter_suffix(
|
|
214
|
+
file,
|
|
215
|
+
name,
|
|
216
|
+
):
|
|
218
217
|
self.data = data
|
|
219
218
|
|
|
220
219
|
# VALIDATE: check the data that reading should not empty.
|
|
@@ -246,7 +245,10 @@ class SimLoad:
|
|
|
246
245
|
:rtype: Iterator[tuple[str, DictData]]
|
|
247
246
|
"""
|
|
248
247
|
exclude: list[str] = excluded or []
|
|
249
|
-
for file in
|
|
248
|
+
for file in conf.conf_path.rglob("*"):
|
|
249
|
+
|
|
250
|
+
if not file.is_file():
|
|
251
|
+
continue
|
|
250
252
|
|
|
251
253
|
for key, data in cls.filter_suffix(file).items():
|
|
252
254
|
|
|
@@ -362,7 +364,7 @@ class FileLog(BaseLog):
|
|
|
362
364
|
"""
|
|
363
365
|
|
|
364
366
|
filename_fmt: ClassVar[str] = (
|
|
365
|
-
"
|
|
367
|
+
"workflow={name}/release={release:%Y%m%d%H%M%S}"
|
|
366
368
|
)
|
|
367
369
|
|
|
368
370
|
def do_before(self) -> None:
|
|
@@ -378,18 +380,16 @@ class FileLog(BaseLog):
|
|
|
378
380
|
|
|
379
381
|
:rtype: Iterator[Self]
|
|
380
382
|
"""
|
|
381
|
-
pointer: Path = config.
|
|
383
|
+
pointer: Path = config.log_path / f"workflow={name}"
|
|
382
384
|
if not pointer.exists():
|
|
383
|
-
raise FileNotFoundError(
|
|
384
|
-
f"Pointer: ./logs/workflow={name} does not found."
|
|
385
|
-
)
|
|
385
|
+
raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
|
|
386
386
|
|
|
387
387
|
for file in pointer.glob("./release=*/*.log"):
|
|
388
388
|
with file.open(mode="r", encoding="utf-8") as f:
|
|
389
389
|
yield cls.model_validate(obj=json.load(f))
|
|
390
390
|
|
|
391
391
|
@classmethod
|
|
392
|
-
def
|
|
392
|
+
def find_log_with_release(
|
|
393
393
|
cls,
|
|
394
394
|
name: str,
|
|
395
395
|
release: datetime | None = None,
|
|
@@ -410,8 +410,7 @@ class FileLog(BaseLog):
|
|
|
410
410
|
raise NotImplementedError("Find latest log does not implement yet.")
|
|
411
411
|
|
|
412
412
|
pointer: Path = (
|
|
413
|
-
config.
|
|
414
|
-
/ f"./logs/workflow={name}/release={release:%Y%m%d%H%M%S}"
|
|
413
|
+
config.log_path / f"workflow={name}/release={release:%Y%m%d%H%M%S}"
|
|
415
414
|
)
|
|
416
415
|
if not pointer.exists():
|
|
417
416
|
raise FileNotFoundError(
|
|
@@ -440,7 +439,7 @@ class FileLog(BaseLog):
|
|
|
440
439
|
return False
|
|
441
440
|
|
|
442
441
|
# NOTE: create pointer path that use the same logic of pointer method.
|
|
443
|
-
pointer: Path = config.
|
|
442
|
+
pointer: Path = config.log_path / cls.filename_fmt.format(
|
|
444
443
|
name=name, release=release
|
|
445
444
|
)
|
|
446
445
|
|
|
@@ -451,7 +450,7 @@ class FileLog(BaseLog):
|
|
|
451
450
|
|
|
452
451
|
:rtype: Path
|
|
453
452
|
"""
|
|
454
|
-
return config.
|
|
453
|
+
return config.log_path / self.filename_fmt.format(
|
|
455
454
|
name=self.name, release=self.release
|
|
456
455
|
)
|
|
457
456
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: ddeutil-workflow
|
|
3
|
-
Version: 0.0.26
|
|
3
|
+
Version: 0.0.26.post0
|
|
4
4
|
Summary: Lightweight workflow orchestration with less dependencies
|
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -99,7 +99,7 @@ use-case.
|
|
|
99
99
|
run-py-local:
|
|
100
100
|
|
|
101
101
|
# Validate model that use to parsing exists for template file
|
|
102
|
-
type:
|
|
102
|
+
type: Workflow
|
|
103
103
|
on:
|
|
104
104
|
# If workflow deploy to schedule, it will running every 5 minutes
|
|
105
105
|
# with Asia/Bangkok timezone.
|
|
@@ -150,7 +150,7 @@ So, this package provide the `Schedule` template for this action.
|
|
|
150
150
|
schedule-run-local-wf:
|
|
151
151
|
|
|
152
152
|
# Validate model that use to parsing exists for template file
|
|
153
|
-
type:
|
|
153
|
+
type: Schedule
|
|
154
154
|
workflows:
|
|
155
155
|
|
|
156
156
|
# Map existing workflow that want to deploy with scheduler application.
|
|
@@ -168,35 +168,36 @@ The main configuration that use to dynamic changing with your propose of this
|
|
|
168
168
|
application. If any configuration values do not set yet, it will use default value
|
|
169
169
|
and do not raise any error to you.
|
|
170
170
|
|
|
171
|
-
| Environment
|
|
172
|
-
|
|
173
|
-
|
|
|
174
|
-
|
|
|
175
|
-
|
|
|
176
|
-
|
|
|
177
|
-
|
|
|
178
|
-
|
|
|
179
|
-
|
|
|
180
|
-
|
|
|
181
|
-
|
|
|
182
|
-
|
|
|
183
|
-
|
|
|
184
|
-
|
|
|
185
|
-
|
|
|
186
|
-
|
|
|
187
|
-
|
|
|
188
|
-
|
|
|
189
|
-
|
|
|
190
|
-
|
|
|
191
|
-
|
|
|
192
|
-
|
|
|
171
|
+
| Environment | Component | Default | Description | Remark |
|
|
172
|
+
|:-------------------------------------------|:---------:|:-----------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------|--------|
|
|
173
|
+
| **WORKFLOW_ROOT_PATH** | Core | `.` | The root path of the workflow application. | |
|
|
174
|
+
| **WORKFLOW_CORE_REGISTRY** | Core | `src,src.ddeutil.workflow,tests,tests.utils` | List of importable string for the hook stage. | |
|
|
175
|
+
| **WORKFLOW_CORE_REGISTRY_FILTER** | Core | `src.ddeutil.workflow.utils,ddeutil.workflow.utils` | List of importable string for the filter template. | |
|
|
176
|
+
| **WORKFLOW_CORE_PATH_CONF** | Core | `conf` | The config path that keep all template `.yaml` files. | |
|
|
177
|
+
| **WORKFLOW_CORE_TIMEZONE** | Core | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. | |
|
|
178
|
+
| **WORKFLOW_CORE_STAGE_DEFAULT_ID** | Core | `true` | A flag that enable default stage ID that use for catch an execution output. | |
|
|
179
|
+
| **WORKFLOW_CORE_STAGE_RAISE_ERROR** | Core | `false` | A flag that all stage raise StageException from stage execution. | |
|
|
180
|
+
| **WORKFLOW_CORE_JOB_DEFAULT_ID** | Core | `false` | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. | |
|
|
181
|
+
| **WORKFLOW_CORE_JOB_RAISE_ERROR** | Core | `true` | A flag that all job raise JobException from job strategy execution. | |
|
|
182
|
+
| **WORKFLOW_CORE_MAX_NUM_POKING** | Core | `4` | . | |
|
|
183
|
+
| **WORKFLOW_CORE_MAX_JOB_PARALLEL** | Core | `2` | The maximum job number that able to run parallel in workflow executor. | |
|
|
184
|
+
| **WORKFLOW_CORE_MAX_JOB_EXEC_TIMEOUT** | Core | `600` | | |
|
|
185
|
+
| **WORKFLOW_CORE_MAX_CRON_PER_WORKFLOW** | Core | `5` | | |
|
|
186
|
+
| **WORKFLOW_CORE_MAX_QUEUE_COMPLETE_HIST** | Core | `16` | | |
|
|
187
|
+
| **WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE** | Core | `true` | A flog that enable generating ID with `md5` algorithm. | |
|
|
188
|
+
| **WORKFLOW_LOG_PATH** | Log | `./logs` | The log path of the workflow saving log. | |
|
|
189
|
+
| **WORKFLOW_LOG_DEBUG_MODE** | Log | `true` | A flag that enable logging with debug level mode. | |
|
|
190
|
+
| **WORKFLOW_LOG_ENABLE_WRITE** | Log | `true` | A flag that enable logging object saving log to its destination. | |
|
|
191
|
+
| **WORKFLOW_APP_MAX_PROCESS** | Schedule | `2` | The maximum process worker number that run in scheduler app module. | |
|
|
192
|
+
| **WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS** | Schedule | `100` | A schedule per process that run parallel. | |
|
|
193
|
+
| **WORKFLOW_APP_STOP_BOUNDARY_DELTA** | Schedule | `'{"minutes": 5, "seconds": 20}'` | A time delta value that use to stop scheduler app in json string format. | |
|
|
193
194
|
|
|
194
195
|
**API Application**:
|
|
195
196
|
|
|
196
|
-
| Environment
|
|
197
|
-
|
|
198
|
-
|
|
|
199
|
-
|
|
|
197
|
+
| Environment | Component | Default | Description | Remark |
|
|
198
|
+
|:----------------------------------------|:-----------:|---------|------------------------------------------------------------------------------------|--------|
|
|
199
|
+
| **WORKFLOW_API_ENABLE_ROUTE_WORKFLOW** | API | `true` | A flag that enable workflow route to manage execute manually and workflow logging. | |
|
|
200
|
+
| **WORKFLOW_API_ENABLE_ROUTE_SCHEDULE** | API | `true` | A flag that enable run scheduler. | |
|
|
200
201
|
|
|
201
202
|
## :rocket: Deployment
|
|
202
203
|
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
ddeutil/workflow/__about__.py,sha256=
|
|
1
|
+
ddeutil/workflow/__about__.py,sha256=9NeOhqXsrgLKL6AmtINFl4dct9jvs8oismQYXutnmJ8,34
|
|
2
2
|
ddeutil/workflow/__cron.py,sha256=uA8XcbY_GwA9rJSHaHUaXaJyGDObJN0ZeYlJSinL8y8,26880
|
|
3
3
|
ddeutil/workflow/__init__.py,sha256=49eGrCuchPVZKMybRouAviNhbulK_F6VwCmLm76hIss,1478
|
|
4
4
|
ddeutil/workflow/__types.py,sha256=Ia7f38kvL3NibwmRKi0wQ1ud_45Z-SojYGhNJwIqcu8,3713
|
|
5
|
-
ddeutil/workflow/conf.py,sha256=
|
|
5
|
+
ddeutil/workflow/conf.py,sha256=MtjeaapHaEuW8GjZadA56KpkmbPQg6Ws_CHcmmAzaFc,15174
|
|
6
6
|
ddeutil/workflow/cron.py,sha256=75A0hqevvouziKoLALncLJspVAeki9qCH3zniAJaxzY,7513
|
|
7
7
|
ddeutil/workflow/exceptions.py,sha256=P56K7VD3etGm9y-k_GXrzEyqsTCaz9EJazTIshZDf9g,943
|
|
8
8
|
ddeutil/workflow/job.py,sha256=cvSLMdc1sMl1MeU7so7Oe2SdRYxQwt6hm55mLV1iP-Y,24219
|
|
@@ -15,9 +15,9 @@ ddeutil/workflow/workflow.py,sha256=AD0rs1tRT2EpvUyNVAEr2bBPgF6-KOzGmLedR3o4y0Q,
|
|
|
15
15
|
ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
|
|
16
16
|
ddeutil/workflow/api/api.py,sha256=Md1cz3Edc7_uz63s_L_i-R3IE4mkO3aTADrX8GOGU-Y,5644
|
|
17
17
|
ddeutil/workflow/api/repeat.py,sha256=zyvsrXKk-3-_N8ZRZSki0Mueshugum2jtqctEOp9QSc,4927
|
|
18
|
-
ddeutil/workflow/api/route.py,sha256=
|
|
19
|
-
ddeutil_workflow-0.0.26.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
|
20
|
-
ddeutil_workflow-0.0.26.dist-info/METADATA,sha256=
|
|
21
|
-
ddeutil_workflow-0.0.26.dist-info/WHEEL,sha256=A3WOREP4zgxI0fKrHUG8DC8013e3dK3n7a6HDbcEIwE,91
|
|
22
|
-
ddeutil_workflow-0.0.26.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
|
23
|
-
ddeutil_workflow-0.0.26.dist-info/RECORD,,
|
|
18
|
+
ddeutil/workflow/api/route.py,sha256=v96jNbgjM1cJ2MpVSRWs2kgRqF8DQElEBdRZrVFEpEw,8578
|
|
19
|
+
ddeutil_workflow-0.0.26.post0.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
|
20
|
+
ddeutil_workflow-0.0.26.post0.dist-info/METADATA,sha256=GRkczE6ZJ7NhFefqGI2GbzGd6Lu2FDGO6oUafA8n4nw,14364
|
|
21
|
+
ddeutil_workflow-0.0.26.post0.dist-info/WHEEL,sha256=A3WOREP4zgxI0fKrHUG8DC8013e3dK3n7a6HDbcEIwE,91
|
|
22
|
+
ddeutil_workflow-0.0.26.post0.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
|
23
|
+
ddeutil_workflow-0.0.26.post0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|