ddeutil-workflow 0.0.66__py3-none-any.whl → 0.0.68__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/api/__init__.py +20 -19
- ddeutil/workflow/api/{logs.py → log_conf.py} +28 -15
- ddeutil/workflow/api/routes/__init__.py +3 -3
- ddeutil/workflow/api/routes/job.py +42 -16
- ddeutil/workflow/api/routes/logs.py +7 -7
- ddeutil/workflow/api/routes/workflows.py +10 -9
- ddeutil/workflow/cli.py +62 -9
- ddeutil/workflow/conf.py +7 -3
- ddeutil/workflow/errors.py +0 -3
- ddeutil/workflow/event.py +4 -3
- ddeutil/workflow/logs.py +18 -14
- ddeutil/workflow/result.py +12 -7
- ddeutil/workflow/stages.py +149 -35
- ddeutil/workflow/utils.py +1 -52
- {ddeutil_workflow-0.0.66.dist-info → ddeutil_workflow-0.0.68.dist-info}/METADATA +14 -36
- ddeutil_workflow-0.0.68.dist-info/RECORD +29 -0
- {ddeutil_workflow-0.0.66.dist-info → ddeutil_workflow-0.0.68.dist-info}/WHEEL +1 -1
- ddeutil_workflow-0.0.66.dist-info/RECORD +0 -29
- {ddeutil_workflow-0.0.66.dist-info → ddeutil_workflow-0.0.68.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.66.dist-info → ddeutil_workflow-0.0.68.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.66.dist-info → ddeutil_workflow-0.0.68.dist-info}/top_level.txt +0 -0
ddeutil/workflow/__about__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__: str = "0.0.
|
1
|
+
__version__: str = "0.0.68"
|
ddeutil/workflow/api/__init__.py
CHANGED
@@ -6,6 +6,7 @@
|
|
6
6
|
from __future__ import annotations
|
7
7
|
|
8
8
|
import contextlib
|
9
|
+
import logging
|
9
10
|
from collections.abc import AsyncIterator
|
10
11
|
|
11
12
|
from dotenv import load_dotenv
|
@@ -19,11 +20,10 @@ from fastapi.responses import UJSONResponse
|
|
19
20
|
|
20
21
|
from ..__about__ import __version__
|
21
22
|
from ..conf import api_config
|
22
|
-
from ..logs import get_logger
|
23
23
|
from .routes import job, log, workflow
|
24
24
|
|
25
25
|
load_dotenv()
|
26
|
-
logger =
|
26
|
+
logger = logging.getLogger("uvicorn.error")
|
27
27
|
|
28
28
|
|
29
29
|
@contextlib.asynccontextmanager
|
@@ -58,12 +58,16 @@ app.add_middleware(
|
|
58
58
|
|
59
59
|
|
60
60
|
@app.get(path="/", response_class=UJSONResponse)
|
61
|
-
async def health():
|
61
|
+
async def health() -> UJSONResponse:
|
62
62
|
"""Index view that not return any template without json status."""
|
63
|
-
|
63
|
+
logger.info("[API]: Workflow API Application already running ...")
|
64
|
+
return UJSONResponse(
|
65
|
+
content={"message": "Workflow already start up with healthy status."},
|
66
|
+
status_code=st.HTTP_200_OK,
|
67
|
+
)
|
64
68
|
|
65
69
|
|
66
|
-
# NOTE Add the jobs and logs routes by default.
|
70
|
+
# NOTE: Add the jobs and logs routes by default.
|
67
71
|
app.include_router(job, prefix=api_config.prefix_path)
|
68
72
|
app.include_router(log, prefix=api_config.prefix_path)
|
69
73
|
app.include_router(workflow, prefix=api_config.prefix_path)
|
@@ -71,21 +75,18 @@ app.include_router(workflow, prefix=api_config.prefix_path)
|
|
71
75
|
|
72
76
|
@app.exception_handler(RequestValidationError)
|
73
77
|
async def validation_exception_handler(
|
74
|
-
request: Request,
|
75
|
-
|
78
|
+
request: Request,
|
79
|
+
exc: RequestValidationError,
|
80
|
+
) -> UJSONResponse:
|
81
|
+
"""Error Handler for model validate does not valid."""
|
76
82
|
_ = request
|
77
83
|
return UJSONResponse(
|
78
84
|
status_code=st.HTTP_422_UNPROCESSABLE_ENTITY,
|
79
|
-
content=jsonable_encoder(
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
uvicorn.run(
|
87
|
-
app,
|
88
|
-
host="0.0.0.0",
|
89
|
-
port=80,
|
90
|
-
log_level="DEBUG",
|
85
|
+
content=jsonable_encoder(
|
86
|
+
{
|
87
|
+
"message": "Body does not parsing with model.",
|
88
|
+
"detail": exc.errors(),
|
89
|
+
"body": exc.body,
|
90
|
+
}
|
91
|
+
),
|
91
92
|
)
|
@@ -1,6 +1,8 @@
|
|
1
|
+
from typing import Any
|
2
|
+
|
1
3
|
from ..conf import config
|
2
4
|
|
3
|
-
LOGGING_CONFIG = { # pragma: no cov
|
5
|
+
LOGGING_CONFIG: dict[str, Any] = { # pragma: no cov
|
4
6
|
"version": 1,
|
5
7
|
"disable_existing_loggers": False,
|
6
8
|
"formatters": {
|
@@ -22,38 +24,49 @@ LOGGING_CONFIG = { # pragma: no cov
|
|
22
24
|
"stream": "ext://sys.stderr",
|
23
25
|
},
|
24
26
|
"stream_handler": {
|
27
|
+
# "formatter": "standard",
|
25
28
|
"formatter": "custom_formatter",
|
26
29
|
"class": "logging.StreamHandler",
|
27
30
|
"stream": "ext://sys.stdout",
|
28
31
|
},
|
29
|
-
"file_handler": {
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
},
|
32
|
+
# "file_handler": {
|
33
|
+
# "formatter": "custom_formatter",
|
34
|
+
# "class": "logging.handlers.RotatingFileHandler",
|
35
|
+
# "filename": "logs/app.log",
|
36
|
+
# "maxBytes": 1024 * 1024 * 1,
|
37
|
+
# "backupCount": 3,
|
38
|
+
# },
|
36
39
|
},
|
37
40
|
"loggers": {
|
38
41
|
"uvicorn": {
|
39
|
-
"handlers": ["default", "file_handler"],
|
42
|
+
# "handlers": ["default", "file_handler"],
|
43
|
+
"handlers": ["default"],
|
40
44
|
"level": "DEBUG" if config.debug else "INFO",
|
41
45
|
"propagate": False,
|
42
46
|
},
|
43
47
|
"uvicorn.access": {
|
44
|
-
"handlers": ["stream_handler", "file_handler"],
|
48
|
+
# "handlers": ["stream_handler", "file_handler"],
|
49
|
+
"handlers": ["stream_handler"],
|
45
50
|
"level": "DEBUG" if config.debug else "INFO",
|
46
51
|
"propagate": False,
|
47
52
|
},
|
48
53
|
"uvicorn.error": {
|
49
|
-
"handlers": ["stream_handler", "file_handler"],
|
54
|
+
# "handlers": ["stream_handler", "file_handler"],
|
55
|
+
"handlers": ["stream_handler"],
|
50
56
|
"level": "DEBUG" if config.debug else "INFO",
|
51
57
|
"propagate": False,
|
52
58
|
},
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
59
|
+
"uvicorn.asgi": {
|
60
|
+
# "handlers": ["stream_handler", "file_handler"],
|
61
|
+
"handlers": ["stream_handler"],
|
62
|
+
"level": "TRACE",
|
63
|
+
"propagate": False,
|
64
|
+
},
|
65
|
+
# "ddeutil.workflow": {
|
66
|
+
# "handlers": ["stream_handler"],
|
67
|
+
# "level": "INFO",
|
68
|
+
# # "propagate": False,
|
69
|
+
# "propagate": True,
|
57
70
|
# },
|
58
71
|
},
|
59
72
|
}
|
@@ -3,6 +3,6 @@
|
|
3
3
|
# Licensed under the MIT License. See LICENSE in the project root for
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
|
-
from .job import
|
7
|
-
from .logs import
|
8
|
-
from .workflows import
|
6
|
+
from .job import router as job
|
7
|
+
from .logs import router as log
|
8
|
+
from .workflows import router as workflow
|
@@ -5,20 +5,21 @@
|
|
5
5
|
# ------------------------------------------------------------------------------
|
6
6
|
from __future__ import annotations
|
7
7
|
|
8
|
+
import logging
|
8
9
|
from typing import Any, Optional
|
9
10
|
|
10
11
|
from fastapi import APIRouter
|
12
|
+
from fastapi import status as st
|
11
13
|
from fastapi.responses import UJSONResponse
|
12
14
|
from pydantic import BaseModel, Field
|
13
15
|
|
14
16
|
from ...__types import DictData
|
15
17
|
from ...errors import JobError
|
16
18
|
from ...job import Job
|
17
|
-
from ...logs import get_logger
|
18
19
|
from ...result import Result
|
19
20
|
|
20
|
-
logger =
|
21
|
-
|
21
|
+
logger = logging.getLogger("uvicorn.error")
|
22
|
+
router = APIRouter(prefix="/job", tags=["job"])
|
22
23
|
|
23
24
|
|
24
25
|
class ResultCreate(BaseModel):
|
@@ -32,14 +33,19 @@ class ResultCreate(BaseModel):
|
|
32
33
|
)
|
33
34
|
|
34
35
|
|
35
|
-
@
|
36
|
+
@router.post(
|
37
|
+
path="/execute/",
|
38
|
+
response_class=UJSONResponse,
|
39
|
+
status_code=st.HTTP_200_OK,
|
40
|
+
)
|
36
41
|
async def job_execute(
|
37
42
|
result: ResultCreate,
|
38
43
|
job: Job,
|
39
44
|
params: dict[str, Any],
|
40
45
|
extras: Optional[dict[str, Any]] = None,
|
41
|
-
):
|
46
|
+
) -> UJSONResponse:
|
42
47
|
"""Execute job via RestAPI with execute route path."""
|
48
|
+
logger.info("[API]: Start execute job ...")
|
43
49
|
rs: Result = Result(
|
44
50
|
run_id=result.run_id,
|
45
51
|
parent_run_id=result.parent_run_id,
|
@@ -61,15 +67,35 @@ async def job_execute(
|
|
61
67
|
)
|
62
68
|
except JobError as err:
|
63
69
|
rs.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
|
70
|
+
return UJSONResponse(
|
71
|
+
content={
|
72
|
+
"message": str(err),
|
73
|
+
"result": {
|
74
|
+
"run_id": rs.run_id,
|
75
|
+
"parent_run_id": rs.parent_run_id,
|
76
|
+
},
|
77
|
+
"job": job.model_dump(
|
78
|
+
by_alias=True,
|
79
|
+
exclude_none=False,
|
80
|
+
exclude_unset=True,
|
81
|
+
),
|
82
|
+
"params": params,
|
83
|
+
"context": context,
|
84
|
+
},
|
85
|
+
status_code=st.HTTP_500_INTERNAL_SERVER_ERROR,
|
86
|
+
)
|
64
87
|
|
65
|
-
return
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
88
|
+
return UJSONResponse(
|
89
|
+
content={
|
90
|
+
"message": "Execute job via RestAPI successful.",
|
91
|
+
"result": {"run_id": rs.run_id, "parent_run_id": rs.parent_run_id},
|
92
|
+
"job": job.model_dump(
|
93
|
+
by_alias=True,
|
94
|
+
exclude_none=False,
|
95
|
+
exclude_unset=True,
|
96
|
+
),
|
97
|
+
"params": params,
|
98
|
+
"context": context,
|
99
|
+
},
|
100
|
+
status_code=st.HTTP_200_OK,
|
101
|
+
)
|
@@ -13,14 +13,14 @@ from fastapi.responses import UJSONResponse
|
|
13
13
|
from ...logs import get_audit
|
14
14
|
from ...result import Result
|
15
15
|
|
16
|
-
|
16
|
+
router = APIRouter(
|
17
17
|
prefix="/logs",
|
18
18
|
tags=["logs"],
|
19
19
|
default_response_class=UJSONResponse,
|
20
20
|
)
|
21
21
|
|
22
22
|
|
23
|
-
@
|
23
|
+
@router.get(
|
24
24
|
path="/traces/",
|
25
25
|
response_class=UJSONResponse,
|
26
26
|
status_code=st.HTTP_200_OK,
|
@@ -50,7 +50,7 @@ async def get_traces(
|
|
50
50
|
}
|
51
51
|
|
52
52
|
|
53
|
-
@
|
53
|
+
@router.get(
|
54
54
|
path="/traces/{run_id}",
|
55
55
|
response_class=UJSONResponse,
|
56
56
|
status_code=st.HTTP_200_OK,
|
@@ -77,7 +77,7 @@ async def get_trace_with_id(run_id: str):
|
|
77
77
|
}
|
78
78
|
|
79
79
|
|
80
|
-
@
|
80
|
+
@router.get(
|
81
81
|
path="/audits/",
|
82
82
|
response_class=UJSONResponse,
|
83
83
|
status_code=st.HTTP_200_OK,
|
@@ -94,7 +94,7 @@ async def get_audits():
|
|
94
94
|
}
|
95
95
|
|
96
96
|
|
97
|
-
@
|
97
|
+
@router.get(
|
98
98
|
path="/audits/{workflow}/",
|
99
99
|
response_class=UJSONResponse,
|
100
100
|
status_code=st.HTTP_200_OK,
|
@@ -113,7 +113,7 @@ async def get_audit_with_workflow(workflow: str):
|
|
113
113
|
}
|
114
114
|
|
115
115
|
|
116
|
-
@
|
116
|
+
@router.get(
|
117
117
|
path="/audits/{workflow}/{release}",
|
118
118
|
response_class=UJSONResponse,
|
119
119
|
status_code=st.HTTP_200_OK,
|
@@ -140,7 +140,7 @@ async def get_audit_with_workflow_release(
|
|
140
140
|
}
|
141
141
|
|
142
142
|
|
143
|
-
@
|
143
|
+
@router.get(
|
144
144
|
path="/audits/{workflow}/{release}/{run_id}",
|
145
145
|
response_class=UJSONResponse,
|
146
146
|
status_code=st.HTTP_200_OK,
|
@@ -5,6 +5,7 @@
|
|
5
5
|
# ------------------------------------------------------------------------------
|
6
6
|
from __future__ import annotations
|
7
7
|
|
8
|
+
import logging
|
8
9
|
from dataclasses import asdict
|
9
10
|
from datetime import datetime
|
10
11
|
from typing import Any
|
@@ -16,19 +17,19 @@ from pydantic import BaseModel
|
|
16
17
|
|
17
18
|
from ...__types import DictData
|
18
19
|
from ...conf import Loader
|
19
|
-
from ...logs import
|
20
|
+
from ...logs import AuditModel, get_audit
|
20
21
|
from ...result import Result
|
21
22
|
from ...workflow import Workflow
|
22
23
|
|
23
|
-
logger =
|
24
|
-
|
24
|
+
logger = logging.getLogger("uvicorn.error")
|
25
|
+
router = APIRouter(
|
25
26
|
prefix="/workflows",
|
26
27
|
tags=["workflows"],
|
27
28
|
default_response_class=UJSONResponse,
|
28
29
|
)
|
29
30
|
|
30
31
|
|
31
|
-
@
|
32
|
+
@router.get(path="/", status_code=st.HTTP_200_OK)
|
32
33
|
async def get_workflows() -> DictData:
|
33
34
|
"""Return all workflow workflows that exists in config path."""
|
34
35
|
workflows: DictData = dict(Loader.finds(Workflow))
|
@@ -39,7 +40,7 @@ async def get_workflows() -> DictData:
|
|
39
40
|
}
|
40
41
|
|
41
42
|
|
42
|
-
@
|
43
|
+
@router.get(path="/{name}", status_code=st.HTTP_200_OK)
|
43
44
|
async def get_workflow_by_name(name: str) -> DictData:
|
44
45
|
"""Return model of workflow that passing an input workflow name."""
|
45
46
|
try:
|
@@ -63,7 +64,7 @@ class ExecutePayload(BaseModel):
|
|
63
64
|
params: dict[str, Any]
|
64
65
|
|
65
66
|
|
66
|
-
@
|
67
|
+
@router.post(path="/{name}/execute", status_code=st.HTTP_202_ACCEPTED)
|
67
68
|
async def workflow_execute(name: str, payload: ExecutePayload) -> DictData:
|
68
69
|
"""Return model of workflow that passing an input workflow name."""
|
69
70
|
try:
|
@@ -88,7 +89,7 @@ async def workflow_execute(name: str, payload: ExecutePayload) -> DictData:
|
|
88
89
|
return asdict(result)
|
89
90
|
|
90
91
|
|
91
|
-
@
|
92
|
+
@router.get(path="/{name}/audits", status_code=st.HTTP_200_OK)
|
92
93
|
async def get_workflow_audits(name: str):
|
93
94
|
try:
|
94
95
|
return {
|
@@ -109,11 +110,11 @@ async def get_workflow_audits(name: str):
|
|
109
110
|
) from None
|
110
111
|
|
111
112
|
|
112
|
-
@
|
113
|
+
@router.get(path="/{name}/audits/{release}", status_code=st.HTTP_200_OK)
|
113
114
|
async def get_workflow_release_audit(name: str, release: str):
|
114
115
|
"""Get Workflow audit log with an input release value."""
|
115
116
|
try:
|
116
|
-
audit:
|
117
|
+
audit: AuditModel = get_audit().find_audit_with_release(
|
117
118
|
name=name,
|
118
119
|
release=datetime.strptime(release, "%Y%m%d%H%M%S"),
|
119
120
|
)
|
ddeutil/workflow/cli.py
CHANGED
@@ -1,12 +1,17 @@
|
|
1
1
|
import json
|
2
|
-
from
|
2
|
+
from pathlib import Path
|
3
|
+
from platform import python_version
|
4
|
+
from typing import Annotated, Any, Optional
|
3
5
|
|
4
6
|
import typer
|
5
7
|
import uvicorn
|
6
8
|
|
7
9
|
from .__about__ import __version__
|
10
|
+
from .__types import DictData
|
8
11
|
from .api import app as fastapp
|
9
|
-
from .
|
12
|
+
from .errors import JobError
|
13
|
+
from .job import Job
|
14
|
+
from .result import Result
|
10
15
|
|
11
16
|
app = typer.Typer(
|
12
17
|
pretty_exceptions_enable=True,
|
@@ -15,22 +20,26 @@ app = typer.Typer(
|
|
15
20
|
|
16
21
|
@app.callback()
|
17
22
|
def callback():
|
23
|
+
"""Manage Workflow CLI app.
|
24
|
+
|
25
|
+
Use it with the interface workflow engine.
|
18
26
|
"""
|
19
|
-
Awesome Portal Gun
|
20
|
-
"""
|
21
|
-
typer.echo("Start call from callback function")
|
22
27
|
|
23
28
|
|
24
29
|
@app.command()
|
25
30
|
def version():
|
26
31
|
"""Get the ddeutil-workflow package version."""
|
27
|
-
typer.echo(__version__)
|
32
|
+
typer.echo(f"ddeutil-workflow=={__version__}")
|
33
|
+
typer.echo(f"python-version=={python_version()}")
|
28
34
|
|
29
35
|
|
30
36
|
@app.command()
|
31
37
|
def job(
|
32
38
|
params: Annotated[str, typer.Option(help="A job execute parameters")],
|
33
|
-
)
|
39
|
+
job: Annotated[str, typer.Option(help="A job model")],
|
40
|
+
parent_run_id: Annotated[str, typer.Option(help="A parent running ID")],
|
41
|
+
run_id: Annotated[Optional[str], typer.Option(help="A running ID")] = None,
|
42
|
+
) -> None:
|
34
43
|
"""Job execution on the local.
|
35
44
|
|
36
45
|
Example:
|
@@ -39,8 +48,32 @@ def job(
|
|
39
48
|
try:
|
40
49
|
params_dict: dict[str, Any] = json.loads(params)
|
41
50
|
except json.JSONDecodeError as e:
|
42
|
-
raise ValueError(f"
|
51
|
+
raise ValueError(f"Params does not support format: {params!r}.") from e
|
52
|
+
|
53
|
+
try:
|
54
|
+
job_dict: dict[str, Any] = json.loads(job)
|
55
|
+
_job: Job = Job.model_validate(obj=job_dict)
|
56
|
+
except json.JSONDecodeError as e:
|
57
|
+
raise ValueError(f"Params does not support format: {params!r}.") from e
|
58
|
+
|
43
59
|
typer.echo(f"Job params: {params_dict}")
|
60
|
+
rs: Result = Result(
|
61
|
+
run_id=run_id,
|
62
|
+
parent_run_id=parent_run_id,
|
63
|
+
)
|
64
|
+
|
65
|
+
context: DictData = {}
|
66
|
+
try:
|
67
|
+
_job.set_outputs(
|
68
|
+
_job.execute(
|
69
|
+
params=params_dict,
|
70
|
+
run_id=rs.run_id,
|
71
|
+
parent_run_id=rs.parent_run_id,
|
72
|
+
).context,
|
73
|
+
to=context,
|
74
|
+
)
|
75
|
+
except JobError as err:
|
76
|
+
rs.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
|
44
77
|
|
45
78
|
|
46
79
|
@app.command()
|
@@ -48,19 +81,39 @@ def api(
|
|
48
81
|
host: Annotated[str, typer.Option(help="A host url.")] = "0.0.0.0",
|
49
82
|
port: Annotated[int, typer.Option(help="A port url.")] = 80,
|
50
83
|
debug: Annotated[bool, typer.Option(help="A debug mode flag")] = True,
|
84
|
+
workers: Annotated[int, typer.Option(help="A worker number")] = None,
|
85
|
+
reload: Annotated[bool, typer.Option(help="A reload flag")] = False,
|
51
86
|
):
|
52
87
|
"""
|
53
88
|
Provision API application from the FastAPI.
|
54
89
|
"""
|
90
|
+
from .api.log_conf import LOGGING_CONFIG
|
91
|
+
|
92
|
+
# LOGGING_CONFIG = {}
|
55
93
|
|
56
94
|
uvicorn.run(
|
57
95
|
fastapp,
|
58
96
|
host=host,
|
59
97
|
port=port,
|
60
98
|
log_config=uvicorn.config.LOGGING_CONFIG | LOGGING_CONFIG,
|
61
|
-
|
99
|
+
# NOTE: Logging level of uvicorn should be lowered case.
|
100
|
+
log_level=("debug" if debug else "info"),
|
101
|
+
workers=workers,
|
102
|
+
reload=reload,
|
62
103
|
)
|
63
104
|
|
64
105
|
|
106
|
+
@app.command()
|
107
|
+
def make(
|
108
|
+
name: Annotated[Path, typer.Argument()],
|
109
|
+
) -> None:
|
110
|
+
"""
|
111
|
+
Create Workflow YAML template.
|
112
|
+
|
113
|
+
:param name:
|
114
|
+
"""
|
115
|
+
typer.echo(f"Start create YAML template filename: {name.resolve()}")
|
116
|
+
|
117
|
+
|
65
118
|
if __name__ == "__main__":
|
66
119
|
app()
|
ddeutil/workflow/conf.py
CHANGED
@@ -109,9 +109,9 @@ class Config: # pragma: no cov
|
|
109
109
|
return env(
|
110
110
|
"LOG_FORMAT",
|
111
111
|
(
|
112
|
-
"%(asctime)s.%(msecs)03d (%(
|
112
|
+
"%(asctime)s.%(msecs)03d (%(process)-5d, "
|
113
113
|
"%(thread)-5d) [%(levelname)-7s] %(message)-120s "
|
114
|
-
"(%(filename)s:%(lineno)s)"
|
114
|
+
"(%(filename)s:%(lineno)s) (%(name)-10s)"
|
115
115
|
),
|
116
116
|
)
|
117
117
|
|
@@ -161,9 +161,13 @@ class Config: # pragma: no cov
|
|
161
161
|
class APIConfig:
|
162
162
|
"""API Config object."""
|
163
163
|
|
164
|
+
@property
|
165
|
+
def version(self) -> str:
|
166
|
+
return env("API_VERSION", "1")
|
167
|
+
|
164
168
|
@property
|
165
169
|
def prefix_path(self) -> str:
|
166
|
-
return env("API_PREFIX_PATH", "/api/
|
170
|
+
return env("API_PREFIX_PATH", f"/api/v{self.version}")
|
167
171
|
|
168
172
|
|
169
173
|
class BaseLoad(ABC): # pragma: no cov
|
ddeutil/workflow/errors.py
CHANGED
ddeutil/workflow/event.py
CHANGED
@@ -3,8 +3,9 @@
|
|
3
3
|
# Licensed under the MIT License. See LICENSE in the project root for
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
|
-
"""Event module
|
7
|
-
|
6
|
+
"""An Event module keep all triggerable object to the Workflow model. The simple
|
7
|
+
event trigger that use to run workflow is `Crontab` model.
|
8
|
+
Now, it has only `Crontab` and `CrontabYear` event models in this module because
|
8
9
|
I think it is the core event for workflow orchestration.
|
9
10
|
"""
|
10
11
|
from __future__ import annotations
|
@@ -95,7 +96,7 @@ class Crontab(BaseModel):
|
|
95
96
|
tz: Annotated[
|
96
97
|
TimeZoneName,
|
97
98
|
Field(
|
98
|
-
description="A timezone string value",
|
99
|
+
description="A timezone string value.",
|
99
100
|
alias="timezone",
|
100
101
|
),
|
101
102
|
] = "UTC"
|
ddeutil/workflow/logs.py
CHANGED
@@ -37,33 +37,34 @@ METADATA: str = "metadata.json"
|
|
37
37
|
|
38
38
|
|
39
39
|
@lru_cache
|
40
|
-
def
|
41
|
-
"""Return logger object with an input module name
|
40
|
+
def set_logging(name: str) -> logging.Logger:
|
41
|
+
"""Return logger object with an input module name that already implement the
|
42
|
+
custom handler and formatter from this package config.
|
42
43
|
|
43
44
|
:param name: (str) A module name that want to log.
|
45
|
+
|
46
|
+
:rtype: logging.Logger
|
44
47
|
"""
|
45
|
-
|
48
|
+
_logger = logging.getLogger(name)
|
46
49
|
|
47
50
|
# NOTE: Developers using this package can then disable all logging just for
|
48
51
|
# this package by;
|
49
52
|
#
|
50
53
|
# `logging.getLogger('ddeutil.workflow').propagate = False`
|
51
54
|
#
|
52
|
-
|
55
|
+
_logger.addHandler(logging.NullHandler())
|
53
56
|
|
54
57
|
formatter = logging.Formatter(
|
55
|
-
fmt=config.log_format,
|
56
|
-
datefmt=config.log_datetime_format,
|
58
|
+
fmt=config.log_format, datefmt=config.log_datetime_format
|
57
59
|
)
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
return lg
|
60
|
+
stream_handler = logging.StreamHandler()
|
61
|
+
stream_handler.setFormatter(formatter)
|
62
|
+
_logger.addHandler(stream_handler)
|
63
|
+
_logger.setLevel(logging.DEBUG if config.debug else logging.INFO)
|
64
|
+
return _logger
|
64
65
|
|
65
66
|
|
66
|
-
logger =
|
67
|
+
logger = logging.getLogger("ddeutil.workflow")
|
67
68
|
|
68
69
|
|
69
70
|
def get_dt_tznow() -> datetime: # pragma: no cov
|
@@ -689,6 +690,9 @@ class BaseAudit(BaseModel, ABC):
|
|
689
690
|
"""
|
690
691
|
if dynamic("enable_write_audit", extras=self.extras):
|
691
692
|
self.do_before()
|
693
|
+
|
694
|
+
# NOTE: Start setting log config in this line with cache.
|
695
|
+
set_logging("ddeutil.workflow")
|
692
696
|
return self
|
693
697
|
|
694
698
|
@classmethod
|
@@ -732,7 +736,7 @@ class BaseAudit(BaseModel, ABC):
|
|
732
736
|
@abstractmethod
|
733
737
|
def save(self, excluded: Optional[list[str]]) -> None: # pragma: no cov
|
734
738
|
"""Save this model logging to target logging store."""
|
735
|
-
raise NotImplementedError("Audit should implement
|
739
|
+
raise NotImplementedError("Audit should implement `save` method.")
|
736
740
|
|
737
741
|
|
738
742
|
class FileAudit(BaseAudit):
|
ddeutil/workflow/result.py
CHANGED
@@ -11,7 +11,7 @@ from __future__ import annotations
|
|
11
11
|
|
12
12
|
from dataclasses import field
|
13
13
|
from datetime import datetime
|
14
|
-
from enum import
|
14
|
+
from enum import Enum
|
15
15
|
from typing import Optional, Union
|
16
16
|
|
17
17
|
from pydantic import ConfigDict
|
@@ -36,16 +36,16 @@ from .logs import TraceModel, get_dt_tznow, get_trace
|
|
36
36
|
from .utils import default_gen_id, gen_id, get_dt_now
|
37
37
|
|
38
38
|
|
39
|
-
class Status(
|
39
|
+
class Status(str, Enum):
|
40
40
|
"""Status Int Enum object that use for tracking execution status to the
|
41
41
|
Result dataclass object.
|
42
42
|
"""
|
43
43
|
|
44
|
-
SUCCESS =
|
45
|
-
FAILED =
|
46
|
-
WAIT =
|
47
|
-
SKIP =
|
48
|
-
CANCEL =
|
44
|
+
SUCCESS = "SUCCESS"
|
45
|
+
FAILED = "FAILED"
|
46
|
+
WAIT = "WAIT"
|
47
|
+
SKIP = "SKIP"
|
48
|
+
CANCEL = "CANCEL"
|
49
49
|
|
50
50
|
@property
|
51
51
|
def emoji(self) -> str: # pragma: no cov
|
@@ -67,6 +67,9 @@ class Status(IntEnum):
|
|
67
67
|
def __str__(self) -> str:
|
68
68
|
return self.name
|
69
69
|
|
70
|
+
def is_result(self) -> bool:
|
71
|
+
return self in ResultStatuses
|
72
|
+
|
70
73
|
|
71
74
|
SUCCESS = Status.SUCCESS
|
72
75
|
FAILED = Status.FAILED
|
@@ -74,6 +77,8 @@ WAIT = Status.WAIT
|
|
74
77
|
SKIP = Status.SKIP
|
75
78
|
CANCEL = Status.CANCEL
|
76
79
|
|
80
|
+
ResultStatuses: list[Status] = [SUCCESS, FAILED, CANCEL, SKIP]
|
81
|
+
|
77
82
|
|
78
83
|
def validate_statuses(statuses: list[Status]) -> Status:
|
79
84
|
"""Validate the final status from list of Status object.
|
ddeutil/workflow/stages.py
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
# Licensed under the MIT License. See LICENSE in the project root for
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
|
-
"""Stages module include all stage model that implemented to be the minimum execution
|
6
|
+
r"""Stages module include all stage model that implemented to be the minimum execution
|
7
7
|
layer of this workflow core engine. The stage handle the minimize task that run
|
8
8
|
in a thread (same thread at its job owner) that mean it is the lowest executor that
|
9
9
|
you can track logs.
|
@@ -15,17 +15,39 @@ have a lot of use-case, and it should does not worry about it error output.
|
|
15
15
|
So, I will create `handler_execute` for any exception class that raise from
|
16
16
|
the stage execution method.
|
17
17
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
18
|
+
Handler --> Ok --> Result
|
19
|
+
|-status: SUCCESS
|
20
|
+
╰-context:
|
21
|
+
╰-outputs: ...
|
22
|
+
|
23
|
+
--> Ok --> Result
|
24
|
+
╰-status: CANCEL
|
23
25
|
|
24
|
-
-->
|
26
|
+
--> Ok --> Result
|
27
|
+
╰-status: SKIP
|
28
|
+
|
29
|
+
--> Ok --> Result
|
30
|
+
|-status: FAILED
|
31
|
+
╰-errors:
|
32
|
+
|-name: ...
|
33
|
+
╰-message: ...
|
25
34
|
|
26
35
|
On the context I/O that pass to a stage object at execute process. The
|
27
36
|
execute method receives a `params={"params": {...}}` value for passing template
|
28
37
|
searching.
|
38
|
+
|
39
|
+
All stages model inherit from `BaseStage` or `AsyncBaseStage` models that has the
|
40
|
+
base fields:
|
41
|
+
|
42
|
+
| field | alias | data type | default | description |
|
43
|
+
|-----------|-------|-------------|:--------:|-----------------------------------------------------------------------|
|
44
|
+
| id | | str \| None | `None` | A stage ID that use to keep execution output or getting by job owner. |
|
45
|
+
| name | | str | | A stage name that want to log when start execution. |
|
46
|
+
| condition | if | str \| None | `None` | A stage condition statement to allow stage executable. |
|
47
|
+
| extras | | dict | `dict()` | An extra parameter that override core config values. |
|
48
|
+
|
49
|
+
It has a special base class is `BaseRetryStage` that inherit from `AsyncBaseStage`
|
50
|
+
that use to handle retry execution when it got any error with `retry` field.
|
29
51
|
"""
|
30
52
|
from __future__ import annotations
|
31
53
|
|
@@ -62,10 +84,9 @@ from pydantic import BaseModel, Field, ValidationError
|
|
62
84
|
from pydantic.functional_validators import field_validator, model_validator
|
63
85
|
from typing_extensions import Self
|
64
86
|
|
65
|
-
from . import StageCancelError, StageRetryError
|
66
87
|
from .__types import DictData, DictStr, StrOrInt, StrOrNone, TupleStr
|
67
88
|
from .conf import dynamic, pass_env
|
68
|
-
from .errors import StageError, StageSkipError, to_dict
|
89
|
+
from .errors import StageCancelError, StageError, StageSkipError, to_dict
|
69
90
|
from .result import (
|
70
91
|
CANCEL,
|
71
92
|
FAILED,
|
@@ -252,16 +273,20 @@ class BaseStage(BaseModel, ABC):
|
|
252
273
|
f"[STAGE]: Handler {to_train(self.__class__.__name__)}: "
|
253
274
|
f"{self.name!r}."
|
254
275
|
)
|
276
|
+
|
277
|
+
# NOTE: Show the description of this stage before execution.
|
255
278
|
if self.desc:
|
256
279
|
result.trace.debug(f"[STAGE]: Description:||{self.desc}||")
|
257
280
|
|
281
|
+
# VALIDATE: Checking stage condition before execution.
|
258
282
|
if self.is_skipped(params):
|
259
283
|
raise StageSkipError(
|
260
284
|
f"Skip because condition {self.condition} was valid."
|
261
285
|
)
|
286
|
+
|
262
287
|
# NOTE: Start call wrapped execution method that will use custom
|
263
288
|
# execution before the real execution from inherit stage model.
|
264
|
-
result_caught: Result = self.
|
289
|
+
result_caught: Result = self._execute(
|
265
290
|
params, result=result, event=event
|
266
291
|
)
|
267
292
|
if result_caught.status == WAIT:
|
@@ -296,7 +321,7 @@ class BaseStage(BaseModel, ABC):
|
|
296
321
|
)
|
297
322
|
return result.catch(status=FAILED, context={"errors": to_dict(e)})
|
298
323
|
|
299
|
-
def
|
324
|
+
def _execute(
|
300
325
|
self, params: DictData, result: Result, event: Optional[Event]
|
301
326
|
) -> Result:
|
302
327
|
"""Wrapped the execute method before returning to handler execution.
|
@@ -447,6 +472,13 @@ class BaseStage(BaseModel, ABC):
|
|
447
472
|
"""
|
448
473
|
return False
|
449
474
|
|
475
|
+
def docs(self) -> str: # pragma: no cov
|
476
|
+
"""Return generated document that will be the interface of this stage.
|
477
|
+
|
478
|
+
:rtype: str
|
479
|
+
"""
|
480
|
+
return self.desc
|
481
|
+
|
450
482
|
|
451
483
|
class BaseAsyncStage(BaseStage, ABC):
|
452
484
|
"""Base Async Stage model to make any stage model allow async execution for
|
@@ -514,11 +546,14 @@ class BaseAsyncStage(BaseStage, ABC):
|
|
514
546
|
f"[STAGE]: Handler {to_train(self.__class__.__name__)}: "
|
515
547
|
f"{self.name!r}."
|
516
548
|
)
|
549
|
+
|
550
|
+
# NOTE: Show the description of this stage before execution.
|
517
551
|
if self.desc:
|
518
552
|
await result.trace.adebug(
|
519
553
|
f"[STAGE]: Description:||{self.desc}||"
|
520
554
|
)
|
521
555
|
|
556
|
+
# VALIDATE: Checking stage condition before execution.
|
522
557
|
if self.is_skipped(params=params):
|
523
558
|
raise StageSkipError(
|
524
559
|
f"Skip because condition {self.condition} was valid."
|
@@ -526,7 +561,7 @@ class BaseAsyncStage(BaseStage, ABC):
|
|
526
561
|
|
527
562
|
# NOTE: Start call wrapped execution method that will use custom
|
528
563
|
# execution before the real execution from inherit stage model.
|
529
|
-
result_caught: Result = await self.
|
564
|
+
result_caught: Result = await self._axecute(
|
530
565
|
params, result=result, event=event
|
531
566
|
)
|
532
567
|
if result_caught.status == WAIT:
|
@@ -561,7 +596,7 @@ class BaseAsyncStage(BaseStage, ABC):
|
|
561
596
|
)
|
562
597
|
return result.catch(status=FAILED, context={"errors": to_dict(e)})
|
563
598
|
|
564
|
-
async def
|
599
|
+
async def _axecute(
|
565
600
|
self, params: DictData, result: Result, event: Optional[Event]
|
566
601
|
) -> Result:
|
567
602
|
"""Wrapped the axecute method before returning to handler axecute.
|
@@ -588,10 +623,10 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
588
623
|
default=0,
|
589
624
|
ge=0,
|
590
625
|
lt=20,
|
591
|
-
description="
|
626
|
+
description="A retry number if stage execution get the error.",
|
592
627
|
)
|
593
628
|
|
594
|
-
def
|
629
|
+
def _execute(
|
595
630
|
self,
|
596
631
|
params: DictData,
|
597
632
|
result: Result,
|
@@ -610,15 +645,50 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
610
645
|
:rtype: Result
|
611
646
|
"""
|
612
647
|
current_retry: int = 0
|
613
|
-
|
648
|
+
exception: Exception
|
649
|
+
|
650
|
+
# NOTE: First execution for not pass to retry step if it passes.
|
651
|
+
try:
|
652
|
+
result.catch(status=WAIT)
|
653
|
+
return self.execute(
|
654
|
+
params | {"retry": current_retry},
|
655
|
+
result=result,
|
656
|
+
event=event,
|
657
|
+
)
|
658
|
+
except Exception as e:
|
659
|
+
current_retry += 1
|
660
|
+
exception = e
|
661
|
+
|
662
|
+
if self.retry == 0:
|
663
|
+
raise exception
|
664
|
+
|
665
|
+
result.trace.warning(
|
666
|
+
f"[STAGE]: Retry count: {current_retry} ... "
|
667
|
+
f"( {exception.__class__.__name__} )"
|
668
|
+
)
|
669
|
+
|
670
|
+
while current_retry < (self.retry + 1):
|
614
671
|
try:
|
615
672
|
result.catch(status=WAIT, context={"retry": current_retry})
|
616
|
-
return self.execute(
|
617
|
-
|
673
|
+
return self.execute(
|
674
|
+
params | {"retry": current_retry},
|
675
|
+
result=result,
|
676
|
+
event=event,
|
677
|
+
)
|
678
|
+
except Exception as e:
|
618
679
|
current_retry += 1
|
619
|
-
|
680
|
+
result.trace.warning(
|
681
|
+
f"[STAGE]: Retry count: {current_retry} ... "
|
682
|
+
f"( {e.__class__.__name__} )"
|
683
|
+
)
|
684
|
+
exception = e
|
685
|
+
|
686
|
+
result.trace.error(
|
687
|
+
f"[STAGE]: Reach the maximum of retry number: {self.retry}."
|
688
|
+
)
|
689
|
+
raise exception
|
620
690
|
|
621
|
-
async def
|
691
|
+
async def _axecute(
|
622
692
|
self,
|
623
693
|
params: DictData,
|
624
694
|
result: Result,
|
@@ -637,13 +707,48 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
637
707
|
:rtype: Result
|
638
708
|
"""
|
639
709
|
current_retry: int = 0
|
640
|
-
|
710
|
+
exception: Exception
|
711
|
+
|
712
|
+
# NOTE: First execution for not pass to retry step if it passes.
|
713
|
+
try:
|
714
|
+
result.catch(status=WAIT)
|
715
|
+
return await self.axecute(
|
716
|
+
params | {"retry": current_retry},
|
717
|
+
result=result,
|
718
|
+
event=event,
|
719
|
+
)
|
720
|
+
except Exception as e:
|
721
|
+
current_retry += 1
|
722
|
+
exception = e
|
723
|
+
|
724
|
+
if self.retry == 0:
|
725
|
+
raise exception
|
726
|
+
|
727
|
+
await result.trace.awarning(
|
728
|
+
f"[STAGE]: Retry count: {current_retry} ... "
|
729
|
+
f"( {exception.__class__.__name__} )"
|
730
|
+
)
|
731
|
+
|
732
|
+
while current_retry < (self.retry + 1):
|
641
733
|
try:
|
642
734
|
result.catch(status=WAIT, context={"retry": current_retry})
|
643
|
-
return await self.axecute(
|
644
|
-
|
735
|
+
return await self.axecute(
|
736
|
+
params | {"retry": current_retry},
|
737
|
+
result=result,
|
738
|
+
event=event,
|
739
|
+
)
|
740
|
+
except Exception as e:
|
645
741
|
current_retry += 1
|
646
|
-
|
742
|
+
await result.trace.awarning(
|
743
|
+
f"[STAGE]: Retry count: {current_retry} ... "
|
744
|
+
f"( {e.__class__.__name__} )"
|
745
|
+
)
|
746
|
+
exception = e
|
747
|
+
|
748
|
+
await result.trace.aerror(
|
749
|
+
f"[STAGE]: Reach the maximum of retry number: {self.retry}."
|
750
|
+
)
|
751
|
+
raise exception
|
647
752
|
|
648
753
|
|
649
754
|
class EmptyStage(BaseAsyncStage):
|
@@ -765,7 +870,7 @@ class EmptyStage(BaseAsyncStage):
|
|
765
870
|
return result.catch(status=SUCCESS)
|
766
871
|
|
767
872
|
|
768
|
-
class BashStage(
|
873
|
+
class BashStage(BaseRetryStage):
|
769
874
|
"""Bash stage executor that execute bash script on the current OS.
|
770
875
|
If your current OS is Windows, it will run on the bash from the current WSL.
|
771
876
|
It will use `bash` for Windows OS and use `sh` for Linux OS.
|
@@ -911,9 +1016,8 @@ class BashStage(BaseAsyncStage):
|
|
911
1016
|
)
|
912
1017
|
if rs.returncode > 0:
|
913
1018
|
e: str = rs.stderr.removesuffix("\n")
|
914
|
-
|
915
|
-
|
916
|
-
)
|
1019
|
+
e_bash: str = bash.replace("\n", "\n\t")
|
1020
|
+
raise StageError(f"Subprocess: {e}\n\t```bash\n\t{e_bash}\n\t```")
|
917
1021
|
return result.catch(
|
918
1022
|
status=SUCCESS,
|
919
1023
|
context={
|
@@ -964,9 +1068,8 @@ class BashStage(BaseAsyncStage):
|
|
964
1068
|
|
965
1069
|
if rs.returncode > 0:
|
966
1070
|
e: str = rs.stderr.removesuffix("\n")
|
967
|
-
|
968
|
-
|
969
|
-
)
|
1071
|
+
e_bash: str = bash.replace("\n", "\n\t")
|
1072
|
+
raise StageError(f"Subprocess: {e}\n\t```bash\n\t{e_bash}\n\t```")
|
970
1073
|
return result.catch(
|
971
1074
|
status=SUCCESS,
|
972
1075
|
context={
|
@@ -977,7 +1080,7 @@ class BashStage(BaseAsyncStage):
|
|
977
1080
|
)
|
978
1081
|
|
979
1082
|
|
980
|
-
class PyStage(
|
1083
|
+
class PyStage(BaseRetryStage):
|
981
1084
|
"""Python stage that running the Python statement with the current globals
|
982
1085
|
and passing an input additional variables via `exec` built-in function.
|
983
1086
|
|
@@ -1164,7 +1267,7 @@ class PyStage(BaseAsyncStage):
|
|
1164
1267
|
)
|
1165
1268
|
|
1166
1269
|
|
1167
|
-
class CallStage(
|
1270
|
+
class CallStage(BaseRetryStage):
|
1168
1271
|
"""Call stage executor that call the Python function from registry with tag
|
1169
1272
|
decorator function in `reusables` module and run it with input arguments.
|
1170
1273
|
|
@@ -1175,7 +1278,7 @@ class CallStage(BaseAsyncStage):
|
|
1175
1278
|
function complexly that you can for your objective to invoked by this stage
|
1176
1279
|
object.
|
1177
1280
|
|
1178
|
-
This stage is the most
|
1281
|
+
This stage is the most powerful stage of this package for run every
|
1179
1282
|
use-case by a custom requirement that you want by creating the Python
|
1180
1283
|
function and adding it to the caller registry value by importer syntax like
|
1181
1284
|
`module.caller.registry` not path style like `module/caller/registry`.
|
@@ -1433,7 +1536,7 @@ class CallStage(BaseAsyncStage):
|
|
1433
1536
|
return args
|
1434
1537
|
|
1435
1538
|
|
1436
|
-
class BaseNestedStage(
|
1539
|
+
class BaseNestedStage(BaseRetryStage, ABC):
|
1437
1540
|
"""Base Nested Stage model. This model is use for checking the child stage
|
1438
1541
|
is the nested stage or not.
|
1439
1542
|
"""
|
@@ -1467,6 +1570,17 @@ class BaseNestedStage(BaseStage, ABC):
|
|
1467
1570
|
else:
|
1468
1571
|
context["errors"] = error.to_dict(with_refs=True)
|
1469
1572
|
|
1573
|
+
async def axecute(
|
1574
|
+
self,
|
1575
|
+
params: DictData,
|
1576
|
+
*,
|
1577
|
+
result: Optional[Result] = None,
|
1578
|
+
event: Optional[Event] = None,
|
1579
|
+
) -> Result:
|
1580
|
+
raise NotImplementedError(
|
1581
|
+
"The nested-stage does not implement the `axecute` method yet."
|
1582
|
+
)
|
1583
|
+
|
1470
1584
|
|
1471
1585
|
class TriggerStage(BaseNestedStage):
|
1472
1586
|
"""Trigger workflow executor stage that run an input trigger Workflow
|
ddeutil/workflow/utils.py
CHANGED
@@ -6,15 +6,13 @@
|
|
6
6
|
"""Utility function model."""
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
|
-
import asyncio
|
10
9
|
import stat
|
11
10
|
import time
|
12
11
|
from collections.abc import Iterator
|
13
12
|
from datetime import date, datetime, timedelta
|
14
|
-
from functools import wraps
|
15
13
|
from hashlib import md5
|
16
14
|
from inspect import isfunction
|
17
|
-
from itertools import
|
15
|
+
from itertools import product
|
18
16
|
from pathlib import Path
|
19
17
|
from random import randrange
|
20
18
|
from typing import Any, Final, Optional, TypeVar, Union, overload
|
@@ -258,34 +256,6 @@ def cross_product(matrix: Matrix) -> Iterator[DictData]:
|
|
258
256
|
)
|
259
257
|
|
260
258
|
|
261
|
-
def batch(iterable: Union[Iterator[Any], range], n: int) -> Iterator[Any]:
|
262
|
-
"""Batch data into iterators of length n. The last batch may be shorter.
|
263
|
-
|
264
|
-
Example:
|
265
|
-
>>> for b in batch(iter('ABCDEFG'), 3):
|
266
|
-
... print(list(b))
|
267
|
-
['A', 'B', 'C']
|
268
|
-
['D', 'E', 'F']
|
269
|
-
['G']
|
270
|
-
|
271
|
-
:param iterable:
|
272
|
-
:param n: (int) A number of returning batch size.
|
273
|
-
|
274
|
-
:rtype: Iterator[Any]
|
275
|
-
"""
|
276
|
-
if n < 1:
|
277
|
-
raise ValueError("n must be at least one")
|
278
|
-
|
279
|
-
it: Iterator[Any] = iter(iterable)
|
280
|
-
while True:
|
281
|
-
chunk_it = islice(it, n)
|
282
|
-
try:
|
283
|
-
first_el = next(chunk_it)
|
284
|
-
except StopIteration:
|
285
|
-
return
|
286
|
-
yield chain((first_el,), chunk_it)
|
287
|
-
|
288
|
-
|
289
259
|
def cut_id(run_id: str, *, num: int = 6) -> str:
|
290
260
|
"""Cutting running ID with length.
|
291
261
|
|
@@ -325,24 +295,3 @@ def dump_all(
|
|
325
295
|
elif isinstance(value, BaseModel):
|
326
296
|
return value.model_dump(by_alias=by_alias)
|
327
297
|
return value
|
328
|
-
|
329
|
-
|
330
|
-
def awaitable(func):
|
331
|
-
"""Dynamic function to async or not depend on the called statement."""
|
332
|
-
|
333
|
-
@wraps(func)
|
334
|
-
async def async_wrapper(*args, **kwargs):
|
335
|
-
return func(*args, **kwargs)
|
336
|
-
|
337
|
-
@wraps(func)
|
338
|
-
def sync_wrapper(*args, **kwargs):
|
339
|
-
return func(*args, **kwargs)
|
340
|
-
|
341
|
-
def dispatch(*args, **kwargs):
|
342
|
-
try:
|
343
|
-
asyncio.get_running_loop()
|
344
|
-
return async_wrapper(*args, **kwargs)
|
345
|
-
except RuntimeError:
|
346
|
-
return sync_wrapper(*args, **kwargs)
|
347
|
-
|
348
|
-
return dispatch
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: ddeutil-workflow
|
3
|
-
Version: 0.0.
|
4
|
-
Summary: Lightweight workflow orchestration
|
3
|
+
Version: 0.0.68
|
4
|
+
Summary: Lightweight workflow orchestration with YAML template
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
6
6
|
License: MIT
|
7
7
|
Project-URL: Homepage, https://github.com/ddeutils/ddeutil-workflow/
|
@@ -24,10 +24,10 @@ Description-Content-Type: text/markdown
|
|
24
24
|
License-File: LICENSE
|
25
25
|
Requires-Dist: ddeutil[checksum]>=0.4.8
|
26
26
|
Requires-Dist: ddeutil-io[toml,yaml]>=0.2.14
|
27
|
-
Requires-Dist: pydantic==2.11.
|
27
|
+
Requires-Dist: pydantic==2.11.5
|
28
28
|
Requires-Dist: pydantic-extra-types==2.10.4
|
29
29
|
Requires-Dist: python-dotenv==1.1.0
|
30
|
-
Requires-Dist: typer
|
30
|
+
Requires-Dist: typer>=0.16.0
|
31
31
|
Provides-Extra: all
|
32
32
|
Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "all"
|
33
33
|
Requires-Dist: uvicorn; extra == "all"
|
@@ -35,18 +35,9 @@ Requires-Dist: httpx; extra == "all"
|
|
35
35
|
Requires-Dist: ujson; extra == "all"
|
36
36
|
Requires-Dist: aiofiles; extra == "all"
|
37
37
|
Requires-Dist: aiohttp; extra == "all"
|
38
|
-
|
39
|
-
Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "api"
|
40
|
-
Requires-Dist: uvicorn; extra == "api"
|
41
|
-
Requires-Dist: httpx; extra == "api"
|
42
|
-
Requires-Dist: ujson; extra == "api"
|
43
|
-
Provides-Extra: async
|
44
|
-
Requires-Dist: aiofiles; extra == "async"
|
45
|
-
Requires-Dist: aiohttp; extra == "async"
|
38
|
+
Requires-Dist: requests==2.32.3; extra == "all"
|
46
39
|
Provides-Extra: docker
|
47
40
|
Requires-Dist: docker==7.1.0; extra == "docker"
|
48
|
-
Provides-Extra: self-hosted
|
49
|
-
Requires-Dist: requests==2.32.3; extra == "self-hosted"
|
50
41
|
Dynamic: license-file
|
51
42
|
|
52
43
|
# Workflow Orchestration
|
@@ -142,10 +133,10 @@ the base deps.
|
|
142
133
|
If you want to install this package with application add-ons, you should add
|
143
134
|
`app` in installation;
|
144
135
|
|
145
|
-
| Use-case | Install Optional
|
146
|
-
|
147
|
-
| Python | `ddeutil-workflow`
|
148
|
-
| FastAPI Server | `ddeutil-workflow[
|
136
|
+
| Use-case | Install Optional | Support |
|
137
|
+
|----------------|-------------------------|:-------:|
|
138
|
+
| Python | `ddeutil-workflow` | ✅ |
|
139
|
+
| FastAPI Server | `ddeutil-workflow[all]` | ✅ |
|
149
140
|
|
150
141
|
## 🎯 Usage
|
151
142
|
|
@@ -300,40 +291,27 @@ it will use default value and do not raise any error to you.
|
|
300
291
|
## :rocket: Deployment
|
301
292
|
|
302
293
|
This package able to run as an application service for receive manual trigger
|
303
|
-
from any node via RestAPI
|
304
|
-
like crontab job but via Python API or FastAPI app.
|
294
|
+
from any node via RestAPI with the FastAPI package.
|
305
295
|
|
306
296
|
### API Server
|
307
297
|
|
308
298
|
This server use FastAPI package to be the base application.
|
309
299
|
|
310
300
|
```shell
|
311
|
-
(.venv) $
|
312
|
-
--host 127.0.0.1 \
|
313
|
-
--port 80 \
|
314
|
-
--no-access-log
|
301
|
+
(.venv) $ workflow-cli api --host 127.0.0.1 --port 80
|
315
302
|
```
|
316
303
|
|
317
304
|
> [!NOTE]
|
318
305
|
> If this package already deploy, it is able to use multiprocess;
|
319
|
-
>
|
320
|
-
|
321
|
-
### Local Schedule
|
322
|
-
|
323
|
-
> [!WARNING]
|
324
|
-
> This CLI does not implement yet.
|
325
|
-
|
326
|
-
```shell
|
327
|
-
(.venv) $ ddeutil-workflow schedule
|
328
|
-
```
|
306
|
+
> `$ workflow-cli api --host 127.0.0.1 --port 80 --workers 4`
|
329
307
|
|
330
308
|
### Docker Container
|
331
309
|
|
332
310
|
Build a Docker container from this package.
|
333
311
|
|
334
312
|
```shell
|
335
|
-
$ docker
|
336
|
-
$ docker run
|
313
|
+
$ docker pull ghcr.io/ddeutils/ddeutil-workflow:latest
|
314
|
+
$ docker run --rm ghcr.io/ddeutils/ddeutil-workflow:latest ddeutil-worker
|
337
315
|
```
|
338
316
|
|
339
317
|
## :speech_balloon: Contribute
|
@@ -0,0 +1,29 @@
|
|
1
|
+
ddeutil/workflow/__about__.py,sha256=sX7SVH5YIxA4EqrZ30bmKtzuaxvEoxEAs2Bc7wFHXgY,28
|
2
|
+
ddeutil/workflow/__cron.py,sha256=BOKQcreiex0SAigrK1gnLxpvOeF3aca_rQwyz9Kfve4,28751
|
3
|
+
ddeutil/workflow/__init__.py,sha256=JfFZlPRDgR2J0rb0SRejt1OSrOrD3GGv9Um14z8MMfs,901
|
4
|
+
ddeutil/workflow/__main__.py,sha256=Qd-f8z2Q2vpiEP2x6PBFsJrpACWDVxFKQk820MhFmHo,59
|
5
|
+
ddeutil/workflow/__types.py,sha256=uNfoRbVmNK5O37UUMVnqcmoghD9oMS1q9fXC0APnjSI,4584
|
6
|
+
ddeutil/workflow/cli.py,sha256=6cxKS3U9oyGTbCI--4x3aiv536PJ5k2KyLRjAQHE6qA,3136
|
7
|
+
ddeutil/workflow/conf.py,sha256=p5QLqRo67nivoHDEj7Rs5P5-qdBvtDa3c9Zaylug6p0,14768
|
8
|
+
ddeutil/workflow/errors.py,sha256=4DaKnyUm8RrUyQA5qakgW0ycSQLO7j-owyoh79LWQ5c,2893
|
9
|
+
ddeutil/workflow/event.py,sha256=PMGXu0wH9ld-nq670QsGcpHBCNr0Yu5Y3hA_EAPIKyo,11104
|
10
|
+
ddeutil/workflow/job.py,sha256=qcbKSOa39256nfJHL0vKJsHrelcRujX5KET2IEGS8dw,38995
|
11
|
+
ddeutil/workflow/logs.py,sha256=4jufDRXTgtU9GZ1fVXL37tnihupD69YFSkD-mNQTr_g,31657
|
12
|
+
ddeutil/workflow/params.py,sha256=Pco3DyjptC5Jkx53dhLL9xlIQdJvNAZs4FLzMUfXpbQ,12402
|
13
|
+
ddeutil/workflow/result.py,sha256=GU84psZFiJ4LRf_HXgz-R98YN4lOUkER0VR7x9DDdOU,7922
|
14
|
+
ddeutil/workflow/reusables.py,sha256=jPrOCbxagqRvRFGXJzIyDa1wKV5AZ4crZyJ10cldQP0,21620
|
15
|
+
ddeutil/workflow/stages.py,sha256=kzMEMRTEuG52EOw51zyVO6LE-oiiqTIRUCk_OMcWZTM,106506
|
16
|
+
ddeutil/workflow/utils.py,sha256=oKrhB-HOogeaO9RGXbe2vAs30A3rMMQxUd2B5pOw8zg,9131
|
17
|
+
ddeutil/workflow/workflow.py,sha256=AcSGqsH1N4LqWhYIcCPy9CoV_AGlXUrBgjpl-gniv6g,28267
|
18
|
+
ddeutil/workflow/api/__init__.py,sha256=W3fe6_NLHSUzr4Tsu79w3pmvrYjpLeP3zBk4mtpPyqg,2843
|
19
|
+
ddeutil/workflow/api/log_conf.py,sha256=P1_a5kjB0dWjSyJvmeKUU8KZdaiNd3UELoL42SiKmzU,2269
|
20
|
+
ddeutil/workflow/api/routes/__init__.py,sha256=JRaJZB0D6mgR17MbZo8yLtdYDtD62AA8MdKlFqhG84M,420
|
21
|
+
ddeutil/workflow/api/routes/job.py,sha256=x809G5gCbJS257txj9eLLTbCbFK8ercXWzPDLuv5gEM,2953
|
22
|
+
ddeutil/workflow/api/routes/logs.py,sha256=ElfXNJmwpeR_nE99RcCOUm2BRfQhly6RE9kZ7xVC530,5284
|
23
|
+
ddeutil/workflow/api/routes/workflows.py,sha256=08p2r7xoKrW1tUMKkGCaffBckdV1VvaF3OKCjFygJdE,4392
|
24
|
+
ddeutil_workflow-0.0.68.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
25
|
+
ddeutil_workflow-0.0.68.dist-info/METADATA,sha256=o3fWmIKifoP9HHQEKiELp10WI2XEUvfLCn5hFw4Ol1M,16072
|
26
|
+
ddeutil_workflow-0.0.68.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
27
|
+
ddeutil_workflow-0.0.68.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
|
28
|
+
ddeutil_workflow-0.0.68.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
29
|
+
ddeutil_workflow-0.0.68.dist-info/RECORD,,
|
@@ -1,29 +0,0 @@
|
|
1
|
-
ddeutil/workflow/__about__.py,sha256=nkpvuhIACvuey4B8paloyuDA6U0uRDZqZ83SUGdlIt4,28
|
2
|
-
ddeutil/workflow/__cron.py,sha256=BOKQcreiex0SAigrK1gnLxpvOeF3aca_rQwyz9Kfve4,28751
|
3
|
-
ddeutil/workflow/__init__.py,sha256=JfFZlPRDgR2J0rb0SRejt1OSrOrD3GGv9Um14z8MMfs,901
|
4
|
-
ddeutil/workflow/__main__.py,sha256=Qd-f8z2Q2vpiEP2x6PBFsJrpACWDVxFKQk820MhFmHo,59
|
5
|
-
ddeutil/workflow/__types.py,sha256=uNfoRbVmNK5O37UUMVnqcmoghD9oMS1q9fXC0APnjSI,4584
|
6
|
-
ddeutil/workflow/cli.py,sha256=__ydzTs_e9C1tHqhnyp__0iKelZJRUwZn8VsyVYw5yo,1458
|
7
|
-
ddeutil/workflow/conf.py,sha256=w1WDWZDCvRVDSz2HnJxeqySzpYWSubJZjTVjXO9imK0,14669
|
8
|
-
ddeutil/workflow/errors.py,sha256=vh_AbLegPbb61TvfwHmjSz0t2SH09RQuGHCKmoyNjn0,2934
|
9
|
-
ddeutil/workflow/event.py,sha256=S2eJAZZx_V5TuQ0l417hFVCtjWXnfNPZBgSCICzxQ48,11041
|
10
|
-
ddeutil/workflow/job.py,sha256=qcbKSOa39256nfJHL0vKJsHrelcRujX5KET2IEGS8dw,38995
|
11
|
-
ddeutil/workflow/logs.py,sha256=4rL8TsRJsYVqyPfLjFW5bSoWtRwUgwmaRONu7nnVxQ8,31374
|
12
|
-
ddeutil/workflow/params.py,sha256=Pco3DyjptC5Jkx53dhLL9xlIQdJvNAZs4FLzMUfXpbQ,12402
|
13
|
-
ddeutil/workflow/result.py,sha256=CzSjK9EQtSS0a1z8b6oBW4gg8t0uVbZ_Ppx4Ckvcork,7786
|
14
|
-
ddeutil/workflow/reusables.py,sha256=jPrOCbxagqRvRFGXJzIyDa1wKV5AZ4crZyJ10cldQP0,21620
|
15
|
-
ddeutil/workflow/stages.py,sha256=s6Jc4A6ApXT40JMrnVf7n7yHG_mmjfHJIOU_niMW5hI,102555
|
16
|
-
ddeutil/workflow/utils.py,sha256=slhBbsBNl0yaSk9EOiCK6UL-o7smgHVsLT7svRqAWXU,10436
|
17
|
-
ddeutil/workflow/workflow.py,sha256=AcSGqsH1N4LqWhYIcCPy9CoV_AGlXUrBgjpl-gniv6g,28267
|
18
|
-
ddeutil/workflow/api/__init__.py,sha256=0UIilYwW29RL6HrCRHACSWvnATJVLSJzXiCMny0bHQk,2627
|
19
|
-
ddeutil/workflow/api/logs.py,sha256=NMTnOnsBrDB5129329xF2myLdrb-z9k1MQrmrP7qXJw,1818
|
20
|
-
ddeutil/workflow/api/routes/__init__.py,sha256=jC1pM7q4_eo45IyO3hQbbe6RnL9B8ibRq_K6aCMP6Ag,434
|
21
|
-
ddeutil/workflow/api/routes/job.py,sha256=32TkNm7QY9gt6fxIqEPjDqPgc8XqDiMPjUb7disSrCw,2143
|
22
|
-
ddeutil/workflow/api/routes/logs.py,sha256=QJH8IF102897WLfCJ29-1g15wl29M9Yq6omroZfbahs,5305
|
23
|
-
ddeutil/workflow/api/routes/workflows.py,sha256=Gmg3e-K5rfi95pbRtWI_aIr5C089sIde_vefZVvh3U0,4420
|
24
|
-
ddeutil_workflow-0.0.66.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
25
|
-
ddeutil_workflow-0.0.66.dist-info/METADATA,sha256=LhCFhlwk3N5wbnylmi7zi3sAJqwxkRVlrRThlNB4Cj0,16676
|
26
|
-
ddeutil_workflow-0.0.66.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
|
27
|
-
ddeutil_workflow-0.0.66.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
|
28
|
-
ddeutil_workflow-0.0.66.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
29
|
-
ddeutil_workflow-0.0.66.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|