ddeutil-workflow 0.0.80__py3-none-any.whl → 0.0.82__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +2 -1
- ddeutil/workflow/__init__.py +19 -6
- ddeutil/workflow/__main__.py +280 -1
- ddeutil/workflow/api/routes/job.py +2 -2
- ddeutil/workflow/api/routes/logs.py +8 -61
- ddeutil/workflow/audits.py +46 -17
- ddeutil/workflow/conf.py +64 -44
- ddeutil/workflow/errors.py +12 -2
- ddeutil/workflow/job.py +70 -16
- ddeutil/workflow/result.py +33 -11
- ddeutil/workflow/reusables.py +16 -17
- ddeutil/workflow/stages.py +172 -134
- ddeutil/workflow/traces.py +64 -24
- ddeutil/workflow/utils.py +7 -15
- ddeutil/workflow/workflow.py +73 -84
- {ddeutil_workflow-0.0.80.dist-info → ddeutil_workflow-0.0.82.dist-info}/METADATA +1 -1
- ddeutil_workflow-0.0.82.dist-info/RECORD +35 -0
- ddeutil/workflow/cli.py +0 -274
- ddeutil_workflow-0.0.80.dist-info/RECORD +0 -36
- {ddeutil_workflow-0.0.80.dist-info → ddeutil_workflow-0.0.82.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.80.dist-info → ddeutil_workflow-0.0.82.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.80.dist-info → ddeutil_workflow-0.0.82.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.80.dist-info → ddeutil_workflow-0.0.82.dist-info}/top_level.txt +0 -0
ddeutil/workflow/cli.py
DELETED
@@ -1,274 +0,0 @@
|
|
1
|
-
# ------------------------------------------------------------------------------
|
2
|
-
# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
|
3
|
-
# Licensed under the MIT License. See LICENSE in the project root for
|
4
|
-
# license information.
|
5
|
-
# ------------------------------------------------------------------------------
|
6
|
-
from __future__ import annotations
|
7
|
-
|
8
|
-
import json
|
9
|
-
from pathlib import Path
|
10
|
-
from platform import python_version
|
11
|
-
from textwrap import dedent
|
12
|
-
from typing import Annotated, Any, Literal, Optional, Union
|
13
|
-
|
14
|
-
import typer
|
15
|
-
from pydantic import Field, TypeAdapter
|
16
|
-
|
17
|
-
from .__about__ import __version__
|
18
|
-
from .__types import DictData
|
19
|
-
from .conf import config
|
20
|
-
from .errors import JobError
|
21
|
-
from .job import Job
|
22
|
-
from .params import Param
|
23
|
-
from .workflow import Workflow
|
24
|
-
|
25
|
-
app = typer.Typer(pretty_exceptions_enable=True)
|
26
|
-
|
27
|
-
|
28
|
-
@app.callback()
|
29
|
-
def callback() -> None:
|
30
|
-
"""Manage Workflow Orchestration CLI.
|
31
|
-
|
32
|
-
Use it with the interface workflow engine.
|
33
|
-
"""
|
34
|
-
|
35
|
-
|
36
|
-
@app.command()
|
37
|
-
def version() -> None:
|
38
|
-
"""Get the ddeutil-workflow package version."""
|
39
|
-
typer.echo(f"ddeutil-workflow=={__version__}")
|
40
|
-
typer.echo(f"python-version=={python_version()}")
|
41
|
-
|
42
|
-
|
43
|
-
@app.command()
|
44
|
-
def init() -> None:
|
45
|
-
"""Initialize a Workflow structure on the current context."""
|
46
|
-
config.conf_path.mkdir(exist_ok=True)
|
47
|
-
(config.conf_path / ".confignore").touch()
|
48
|
-
|
49
|
-
conf_example_path: Path = config.conf_path / "examples"
|
50
|
-
conf_example_path.mkdir(exist_ok=True)
|
51
|
-
|
52
|
-
example_template: Path = conf_example_path / "wf_examples.yml"
|
53
|
-
example_template.write_text(
|
54
|
-
dedent(
|
55
|
-
"""
|
56
|
-
# Example workflow template.
|
57
|
-
wf-example:
|
58
|
-
type: Workflow
|
59
|
-
desc: |
|
60
|
-
An example workflow template that provide the demo of workflow.
|
61
|
-
params:
|
62
|
-
name:
|
63
|
-
type: str
|
64
|
-
default: "World"
|
65
|
-
jobs:
|
66
|
-
first-job:
|
67
|
-
stages:
|
68
|
-
|
69
|
-
- name: "Hello Stage"
|
70
|
-
echo: "Start say hi to the console"
|
71
|
-
|
72
|
-
- name: "Call tasks"
|
73
|
-
uses: tasks/say-hello-func@example
|
74
|
-
with:
|
75
|
-
name: ${{ params.name }}
|
76
|
-
second-job:
|
77
|
-
|
78
|
-
- name: "Hello Env"
|
79
|
-
echo: "Start say hi with ${ WORKFLOW_DEMO_HELLO }"
|
80
|
-
"""
|
81
|
-
).lstrip("\n")
|
82
|
-
)
|
83
|
-
|
84
|
-
if "." in config.registry_caller:
|
85
|
-
task_path = Path("./tasks")
|
86
|
-
task_path.mkdir(exist_ok=True)
|
87
|
-
|
88
|
-
dummy_tasks_path = task_path / "example.py"
|
89
|
-
dummy_tasks_path.write_text(
|
90
|
-
dedent(
|
91
|
-
"""
|
92
|
-
from ddeutil.workflow import Result, tag
|
93
|
-
|
94
|
-
@tag(name="example", alias="say-hello-func")
|
95
|
-
def hello_world_task(name: str, rs: Result) -> dict[str, str]:
|
96
|
-
\"\"\"Logging hello task function\"\"\"
|
97
|
-
rs.trace.info(f"Hello, {name}")
|
98
|
-
return {"name": name}
|
99
|
-
"""
|
100
|
-
).lstrip("\n")
|
101
|
-
)
|
102
|
-
|
103
|
-
init_path = task_path / "__init__.py"
|
104
|
-
init_path.write_text("from .example import hello_world_task\n")
|
105
|
-
|
106
|
-
dotenv_file = Path(".env")
|
107
|
-
mode: str = "a" if dotenv_file.exists() else "w"
|
108
|
-
with dotenv_file.open(mode=mode) as f:
|
109
|
-
f.write("\n# Workflow env vars\n")
|
110
|
-
f.write(
|
111
|
-
"WORKFLOW_DEMO_HELLO=foo\n"
|
112
|
-
"WORKFLOW_CORE_DEBUG_MODE=true\n"
|
113
|
-
"WORKFLOW_LOG_TIMEZONE=Asia/Bangkok\n"
|
114
|
-
"WORKFLOW_LOG_TRACE_ENABLE_WRITE=false\n"
|
115
|
-
"WORKFLOW_LOG_AUDIT_ENABLE_WRITE=true\n"
|
116
|
-
)
|
117
|
-
|
118
|
-
typer.echo("Starter command:")
|
119
|
-
typer.echo(
|
120
|
-
"> `source .env && workflow-cli workflows execute --name=wf-example`"
|
121
|
-
)
|
122
|
-
|
123
|
-
|
124
|
-
@app.command(name="job")
|
125
|
-
def execute_job(
|
126
|
-
params: Annotated[str, typer.Option(help="A job execute parameters")],
|
127
|
-
job: Annotated[str, typer.Option(help="A job model")],
|
128
|
-
run_id: Annotated[str, typer.Option(help="A running ID")],
|
129
|
-
) -> None:
|
130
|
-
"""Job execution on the local.
|
131
|
-
|
132
|
-
Example:
|
133
|
-
... workflow-cli job --params \"{\\\"test\\\": 1}\"
|
134
|
-
"""
|
135
|
-
try:
|
136
|
-
params_dict: dict[str, Any] = json.loads(params)
|
137
|
-
except json.JSONDecodeError as e:
|
138
|
-
raise ValueError(f"Params does not support format: {params!r}.") from e
|
139
|
-
|
140
|
-
try:
|
141
|
-
job_dict: dict[str, Any] = json.loads(job)
|
142
|
-
_job: Job = Job.model_validate(obj=job_dict)
|
143
|
-
except json.JSONDecodeError as e:
|
144
|
-
raise ValueError(f"Jobs does not support format: {job!r}.") from e
|
145
|
-
|
146
|
-
typer.echo(f"Job params: {params_dict}")
|
147
|
-
context: DictData = {}
|
148
|
-
try:
|
149
|
-
_job.set_outputs(
|
150
|
-
_job.execute(params=params_dict, run_id=run_id).context,
|
151
|
-
to=context,
|
152
|
-
)
|
153
|
-
typer.echo("[JOB]: Context result:")
|
154
|
-
typer.echo(json.dumps(context, default=str, indent=0))
|
155
|
-
except JobError as err:
|
156
|
-
typer.echo(f"[JOB]: {err.__class__.__name__}: {err}")
|
157
|
-
|
158
|
-
|
159
|
-
@app.command()
|
160
|
-
def api(
|
161
|
-
host: Annotated[str, typer.Option(help="A host url.")] = "0.0.0.0",
|
162
|
-
port: Annotated[int, typer.Option(help="A port url.")] = 80,
|
163
|
-
debug: Annotated[bool, typer.Option(help="A debug mode flag")] = True,
|
164
|
-
workers: Annotated[int, typer.Option(help="A worker number")] = None,
|
165
|
-
reload: Annotated[bool, typer.Option(help="A reload flag")] = False,
|
166
|
-
):
|
167
|
-
"""
|
168
|
-
Provision API application from the FastAPI.
|
169
|
-
"""
|
170
|
-
import uvicorn
|
171
|
-
|
172
|
-
from .api import app as fastapp
|
173
|
-
from .api.log_conf import LOGGING_CONFIG
|
174
|
-
|
175
|
-
# LOGGING_CONFIG = {}
|
176
|
-
|
177
|
-
uvicorn.run(
|
178
|
-
fastapp,
|
179
|
-
host=host,
|
180
|
-
port=port,
|
181
|
-
log_config=uvicorn.config.LOGGING_CONFIG | LOGGING_CONFIG,
|
182
|
-
# NOTE: Logging level of uvicorn should be lowered case.
|
183
|
-
log_level=("debug" if debug else "info"),
|
184
|
-
workers=workers,
|
185
|
-
reload=reload,
|
186
|
-
)
|
187
|
-
|
188
|
-
|
189
|
-
@app.command()
|
190
|
-
def make(
|
191
|
-
name: Annotated[Path, typer.Argument()],
|
192
|
-
) -> None:
|
193
|
-
"""
|
194
|
-
Create Workflow YAML template.
|
195
|
-
|
196
|
-
:param name:
|
197
|
-
"""
|
198
|
-
typer.echo(f"Start create YAML template filename: {name.resolve()}")
|
199
|
-
|
200
|
-
|
201
|
-
workflow_app = typer.Typer()
|
202
|
-
app.add_typer(workflow_app, name="workflows", help="An Only Workflow CLI.")
|
203
|
-
|
204
|
-
|
205
|
-
@workflow_app.callback()
|
206
|
-
def workflow_callback():
|
207
|
-
"""Manage Only Workflow CLI."""
|
208
|
-
|
209
|
-
|
210
|
-
@workflow_app.command(name="execute")
|
211
|
-
def workflow_execute(
|
212
|
-
name: Annotated[
|
213
|
-
str,
|
214
|
-
typer.Option(help="A name of workflow template."),
|
215
|
-
],
|
216
|
-
params: Annotated[
|
217
|
-
str,
|
218
|
-
typer.Option(help="A workflow execute parameters"),
|
219
|
-
] = "{}",
|
220
|
-
):
|
221
|
-
"""Execute workflow by passing a workflow template name."""
|
222
|
-
try:
|
223
|
-
params_dict: dict[str, Any] = json.loads(params)
|
224
|
-
except json.JSONDecodeError as e:
|
225
|
-
raise ValueError(f"Params does not support format: {params!r}.") from e
|
226
|
-
|
227
|
-
typer.echo(f"Start execute workflow template: {name}")
|
228
|
-
typer.echo(f"... with params: {params_dict}")
|
229
|
-
|
230
|
-
|
231
|
-
WORKFLOW_TYPE = Literal["Workflow"]
|
232
|
-
|
233
|
-
|
234
|
-
class WorkflowSchema(Workflow):
|
235
|
-
"""Override workflow model fields for generate JSON schema file."""
|
236
|
-
|
237
|
-
type: WORKFLOW_TYPE = Field(description="A type of workflow template.")
|
238
|
-
name: Optional[str] = Field(default=None, description="A workflow name.")
|
239
|
-
params: dict[str, Union[Param, str]] = Field(
|
240
|
-
default_factory=dict,
|
241
|
-
description="A parameters that need to use on this workflow.",
|
242
|
-
)
|
243
|
-
|
244
|
-
|
245
|
-
@workflow_app.command(name="json-schema")
|
246
|
-
def workflow_json_schema(
|
247
|
-
output: Annotated[
|
248
|
-
Path,
|
249
|
-
typer.Option(help="An output file to export the JSON schema."),
|
250
|
-
] = Path("./json-schema.json"),
|
251
|
-
) -> None:
|
252
|
-
"""Generate JSON schema file from the Workflow model."""
|
253
|
-
template = dict[str, WorkflowSchema]
|
254
|
-
json_schema = TypeAdapter(template).json_schema(by_alias=True)
|
255
|
-
template_schema: dict[str, str] = {
|
256
|
-
"$schema": "http://json-schema.org/draft-07/schema#",
|
257
|
-
"title": "Workflow Configuration JSON Schema",
|
258
|
-
"version": __version__,
|
259
|
-
}
|
260
|
-
with open(output, mode="w", encoding="utf-8") as f:
|
261
|
-
json.dump(template_schema | json_schema, f, indent=2)
|
262
|
-
|
263
|
-
|
264
|
-
log_app = typer.Typer()
|
265
|
-
app.add_typer(log_app, name="logs", help="An Only Log CLI.")
|
266
|
-
|
267
|
-
|
268
|
-
@log_app.callback()
|
269
|
-
def log_callback():
|
270
|
-
"""Manage Only Log CLI."""
|
271
|
-
|
272
|
-
|
273
|
-
if __name__ == "__main__":
|
274
|
-
app()
|
@@ -1,36 +0,0 @@
|
|
1
|
-
ddeutil/workflow/__about__.py,sha256=0fYC3KyobGtx1NtfXlGCbR9mrpL_yrH0UY_kp0NWaN8,28
|
2
|
-
ddeutil/workflow/__cron.py,sha256=avOagaHl9xXOmizeRWm13cOrty9Tw0vRjFq-xoEgpAY,29167
|
3
|
-
ddeutil/workflow/__init__.py,sha256=pRnUNCrwnKGrEQNSmdU9Ybf1tEQKg4LfsCpyj1Y3mhg,3241
|
4
|
-
ddeutil/workflow/__main__.py,sha256=Qd-f8z2Q2vpiEP2x6PBFsJrpACWDVxFKQk820MhFmHo,59
|
5
|
-
ddeutil/workflow/__types.py,sha256=tA2vsr6mzTSzbWB1sb62c5GgxODlfVRz6FvgLNJtQao,4788
|
6
|
-
ddeutil/workflow/audits.py,sha256=h7WVHe3z_FqtmLRvAxhnSqae8fFtZPN0tICrVr39wP4,25456
|
7
|
-
ddeutil/workflow/cli.py,sha256=aNFOZ3Re_QJBBP6vkT9Lsjrg8wLxrw_LKrl-1SIvSOg,8152
|
8
|
-
ddeutil/workflow/conf.py,sha256=4bFl1ufX7-p6ely7tJnanpTZ0wJoik81yTcmBrbcyxY,16661
|
9
|
-
ddeutil/workflow/errors.py,sha256=UpUIqoyqkvzqjuxtUQ9535l1HeAsyh-plEG0PgDVR2w,5541
|
10
|
-
ddeutil/workflow/event.py,sha256=qm7QHw-Pozm6oIUzAIxpDkPzzVZVtHgJIUlIle0vEfQ,13943
|
11
|
-
ddeutil/workflow/job.py,sha256=lSmOgh4l3_gBJXrTEWULhSSol648h6zPe6zKzz8jDHQ,46597
|
12
|
-
ddeutil/workflow/params.py,sha256=y9f6DEIyae1j4awbj3Kbeq75-U2UPFlKv9K57Hdo_Go,17188
|
13
|
-
ddeutil/workflow/result.py,sha256=BOk3DZMtmdE7xzQYeEYTGFlIkzJQ4Ed3fYzf0zF8Jo8,8963
|
14
|
-
ddeutil/workflow/reusables.py,sha256=g_Cac3yHy0H5ffl4Bb8_eGl284ELxOuX4LI8GYPMZgw,24983
|
15
|
-
ddeutil/workflow/stages.py,sha256=QufIa2b7A_ngOndVoGzyxKm_o5ZrauNeqxAC4vBkKFM,122678
|
16
|
-
ddeutil/workflow/traces.py,sha256=e12_rDnwVo-XR6Ca1KLwCjo3tAwbJP7yXc1YU62YOt8,73415
|
17
|
-
ddeutil/workflow/utils.py,sha256=-E-Z5hN_UTFuWDk-NpfKhNj0QtLfJSvZNDI5NzJsd5E,12122
|
18
|
-
ddeutil/workflow/workflow.py,sha256=Uojf7k7l91sqOlsPMeSPwQmbrB5pgbWEmx9QgKYngmI,42924
|
19
|
-
ddeutil/workflow/api/__init__.py,sha256=5DzYL3ngceoRshh5HYCSVWChqNJSiP01E1bEd8XxPi0,4799
|
20
|
-
ddeutil/workflow/api/log_conf.py,sha256=WfS3udDLSyrP-C80lWOvxxmhd_XWKvQPkwDqKblcH3E,1834
|
21
|
-
ddeutil/workflow/api/routes/__init__.py,sha256=JRaJZB0D6mgR17MbZo8yLtdYDtD62AA8MdKlFqhG84M,420
|
22
|
-
ddeutil/workflow/api/routes/job.py,sha256=8eu2OAOS3fENQ54OO723lFpzgHMyz1D-b_xZj6OnmcA,2550
|
23
|
-
ddeutil/workflow/api/routes/logs.py,sha256=O0I9L059SvtVHZ-TXCShChxbrHKUoT7MYRK0xZWwIMc,5282
|
24
|
-
ddeutil/workflow/api/routes/workflows.py,sha256=0pEZEsIrscRFBXG9gf6nttKw0aNbcdw7NsAZKLoKWtk,4392
|
25
|
-
ddeutil/workflow/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
26
|
-
ddeutil/workflow/plugins/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
27
|
-
ddeutil/workflow/plugins/providers/aws.py,sha256=61uIFBEWt-_D5Sui24qUPier1Hiqlw_RP_eY-rXBCKc,31551
|
28
|
-
ddeutil/workflow/plugins/providers/az.py,sha256=o3dh011lEtmr7-d7FPZJPgXdT0ytFzKfc5xnVxSyXGU,34867
|
29
|
-
ddeutil/workflow/plugins/providers/container.py,sha256=DSN0RWxMjTJN5ANheeMauDaPa3X6Z2E1eGUcctYkENw,22134
|
30
|
-
ddeutil/workflow/plugins/providers/gcs.py,sha256=KgAOdMBvdbMLTH_z_FwVriBFtZfKEYx8_34jzUOVjTY,27460
|
31
|
-
ddeutil_workflow-0.0.80.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
32
|
-
ddeutil_workflow-0.0.80.dist-info/METADATA,sha256=g43RJsjquJoYyQUl_d5Wfuxdoh00tukjjjgW-lL7Hdw,16087
|
33
|
-
ddeutil_workflow-0.0.80.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
34
|
-
ddeutil_workflow-0.0.80.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
|
35
|
-
ddeutil_workflow-0.0.80.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
36
|
-
ddeutil_workflow-0.0.80.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|