ddeutil-workflow 0.0.67__tar.gz → 0.0.69__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ddeutil_workflow-0.0.67/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.69}/PKG-INFO +31 -29
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/README.md +28 -26
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/pyproject.toml +4 -3
- ddeutil_workflow-0.0.69/src/ddeutil/workflow/__about__.py +1 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/__init__.py +14 -12
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/api/__init__.py +20 -19
- ddeutil_workflow-0.0.69/src/ddeutil/workflow/api/log_conf.py +59 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/api/routes/__init__.py +3 -3
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/api/routes/job.py +42 -16
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/api/routes/logs.py +8 -8
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/api/routes/workflows.py +12 -11
- ddeutil_workflow-0.0.69/src/ddeutil/workflow/audits.py +374 -0
- ddeutil_workflow-0.0.69/src/ddeutil/workflow/cli.py +133 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/conf.py +9 -52
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/event.py +6 -5
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/result.py +10 -1
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/stages.py +38 -9
- ddeutil_workflow-0.0.67/src/ddeutil/workflow/logs.py → ddeutil_workflow-0.0.69/src/ddeutil/workflow/traces.py +168 -387
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/utils.py +1 -52
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/workflow.py +8 -16
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69/src/ddeutil_workflow.egg-info}/PKG-INFO +31 -29
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil_workflow.egg-info/SOURCES.txt +5 -4
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil_workflow.egg-info/requires.txt +2 -2
- ddeutil_workflow-0.0.67/tests/test_logs_audit.py → ddeutil_workflow-0.0.69/tests/test_audits.py +1 -1
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test_conf.py +18 -13
- ddeutil_workflow-0.0.67/tests/test_logs_trace.py → ddeutil_workflow-0.0.69/tests/test_traces.py +6 -7
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test_utils.py +0 -10
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test_workflow.py +5 -0
- ddeutil_workflow-0.0.67/src/ddeutil/workflow/__about__.py +0 -1
- ddeutil_workflow-0.0.67/src/ddeutil/workflow/api/logs.py +0 -59
- ddeutil_workflow-0.0.67/src/ddeutil/workflow/cli.py +0 -68
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/LICENSE +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/setup.cfg +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/__cron.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/__main__.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/__types.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/errors.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/job.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/params.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/reusables.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil_workflow.egg-info/dependency_links.txt +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil_workflow.egg-info/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil_workflow.egg-info/top_level.txt +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test__cron.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test__regex.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test_errors.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test_event.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test_job.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test_job_exec.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test_job_exec_strategy.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test_params.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test_result.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test_reusables_call_tag.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test_reusables_func_model.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test_reusables_template.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test_reusables_template_filter.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test_strategy.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test_workflow_exec.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test_workflow_exec_job.py +0 -0
- {ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/tests/test_workflow_release.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ddeutil-workflow
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.69
|
|
4
4
|
Summary: Lightweight workflow orchestration with YAML template
|
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -24,10 +24,10 @@ Description-Content-Type: text/markdown
|
|
|
24
24
|
License-File: LICENSE
|
|
25
25
|
Requires-Dist: ddeutil[checksum]>=0.4.8
|
|
26
26
|
Requires-Dist: ddeutil-io[toml,yaml]>=0.2.14
|
|
27
|
-
Requires-Dist: pydantic==2.11.
|
|
27
|
+
Requires-Dist: pydantic==2.11.5
|
|
28
28
|
Requires-Dist: pydantic-extra-types==2.10.4
|
|
29
29
|
Requires-Dist: python-dotenv==1.1.0
|
|
30
|
-
Requires-Dist: typer
|
|
30
|
+
Requires-Dist: typer>=0.16.0
|
|
31
31
|
Provides-Extra: all
|
|
32
32
|
Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "all"
|
|
33
33
|
Requires-Dist: uvicorn; extra == "all"
|
|
@@ -61,12 +61,13 @@ by a `.yaml` template.
|
|
|
61
61
|
|
|
62
62
|
---
|
|
63
63
|
|
|
64
|
-
**:pushpin: <u>Rules of This Workflow
|
|
64
|
+
**:pushpin: <u>Rules of This Workflow</u>**:
|
|
65
65
|
|
|
66
66
|
1. The Minimum frequency unit of built-in scheduling is **1 Minute** 🕘
|
|
67
67
|
2. **Can not** re-run only failed stage and its pending downstream ↩️
|
|
68
68
|
3. All parallel tasks inside workflow core engine use **Multi-Threading** pool
|
|
69
69
|
(Python 3.13 unlock GIL 🐍🔓)
|
|
70
|
+
4. Recommend to pass a **Secret Value** with environment variable in YAML template 🔐
|
|
70
71
|
|
|
71
72
|
---
|
|
72
73
|
|
|
@@ -230,14 +231,17 @@ class RestAuth(BaseModel):
|
|
|
230
231
|
|
|
231
232
|
@tag("requests", alias="get-api-with-oauth-to-s3")
|
|
232
233
|
def get_api_with_oauth_to_s3(
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
234
|
+
method: str,
|
|
235
|
+
url: str,
|
|
236
|
+
body: dict[str, str],
|
|
237
|
+
auth: RestAuth,
|
|
238
|
+
writing_node: str,
|
|
239
|
+
aws: AwsCredential,
|
|
240
|
+
result: Result,
|
|
240
241
|
) -> dict[str, int]:
|
|
242
|
+
"""Get the data from RestAPI via Authenticate with OAuth and then store to
|
|
243
|
+
AWS S3 service.
|
|
244
|
+
"""
|
|
241
245
|
result.trace.info("[CALLER]: Start get data via RestAPI to S3.")
|
|
242
246
|
result.trace.info(f"... {method}: {url}")
|
|
243
247
|
if method != "post":
|
|
@@ -269,24 +273,22 @@ it will use default value and do not raise any error to you.
|
|
|
269
273
|
> The config value that you will set on the environment should combine with
|
|
270
274
|
> prefix, component, and name which is `WORKFLOW_{component}_{name}` (Upper case).
|
|
271
275
|
|
|
272
|
-
| Name | Component | Default | Description
|
|
273
|
-
|
|
274
|
-
| **REGISTRY_CALLER** |
|
|
275
|
-
| **REGISTRY_FILTER** |
|
|
276
|
-
| **CONF_PATH** |
|
|
277
|
-
| **TIMEZONE** |
|
|
278
|
-
| **STAGE_DEFAULT_ID** |
|
|
279
|
-
| **
|
|
280
|
-
| **
|
|
281
|
-
| **
|
|
282
|
-
| **
|
|
283
|
-
| **
|
|
284
|
-
| **
|
|
285
|
-
| **
|
|
286
|
-
| **
|
|
287
|
-
| **
|
|
288
|
-
| **AUDIT_PATH** | Log | `./audits` | |
|
|
289
|
-
| **AUDIT_ENABLE_WRITE** | Log | `true` | A flag that enable logging object saving log to its destination. |
|
|
276
|
+
| Name | Component | Default | Description |
|
|
277
|
+
|:-----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------|
|
|
278
|
+
| **REGISTRY_CALLER** | CORE | `.` | List of importable string for the call stage. |
|
|
279
|
+
| **REGISTRY_FILTER** | CORE | `ddeutil.workflow.templates` | List of importable string for the filter template. |
|
|
280
|
+
| **CONF_PATH** | CORE | `./conf` | The config path that keep all template `.yaml` files. |
|
|
281
|
+
| **TIMEZONE** | CORE | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
|
282
|
+
| **STAGE_DEFAULT_ID** | CORE | `false` | A flag that enable default stage ID that use for catch an execution output. |
|
|
283
|
+
| **GENERATE_ID_SIMPLE_MODE** | CORE | `true` | A flog that enable generating ID with `md5` algorithm. |
|
|
284
|
+
| **DEBUG_MODE** | LOG | `true` | A flag that enable logging with debug level mode. |
|
|
285
|
+
| **FORMAT** | LOG | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | A trace message console format. |
|
|
286
|
+
| **FORMAT_FILE** | LOG | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})` | A trace message format that use to write to target pointer. |
|
|
287
|
+
| **DATETIME_FORMAT** | LOG | `%Y-%m-%d %H:%M:%S` | A datetime format of the trace log. |
|
|
288
|
+
| **TRACE_PATH** | LOG | `./logs` | A pointer of trace log that use to store. |
|
|
289
|
+
| **TRACE_ENABLE_WRITE** | LOG | `false` | A flag that enable writing trace log. |
|
|
290
|
+
| **AUDIT_PATH** | LOG | `./audits` | A pointer of audit log that use to store. |
|
|
291
|
+
| **AUDIT_ENABLE_WRITE** | LOG | `true` | A flag that enable writing audit log after end execution in the workflow release step. |
|
|
290
292
|
|
|
291
293
|
## :rocket: Deployment
|
|
292
294
|
|
|
@@ -19,12 +19,13 @@ by a `.yaml` template.
|
|
|
19
19
|
|
|
20
20
|
---
|
|
21
21
|
|
|
22
|
-
**:pushpin: <u>Rules of This Workflow
|
|
22
|
+
**:pushpin: <u>Rules of This Workflow</u>**:
|
|
23
23
|
|
|
24
24
|
1. The Minimum frequency unit of built-in scheduling is **1 Minute** 🕘
|
|
25
25
|
2. **Can not** re-run only failed stage and its pending downstream ↩️
|
|
26
26
|
3. All parallel tasks inside workflow core engine use **Multi-Threading** pool
|
|
27
27
|
(Python 3.13 unlock GIL 🐍🔓)
|
|
28
|
+
4. Recommend to pass a **Secret Value** with environment variable in YAML template 🔐
|
|
28
29
|
|
|
29
30
|
---
|
|
30
31
|
|
|
@@ -188,14 +189,17 @@ class RestAuth(BaseModel):
|
|
|
188
189
|
|
|
189
190
|
@tag("requests", alias="get-api-with-oauth-to-s3")
|
|
190
191
|
def get_api_with_oauth_to_s3(
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
192
|
+
method: str,
|
|
193
|
+
url: str,
|
|
194
|
+
body: dict[str, str],
|
|
195
|
+
auth: RestAuth,
|
|
196
|
+
writing_node: str,
|
|
197
|
+
aws: AwsCredential,
|
|
198
|
+
result: Result,
|
|
198
199
|
) -> dict[str, int]:
|
|
200
|
+
"""Get the data from RestAPI via Authenticate with OAuth and then store to
|
|
201
|
+
AWS S3 service.
|
|
202
|
+
"""
|
|
199
203
|
result.trace.info("[CALLER]: Start get data via RestAPI to S3.")
|
|
200
204
|
result.trace.info(f"... {method}: {url}")
|
|
201
205
|
if method != "post":
|
|
@@ -227,24 +231,22 @@ it will use default value and do not raise any error to you.
|
|
|
227
231
|
> The config value that you will set on the environment should combine with
|
|
228
232
|
> prefix, component, and name which is `WORKFLOW_{component}_{name}` (Upper case).
|
|
229
233
|
|
|
230
|
-
| Name | Component | Default | Description
|
|
231
|
-
|
|
232
|
-
| **REGISTRY_CALLER** |
|
|
233
|
-
| **REGISTRY_FILTER** |
|
|
234
|
-
| **CONF_PATH** |
|
|
235
|
-
| **TIMEZONE** |
|
|
236
|
-
| **STAGE_DEFAULT_ID** |
|
|
237
|
-
| **
|
|
238
|
-
| **
|
|
239
|
-
| **
|
|
240
|
-
| **
|
|
241
|
-
| **
|
|
242
|
-
| **
|
|
243
|
-
| **
|
|
244
|
-
| **
|
|
245
|
-
| **
|
|
246
|
-
| **AUDIT_PATH** | Log | `./audits` | |
|
|
247
|
-
| **AUDIT_ENABLE_WRITE** | Log | `true` | A flag that enable logging object saving log to its destination. |
|
|
234
|
+
| Name | Component | Default | Description |
|
|
235
|
+
|:-----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------|
|
|
236
|
+
| **REGISTRY_CALLER** | CORE | `.` | List of importable string for the call stage. |
|
|
237
|
+
| **REGISTRY_FILTER** | CORE | `ddeutil.workflow.templates` | List of importable string for the filter template. |
|
|
238
|
+
| **CONF_PATH** | CORE | `./conf` | The config path that keep all template `.yaml` files. |
|
|
239
|
+
| **TIMEZONE** | CORE | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
|
240
|
+
| **STAGE_DEFAULT_ID** | CORE | `false` | A flag that enable default stage ID that use for catch an execution output. |
|
|
241
|
+
| **GENERATE_ID_SIMPLE_MODE** | CORE | `true` | A flog that enable generating ID with `md5` algorithm. |
|
|
242
|
+
| **DEBUG_MODE** | LOG | `true` | A flag that enable logging with debug level mode. |
|
|
243
|
+
| **FORMAT** | LOG | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | A trace message console format. |
|
|
244
|
+
| **FORMAT_FILE** | LOG | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})` | A trace message format that use to write to target pointer. |
|
|
245
|
+
| **DATETIME_FORMAT** | LOG | `%Y-%m-%d %H:%M:%S` | A datetime format of the trace log. |
|
|
246
|
+
| **TRACE_PATH** | LOG | `./logs` | A pointer of trace log that use to store. |
|
|
247
|
+
| **TRACE_ENABLE_WRITE** | LOG | `false` | A flag that enable writing trace log. |
|
|
248
|
+
| **AUDIT_PATH** | LOG | `./audits` | A pointer of audit log that use to store. |
|
|
249
|
+
| **AUDIT_ENABLE_WRITE** | LOG | `true` | A flag that enable writing audit log after end execution in the workflow release step. |
|
|
248
250
|
|
|
249
251
|
## :rocket: Deployment
|
|
250
252
|
|
|
@@ -27,10 +27,10 @@ requires-python = ">=3.9.13"
|
|
|
27
27
|
dependencies = [
|
|
28
28
|
"ddeutil[checksum]>=0.4.8",
|
|
29
29
|
"ddeutil-io[yaml,toml]>=0.2.14",
|
|
30
|
-
"pydantic==2.11.
|
|
30
|
+
"pydantic==2.11.5",
|
|
31
31
|
"pydantic-extra-types==2.10.4",
|
|
32
32
|
"python-dotenv==1.1.0",
|
|
33
|
-
"typer
|
|
33
|
+
"typer>=0.16.0",
|
|
34
34
|
]
|
|
35
35
|
dynamic = ["version"]
|
|
36
36
|
|
|
@@ -78,9 +78,10 @@ omit = [
|
|
|
78
78
|
"src/ddeutil/workflow/__about__.py",
|
|
79
79
|
"src/ddeutil/workflow/__cron.py",
|
|
80
80
|
"src/ddeutil/workflow/__main__.py",
|
|
81
|
+
"src/ddeutil/workflow/__types.py",
|
|
81
82
|
"src/ddeutil/workflow/cli.py",
|
|
82
83
|
"src/ddeutil/workflow/api/__init__.py",
|
|
83
|
-
"src/ddeutil/workflow/api/
|
|
84
|
+
"src/ddeutil/workflow/api/log_conf.py",
|
|
84
85
|
"src/ddeutil/workflow/api/routes/__init__.py",
|
|
85
86
|
"src/ddeutil/workflow/api/routes/job.py",
|
|
86
87
|
"src/ddeutil/workflow/api/routes/logs.py",
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__: str = "0.0.69"
|
|
@@ -5,23 +5,16 @@
|
|
|
5
5
|
# ------------------------------------------------------------------------------
|
|
6
6
|
from .__cron import CronJob, CronRunner
|
|
7
7
|
from .__types import DictData, DictStr, Matrix, Re, TupleStr
|
|
8
|
-
from .
|
|
9
|
-
from .errors import *
|
|
10
|
-
from .event import *
|
|
11
|
-
from .job import *
|
|
12
|
-
from .logs import (
|
|
8
|
+
from .audits import (
|
|
13
9
|
Audit,
|
|
14
10
|
AuditModel,
|
|
15
11
|
FileAudit,
|
|
16
|
-
FileTrace,
|
|
17
|
-
Trace,
|
|
18
|
-
TraceData,
|
|
19
|
-
TraceMeta,
|
|
20
|
-
TraceModel,
|
|
21
12
|
get_audit,
|
|
22
|
-
get_dt_tznow,
|
|
23
|
-
get_trace,
|
|
24
13
|
)
|
|
14
|
+
from .conf import *
|
|
15
|
+
from .errors import *
|
|
16
|
+
from .event import *
|
|
17
|
+
from .job import *
|
|
25
18
|
from .params import *
|
|
26
19
|
from .result import (
|
|
27
20
|
CANCEL,
|
|
@@ -34,5 +27,14 @@ from .result import (
|
|
|
34
27
|
)
|
|
35
28
|
from .reusables import *
|
|
36
29
|
from .stages import *
|
|
30
|
+
from .traces import (
|
|
31
|
+
ConsoleTrace,
|
|
32
|
+
FileTrace,
|
|
33
|
+
Trace,
|
|
34
|
+
TraceData,
|
|
35
|
+
TraceMeta,
|
|
36
|
+
TraceModel,
|
|
37
|
+
get_trace,
|
|
38
|
+
)
|
|
37
39
|
from .utils import *
|
|
38
40
|
from .workflow import *
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
from __future__ import annotations
|
|
7
7
|
|
|
8
8
|
import contextlib
|
|
9
|
+
import logging
|
|
9
10
|
from collections.abc import AsyncIterator
|
|
10
11
|
|
|
11
12
|
from dotenv import load_dotenv
|
|
@@ -19,11 +20,10 @@ from fastapi.responses import UJSONResponse
|
|
|
19
20
|
|
|
20
21
|
from ..__about__ import __version__
|
|
21
22
|
from ..conf import api_config
|
|
22
|
-
from ..logs import get_logger
|
|
23
23
|
from .routes import job, log, workflow
|
|
24
24
|
|
|
25
25
|
load_dotenv()
|
|
26
|
-
logger =
|
|
26
|
+
logger = logging.getLogger("uvicorn.error")
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
@contextlib.asynccontextmanager
|
|
@@ -58,12 +58,16 @@ app.add_middleware(
|
|
|
58
58
|
|
|
59
59
|
|
|
60
60
|
@app.get(path="/", response_class=UJSONResponse)
|
|
61
|
-
async def health():
|
|
61
|
+
async def health() -> UJSONResponse:
|
|
62
62
|
"""Index view that not return any template without json status."""
|
|
63
|
-
|
|
63
|
+
logger.info("[API]: Workflow API Application already running ...")
|
|
64
|
+
return UJSONResponse(
|
|
65
|
+
content={"message": "Workflow already start up with healthy status."},
|
|
66
|
+
status_code=st.HTTP_200_OK,
|
|
67
|
+
)
|
|
64
68
|
|
|
65
69
|
|
|
66
|
-
# NOTE Add the jobs and logs routes by default.
|
|
70
|
+
# NOTE: Add the jobs and logs routes by default.
|
|
67
71
|
app.include_router(job, prefix=api_config.prefix_path)
|
|
68
72
|
app.include_router(log, prefix=api_config.prefix_path)
|
|
69
73
|
app.include_router(workflow, prefix=api_config.prefix_path)
|
|
@@ -71,21 +75,18 @@ app.include_router(workflow, prefix=api_config.prefix_path)
|
|
|
71
75
|
|
|
72
76
|
@app.exception_handler(RequestValidationError)
|
|
73
77
|
async def validation_exception_handler(
|
|
74
|
-
request: Request,
|
|
75
|
-
|
|
78
|
+
request: Request,
|
|
79
|
+
exc: RequestValidationError,
|
|
80
|
+
) -> UJSONResponse:
|
|
81
|
+
"""Error Handler for model validate does not valid."""
|
|
76
82
|
_ = request
|
|
77
83
|
return UJSONResponse(
|
|
78
84
|
status_code=st.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
79
|
-
content=jsonable_encoder(
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
uvicorn.run(
|
|
87
|
-
app,
|
|
88
|
-
host="0.0.0.0",
|
|
89
|
-
port=80,
|
|
90
|
-
log_level="DEBUG",
|
|
85
|
+
content=jsonable_encoder(
|
|
86
|
+
{
|
|
87
|
+
"message": "Body does not parsing with model.",
|
|
88
|
+
"detail": exc.errors(),
|
|
89
|
+
"body": exc.body,
|
|
90
|
+
}
|
|
91
|
+
),
|
|
91
92
|
)
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
from uvicorn.config import LOGGING_CONFIG as LOGGING_CONFIG_UVICORN
|
|
4
|
+
|
|
5
|
+
from ..conf import config
|
|
6
|
+
|
|
7
|
+
LOGGING_CONFIG: dict[str, Any] = { # pragma: no cov
|
|
8
|
+
"version": 1,
|
|
9
|
+
"disable_existing_loggers": False,
|
|
10
|
+
"formatters": {
|
|
11
|
+
"default": LOGGING_CONFIG_UVICORN["formatters"]["default"],
|
|
12
|
+
"access": LOGGING_CONFIG_UVICORN["formatters"]["access"],
|
|
13
|
+
"custom": {
|
|
14
|
+
"format": config.log_format,
|
|
15
|
+
"datefmt": config.log_datetime_format,
|
|
16
|
+
},
|
|
17
|
+
},
|
|
18
|
+
"root": {
|
|
19
|
+
"level": "DEBUG" if config.debug else "INFO",
|
|
20
|
+
},
|
|
21
|
+
"handlers": {
|
|
22
|
+
"default": LOGGING_CONFIG_UVICORN["handlers"]["default"],
|
|
23
|
+
"access": LOGGING_CONFIG_UVICORN["handlers"]["access"],
|
|
24
|
+
"stream_custom": {
|
|
25
|
+
"formatter": "custom",
|
|
26
|
+
"class": "logging.StreamHandler",
|
|
27
|
+
"stream": "ext://sys.stdout",
|
|
28
|
+
},
|
|
29
|
+
# "file_handler": {
|
|
30
|
+
# "formatter": "custom_formatter",
|
|
31
|
+
# "class": "logging.handlers.RotatingFileHandler",
|
|
32
|
+
# "filename": "logs/app.log",
|
|
33
|
+
# "maxBytes": 1024 * 1024 * 1,
|
|
34
|
+
# "backupCount": 3,
|
|
35
|
+
# },
|
|
36
|
+
},
|
|
37
|
+
"loggers": {
|
|
38
|
+
"uvicorn": {
|
|
39
|
+
"handlers": ["default"],
|
|
40
|
+
"level": "DEBUG" if config.debug else "INFO",
|
|
41
|
+
"propagate": False,
|
|
42
|
+
},
|
|
43
|
+
"uvicorn.access": {
|
|
44
|
+
"handlers": ["access"],
|
|
45
|
+
"level": "DEBUG" if config.debug else "INFO",
|
|
46
|
+
"propagate": False,
|
|
47
|
+
},
|
|
48
|
+
"uvicorn.error": {
|
|
49
|
+
"handlers": ["default"],
|
|
50
|
+
"level": "DEBUG" if config.debug else "INFO",
|
|
51
|
+
},
|
|
52
|
+
"ddeutil.workflow": {
|
|
53
|
+
"handlers": ["stream_custom"],
|
|
54
|
+
"level": "INFO",
|
|
55
|
+
# "propagate": False,
|
|
56
|
+
"propagate": True,
|
|
57
|
+
},
|
|
58
|
+
},
|
|
59
|
+
}
|
{ddeutil_workflow-0.0.67 → ddeutil_workflow-0.0.69}/src/ddeutil/workflow/api/routes/__init__.py
RENAMED
|
@@ -3,6 +3,6 @@
|
|
|
3
3
|
# Licensed under the MIT License. See LICENSE in the project root for
|
|
4
4
|
# license information.
|
|
5
5
|
# ------------------------------------------------------------------------------
|
|
6
|
-
from .job import
|
|
7
|
-
from .logs import
|
|
8
|
-
from .workflows import
|
|
6
|
+
from .job import router as job
|
|
7
|
+
from .logs import router as log
|
|
8
|
+
from .workflows import router as workflow
|
|
@@ -5,20 +5,21 @@
|
|
|
5
5
|
# ------------------------------------------------------------------------------
|
|
6
6
|
from __future__ import annotations
|
|
7
7
|
|
|
8
|
+
import logging
|
|
8
9
|
from typing import Any, Optional
|
|
9
10
|
|
|
10
11
|
from fastapi import APIRouter
|
|
12
|
+
from fastapi import status as st
|
|
11
13
|
from fastapi.responses import UJSONResponse
|
|
12
14
|
from pydantic import BaseModel, Field
|
|
13
15
|
|
|
14
16
|
from ...__types import DictData
|
|
15
17
|
from ...errors import JobError
|
|
16
18
|
from ...job import Job
|
|
17
|
-
from ...logs import get_logger
|
|
18
19
|
from ...result import Result
|
|
19
20
|
|
|
20
|
-
logger =
|
|
21
|
-
|
|
21
|
+
logger = logging.getLogger("uvicorn.error")
|
|
22
|
+
router = APIRouter(prefix="/job", tags=["job"])
|
|
22
23
|
|
|
23
24
|
|
|
24
25
|
class ResultCreate(BaseModel):
|
|
@@ -32,14 +33,19 @@ class ResultCreate(BaseModel):
|
|
|
32
33
|
)
|
|
33
34
|
|
|
34
35
|
|
|
35
|
-
@
|
|
36
|
+
@router.post(
|
|
37
|
+
path="/execute/",
|
|
38
|
+
response_class=UJSONResponse,
|
|
39
|
+
status_code=st.HTTP_200_OK,
|
|
40
|
+
)
|
|
36
41
|
async def job_execute(
|
|
37
42
|
result: ResultCreate,
|
|
38
43
|
job: Job,
|
|
39
44
|
params: dict[str, Any],
|
|
40
45
|
extras: Optional[dict[str, Any]] = None,
|
|
41
|
-
):
|
|
46
|
+
) -> UJSONResponse:
|
|
42
47
|
"""Execute job via RestAPI with execute route path."""
|
|
48
|
+
logger.info("[API]: Start execute job ...")
|
|
43
49
|
rs: Result = Result(
|
|
44
50
|
run_id=result.run_id,
|
|
45
51
|
parent_run_id=result.parent_run_id,
|
|
@@ -61,15 +67,35 @@ async def job_execute(
|
|
|
61
67
|
)
|
|
62
68
|
except JobError as err:
|
|
63
69
|
rs.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
|
|
70
|
+
return UJSONResponse(
|
|
71
|
+
content={
|
|
72
|
+
"message": str(err),
|
|
73
|
+
"result": {
|
|
74
|
+
"run_id": rs.run_id,
|
|
75
|
+
"parent_run_id": rs.parent_run_id,
|
|
76
|
+
},
|
|
77
|
+
"job": job.model_dump(
|
|
78
|
+
by_alias=True,
|
|
79
|
+
exclude_none=False,
|
|
80
|
+
exclude_unset=True,
|
|
81
|
+
),
|
|
82
|
+
"params": params,
|
|
83
|
+
"context": context,
|
|
84
|
+
},
|
|
85
|
+
status_code=st.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
86
|
+
)
|
|
64
87
|
|
|
65
|
-
return
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
88
|
+
return UJSONResponse(
|
|
89
|
+
content={
|
|
90
|
+
"message": "Execute job via RestAPI successful.",
|
|
91
|
+
"result": {"run_id": rs.run_id, "parent_run_id": rs.parent_run_id},
|
|
92
|
+
"job": job.model_dump(
|
|
93
|
+
by_alias=True,
|
|
94
|
+
exclude_none=False,
|
|
95
|
+
exclude_unset=True,
|
|
96
|
+
),
|
|
97
|
+
"params": params,
|
|
98
|
+
"context": context,
|
|
99
|
+
},
|
|
100
|
+
status_code=st.HTTP_200_OK,
|
|
101
|
+
)
|
|
@@ -10,17 +10,17 @@ from fastapi import APIRouter, Path, Query
|
|
|
10
10
|
from fastapi import status as st
|
|
11
11
|
from fastapi.responses import UJSONResponse
|
|
12
12
|
|
|
13
|
-
from ...
|
|
13
|
+
from ...audits import get_audit
|
|
14
14
|
from ...result import Result
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
router = APIRouter(
|
|
17
17
|
prefix="/logs",
|
|
18
18
|
tags=["logs"],
|
|
19
19
|
default_response_class=UJSONResponse,
|
|
20
20
|
)
|
|
21
21
|
|
|
22
22
|
|
|
23
|
-
@
|
|
23
|
+
@router.get(
|
|
24
24
|
path="/traces/",
|
|
25
25
|
response_class=UJSONResponse,
|
|
26
26
|
status_code=st.HTTP_200_OK,
|
|
@@ -50,7 +50,7 @@ async def get_traces(
|
|
|
50
50
|
}
|
|
51
51
|
|
|
52
52
|
|
|
53
|
-
@
|
|
53
|
+
@router.get(
|
|
54
54
|
path="/traces/{run_id}",
|
|
55
55
|
response_class=UJSONResponse,
|
|
56
56
|
status_code=st.HTTP_200_OK,
|
|
@@ -77,7 +77,7 @@ async def get_trace_with_id(run_id: str):
|
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
|
|
80
|
-
@
|
|
80
|
+
@router.get(
|
|
81
81
|
path="/audits/",
|
|
82
82
|
response_class=UJSONResponse,
|
|
83
83
|
status_code=st.HTTP_200_OK,
|
|
@@ -94,7 +94,7 @@ async def get_audits():
|
|
|
94
94
|
}
|
|
95
95
|
|
|
96
96
|
|
|
97
|
-
@
|
|
97
|
+
@router.get(
|
|
98
98
|
path="/audits/{workflow}/",
|
|
99
99
|
response_class=UJSONResponse,
|
|
100
100
|
status_code=st.HTTP_200_OK,
|
|
@@ -113,7 +113,7 @@ async def get_audit_with_workflow(workflow: str):
|
|
|
113
113
|
}
|
|
114
114
|
|
|
115
115
|
|
|
116
|
-
@
|
|
116
|
+
@router.get(
|
|
117
117
|
path="/audits/{workflow}/{release}",
|
|
118
118
|
response_class=UJSONResponse,
|
|
119
119
|
status_code=st.HTTP_200_OK,
|
|
@@ -140,7 +140,7 @@ async def get_audit_with_workflow_release(
|
|
|
140
140
|
}
|
|
141
141
|
|
|
142
142
|
|
|
143
|
-
@
|
|
143
|
+
@router.get(
|
|
144
144
|
path="/audits/{workflow}/{release}/{run_id}",
|
|
145
145
|
response_class=UJSONResponse,
|
|
146
146
|
status_code=st.HTTP_200_OK,
|