ddeutil-workflow 0.0.72__tar.gz → 0.0.74__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ddeutil_workflow-0.0.72/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.74}/PKG-INFO +27 -17
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/README.md +26 -16
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/pyproject.toml +3 -1
- ddeutil_workflow-0.0.74/src/ddeutil/workflow/__about__.py +1 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/__cron.py +14 -8
- ddeutil_workflow-0.0.74/src/ddeutil/workflow/__init__.py +149 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/__types.py +53 -41
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/api/__init__.py +74 -3
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/api/routes/job.py +15 -29
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/api/routes/logs.py +9 -9
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/api/routes/workflows.py +3 -3
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/audits.py +70 -55
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/cli.py +1 -15
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/conf.py +78 -26
- ddeutil_workflow-0.0.74/src/ddeutil/workflow/errors.py +190 -0
- ddeutil_workflow-0.0.74/src/ddeutil/workflow/event.py +440 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/job.py +331 -192
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/params.py +37 -7
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/result.py +96 -70
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/reusables.py +56 -6
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/stages.py +1088 -575
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/traces.py +218 -128
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/utils.py +60 -8
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/workflow.py +424 -290
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74/src/ddeutil_workflow.egg-info}/PKG-INFO +27 -17
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_conf.py +38 -8
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_event.py +55 -84
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_job_exec_strategy.py +28 -23
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_result.py +0 -17
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_traces.py +11 -6
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_workflow.py +58 -29
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_workflow_exec_job.py +8 -6
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_workflow_release.py +10 -12
- ddeutil_workflow-0.0.72/src/ddeutil/workflow/__about__.py +0 -1
- ddeutil_workflow-0.0.72/src/ddeutil/workflow/__init__.py +0 -40
- ddeutil_workflow-0.0.72/src/ddeutil/workflow/errors.py +0 -123
- ddeutil_workflow-0.0.72/src/ddeutil/workflow/event.py +0 -341
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/LICENSE +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/setup.cfg +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/__main__.py +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/api/log_conf.py +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil/workflow/api/routes/__init__.py +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil_workflow.egg-info/SOURCES.txt +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil_workflow.egg-info/dependency_links.txt +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil_workflow.egg-info/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil_workflow.egg-info/requires.txt +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/src/ddeutil_workflow.egg-info/top_level.txt +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test__cron.py +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test__regex.py +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_audits.py +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_cli.py +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_errors.py +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_job.py +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_job_exec.py +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_params.py +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_reusables_call_tag.py +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_reusables_func_model.py +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_reusables_template.py +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_reusables_template_filter.py +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_strategy.py +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_utils.py +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_workflow_exec.py +0 -0
- {ddeutil_workflow-0.0.72 → ddeutil_workflow-0.0.74}/tests/test_workflow_rerun.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: ddeutil-workflow
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.74
|
4
4
|
Summary: Lightweight workflow orchestration with YAML template
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
6
6
|
License: MIT
|
@@ -68,6 +68,7 @@ by a `.yaml` template.
|
|
68
68
|
3. All parallel tasks inside workflow core engine use **Multi-Threading** pool
|
69
69
|
(Python 3.13 unlock GIL 🐍🔓)
|
70
70
|
4. Recommend to pass a **Secret Value** with environment variable in YAML template 🔐
|
71
|
+
5. Any datatime value convert to **No Timezone**
|
71
72
|
|
72
73
|
---
|
73
74
|
|
@@ -139,6 +140,15 @@ If you want to install this package with application add-ons, you should add
|
|
139
140
|
| Python | `ddeutil-workflow` | ✅ |
|
140
141
|
| FastAPI Server | `ddeutil-workflow[all]` | ✅ |
|
141
142
|
|
143
|
+
## 📖 Documentation
|
144
|
+
|
145
|
+
For comprehensive API documentation, examples, and best practices:
|
146
|
+
|
147
|
+
- **[Full Documentation](https://ddeutils.github.io/ddeutil-workflow/)** - Complete user guide and API reference
|
148
|
+
- **[Getting Started](https://ddeutils.github.io/ddeutil-workflow/getting-started/)** - Quick start guide
|
149
|
+
- **[API Reference](https://ddeutils.github.io/ddeutil-workflow/api/workflow/)** - Detailed API documentation
|
150
|
+
- **[Examples](https://ddeutils.github.io/ddeutil-workflow/examples/)** - Real-world usage examples
|
151
|
+
|
142
152
|
## 🎯 Usage
|
143
153
|
|
144
154
|
This is examples that use workflow file for running common Data Engineering
|
@@ -273,22 +283,22 @@ it will use default value and do not raise any error to you.
|
|
273
283
|
> The config value that you will set on the environment should combine with
|
274
284
|
> prefix, component, and name which is `WORKFLOW_{component}_{name}` (Upper case).
|
275
285
|
|
276
|
-
| Name
|
277
|
-
|
278
|
-
| **REGISTRY_CALLER**
|
279
|
-
| **REGISTRY_FILTER**
|
280
|
-
| **CONF_PATH**
|
281
|
-
| **TIMEZONE**
|
282
|
-
| **STAGE_DEFAULT_ID**
|
283
|
-
| **GENERATE_ID_SIMPLE_MODE**
|
284
|
-
| **DEBUG_MODE**
|
285
|
-
| **FORMAT**
|
286
|
-
| **FORMAT_FILE**
|
287
|
-
| **DATETIME_FORMAT**
|
288
|
-
| **
|
289
|
-
| **TRACE_ENABLE_WRITE**
|
290
|
-
| **
|
291
|
-
| **AUDIT_ENABLE_WRITE**
|
286
|
+
| Name | Component | Default | Description |
|
287
|
+
|:----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------|
|
288
|
+
| **REGISTRY_CALLER** | CORE | `.` | List of importable string for the call stage. |
|
289
|
+
| **REGISTRY_FILTER** | CORE | `ddeutil.workflow.templates` | List of importable string for the filter template. |
|
290
|
+
| **CONF_PATH** | CORE | `./conf` | The config path that keep all template `.yaml` files. |
|
291
|
+
| **TIMEZONE** | CORE | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
292
|
+
| **STAGE_DEFAULT_ID** | CORE | `false` | A flag that enable default stage ID that use for catch an execution output. |
|
293
|
+
| **GENERATE_ID_SIMPLE_MODE** | CORE | `true` | A flog that enable generating ID with `md5` algorithm. |
|
294
|
+
| **DEBUG_MODE** | LOG | `true` | A flag that enable logging with debug level mode. |
|
295
|
+
| **FORMAT** | LOG | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | A trace message console format. |
|
296
|
+
| **FORMAT_FILE** | LOG | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})` | A trace message format that use to write to target pointer. |
|
297
|
+
| **DATETIME_FORMAT** | LOG | `%Y-%m-%d %H:%M:%S` | A datetime format of the trace log. |
|
298
|
+
| **TRACE_URL** | LOG | `file:./logs` | A pointer URL of trace log that use to emit log message. |
|
299
|
+
| **TRACE_ENABLE_WRITE** | LOG | `false` | A flag that enable writing trace log. |
|
300
|
+
| **AUDIT_URL** | LOG | `file:./audits` | A pointer URL of audit log that use to write audit metrix. |
|
301
|
+
| **AUDIT_ENABLE_WRITE** | LOG | `true` | A flag that enable writing audit log after end execution in the workflow release step. |
|
292
302
|
|
293
303
|
## :rocket: Deployment
|
294
304
|
|
@@ -26,6 +26,7 @@ by a `.yaml` template.
|
|
26
26
|
3. All parallel tasks inside workflow core engine use **Multi-Threading** pool
|
27
27
|
(Python 3.13 unlock GIL 🐍🔓)
|
28
28
|
4. Recommend to pass a **Secret Value** with environment variable in YAML template 🔐
|
29
|
+
5. Any datatime value convert to **No Timezone**
|
29
30
|
|
30
31
|
---
|
31
32
|
|
@@ -97,6 +98,15 @@ If you want to install this package with application add-ons, you should add
|
|
97
98
|
| Python | `ddeutil-workflow` | ✅ |
|
98
99
|
| FastAPI Server | `ddeutil-workflow[all]` | ✅ |
|
99
100
|
|
101
|
+
## 📖 Documentation
|
102
|
+
|
103
|
+
For comprehensive API documentation, examples, and best practices:
|
104
|
+
|
105
|
+
- **[Full Documentation](https://ddeutils.github.io/ddeutil-workflow/)** - Complete user guide and API reference
|
106
|
+
- **[Getting Started](https://ddeutils.github.io/ddeutil-workflow/getting-started/)** - Quick start guide
|
107
|
+
- **[API Reference](https://ddeutils.github.io/ddeutil-workflow/api/workflow/)** - Detailed API documentation
|
108
|
+
- **[Examples](https://ddeutils.github.io/ddeutil-workflow/examples/)** - Real-world usage examples
|
109
|
+
|
100
110
|
## 🎯 Usage
|
101
111
|
|
102
112
|
This is examples that use workflow file for running common Data Engineering
|
@@ -231,22 +241,22 @@ it will use default value and do not raise any error to you.
|
|
231
241
|
> The config value that you will set on the environment should combine with
|
232
242
|
> prefix, component, and name which is `WORKFLOW_{component}_{name}` (Upper case).
|
233
243
|
|
234
|
-
| Name
|
235
|
-
|
236
|
-
| **REGISTRY_CALLER**
|
237
|
-
| **REGISTRY_FILTER**
|
238
|
-
| **CONF_PATH**
|
239
|
-
| **TIMEZONE**
|
240
|
-
| **STAGE_DEFAULT_ID**
|
241
|
-
| **GENERATE_ID_SIMPLE_MODE**
|
242
|
-
| **DEBUG_MODE**
|
243
|
-
| **FORMAT**
|
244
|
-
| **FORMAT_FILE**
|
245
|
-
| **DATETIME_FORMAT**
|
246
|
-
| **
|
247
|
-
| **TRACE_ENABLE_WRITE**
|
248
|
-
| **
|
249
|
-
| **AUDIT_ENABLE_WRITE**
|
244
|
+
| Name | Component | Default | Description |
|
245
|
+
|:----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------|
|
246
|
+
| **REGISTRY_CALLER** | CORE | `.` | List of importable string for the call stage. |
|
247
|
+
| **REGISTRY_FILTER** | CORE | `ddeutil.workflow.templates` | List of importable string for the filter template. |
|
248
|
+
| **CONF_PATH** | CORE | `./conf` | The config path that keep all template `.yaml` files. |
|
249
|
+
| **TIMEZONE** | CORE | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
250
|
+
| **STAGE_DEFAULT_ID** | CORE | `false` | A flag that enable default stage ID that use for catch an execution output. |
|
251
|
+
| **GENERATE_ID_SIMPLE_MODE** | CORE | `true` | A flog that enable generating ID with `md5` algorithm. |
|
252
|
+
| **DEBUG_MODE** | LOG | `true` | A flag that enable logging with debug level mode. |
|
253
|
+
| **FORMAT** | LOG | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | A trace message console format. |
|
254
|
+
| **FORMAT_FILE** | LOG | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})` | A trace message format that use to write to target pointer. |
|
255
|
+
| **DATETIME_FORMAT** | LOG | `%Y-%m-%d %H:%M:%S` | A datetime format of the trace log. |
|
256
|
+
| **TRACE_URL** | LOG | `file:./logs` | A pointer URL of trace log that use to emit log message. |
|
257
|
+
| **TRACE_ENABLE_WRITE** | LOG | `false` | A flag that enable writing trace log. |
|
258
|
+
| **AUDIT_URL** | LOG | `file:./audits` | A pointer URL of audit log that use to write audit metrix. |
|
259
|
+
| **AUDIT_ENABLE_WRITE** | LOG | `true` | A flag that enable writing audit log after end execution in the workflow release step. |
|
250
260
|
|
251
261
|
## :rocket: Deployment
|
252
262
|
|
@@ -116,7 +116,7 @@ filterwarnings = [
|
|
116
116
|
]
|
117
117
|
log_cli = true
|
118
118
|
log_cli_level = "DEBUG"
|
119
|
-
log_cli_format = "%(asctime)s [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)"
|
119
|
+
log_cli_format = "%(asctime)s [%(levelname)-7s] (%(cut_id)s) %(message)-120s (%(filename)s:%(lineno)s)"
|
120
120
|
log_cli_date_format = "%Y%m%d %H:%M:%S"
|
121
121
|
|
122
122
|
[tool.black]
|
@@ -130,6 +130,7 @@ exclude = """
|
|
130
130
|
| \\.__pycache__
|
131
131
|
| \\.idea
|
132
132
|
| \\.ruff_cache
|
133
|
+
| \\.rust
|
133
134
|
| \\.mypy_cache
|
134
135
|
| \\.pytest_cache
|
135
136
|
| \\.venv
|
@@ -147,6 +148,7 @@ exclude = [
|
|
147
148
|
".git",
|
148
149
|
".mypy_cache",
|
149
150
|
".ruff_cache",
|
151
|
+
".rust",
|
150
152
|
".venv",
|
151
153
|
"build",
|
152
154
|
"dist",
|
@@ -0,0 +1 @@
|
|
1
|
+
__version__: str = "0.0.74"
|
@@ -38,8 +38,11 @@ class YearReachLimit(Exception):
|
|
38
38
|
def str2cron(value: str) -> str: # pragma: no cov
|
39
39
|
"""Convert Special String with the @ prefix to Crontab value.
|
40
40
|
|
41
|
-
:
|
42
|
-
|
41
|
+
Args:
|
42
|
+
value: A string value that want to convert to cron value.
|
43
|
+
|
44
|
+
Returns:
|
45
|
+
str: The converted cron expression.
|
43
46
|
|
44
47
|
Table:
|
45
48
|
|
@@ -165,9 +168,10 @@ CRON_UNITS_YEAR: Units = CRON_UNITS + (
|
|
165
168
|
class CronPart:
|
166
169
|
"""Part of Cron object that represent a collection of positive integers.
|
167
170
|
|
168
|
-
:
|
169
|
-
|
170
|
-
|
171
|
+
Args:
|
172
|
+
unit: A Unit dataclass object.
|
173
|
+
values: A crontab values that want to validate
|
174
|
+
options: A Options dataclass object.
|
171
175
|
"""
|
172
176
|
|
173
177
|
__slots__: tuple[str, ...] = (
|
@@ -288,7 +292,11 @@ class CronPart:
|
|
288
292
|
"""Parses a string as a range of positive integers. The string should
|
289
293
|
include only `-` and `,` special strings.
|
290
294
|
|
291
|
-
:
|
295
|
+
Args:
|
296
|
+
value: A string value that want to parse
|
297
|
+
|
298
|
+
Returns:
|
299
|
+
tuple[int, ...]: Parsed range of integers.
|
292
300
|
|
293
301
|
TODO: support for `L`, `W`, and `#`
|
294
302
|
---
|
@@ -334,8 +342,6 @@ class CronPart:
|
|
334
342
|
- 15 10 ? * 6L 2002-2005
|
335
343
|
Run at 10:15am UTC on the last Friday of each month during the
|
336
344
|
years 2002 to 2005
|
337
|
-
|
338
|
-
:rtype: tuple[int, ...]
|
339
345
|
"""
|
340
346
|
interval_list: list[list[int]] = []
|
341
347
|
# NOTE: Start replace alternative like JAN to FEB or MON to SUN.
|
@@ -0,0 +1,149 @@
|
|
1
|
+
# ------------------------------------------------------------------------------
|
2
|
+
# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
|
3
|
+
# Licensed under the MIT License. See LICENSE in the project root for
|
4
|
+
# license information.
|
5
|
+
# ------------------------------------------------------------------------------
|
6
|
+
"""DDE Workflow - Lightweight Workflow Orchestration Package.
|
7
|
+
|
8
|
+
This package provides a comprehensive workflow orchestration system with YAML template
|
9
|
+
support. It enables developers to create, manage, and execute complex workflows with
|
10
|
+
minimal configuration.
|
11
|
+
|
12
|
+
Key Features:
|
13
|
+
- YAML-based workflow configuration
|
14
|
+
- Job and stage execution management
|
15
|
+
- Scheduling with cron-like syntax
|
16
|
+
- Parallel and sequential execution support
|
17
|
+
- Comprehensive error handling and logging
|
18
|
+
- Extensible stage types (Bash, Python, Docker, etc.)
|
19
|
+
- Matrix strategy for parameterized workflows
|
20
|
+
- Audit and tracing capabilities
|
21
|
+
|
22
|
+
Main Classes:
|
23
|
+
Workflow: Core workflow orchestration class
|
24
|
+
Job: Execution unit containing stages
|
25
|
+
Stage: Individual task execution unit
|
26
|
+
CronJob: Scheduled workflow execution
|
27
|
+
Audit: Execution tracking and logging
|
28
|
+
Result: Execution status and output management
|
29
|
+
|
30
|
+
Example:
|
31
|
+
Basic workflow usage:
|
32
|
+
|
33
|
+
```python
|
34
|
+
from ddeutil.workflow import Workflow
|
35
|
+
|
36
|
+
# Load workflow from configuration
|
37
|
+
workflow = Workflow.from_conf('my-workflow')
|
38
|
+
|
39
|
+
# Execute with parameters
|
40
|
+
result = workflow.execute({'param1': 'value1'})
|
41
|
+
|
42
|
+
if result.status == 'SUCCESS':
|
43
|
+
print("Workflow completed successfully")
|
44
|
+
```
|
45
|
+
|
46
|
+
Note:
|
47
|
+
This package requires Python 3.9+ and supports both synchronous and
|
48
|
+
asynchronous execution patterns.
|
49
|
+
"""
|
50
|
+
from .__cron import CronRunner
|
51
|
+
from .__types import DictData, DictStr, Matrix, Re, TupleStr
|
52
|
+
from .audits import (
|
53
|
+
Audit,
|
54
|
+
FileAudit,
|
55
|
+
get_audit_model,
|
56
|
+
)
|
57
|
+
from .conf import *
|
58
|
+
from .errors import (
|
59
|
+
BaseError,
|
60
|
+
JobCancelError,
|
61
|
+
JobError,
|
62
|
+
JobSkipError,
|
63
|
+
ResultError,
|
64
|
+
StageCancelError,
|
65
|
+
StageError,
|
66
|
+
StageSkipError,
|
67
|
+
UtilError,
|
68
|
+
WorkflowCancelError,
|
69
|
+
WorkflowError,
|
70
|
+
WorkflowTimeoutError,
|
71
|
+
to_dict,
|
72
|
+
)
|
73
|
+
from .event import (
|
74
|
+
Cron,
|
75
|
+
CronJob,
|
76
|
+
CronJobYear,
|
77
|
+
Crontab,
|
78
|
+
CrontabValue,
|
79
|
+
CrontabYear,
|
80
|
+
Event,
|
81
|
+
Interval,
|
82
|
+
)
|
83
|
+
from .job import (
|
84
|
+
Job,
|
85
|
+
OnAzBatch,
|
86
|
+
OnDocker,
|
87
|
+
OnLocal,
|
88
|
+
OnSelfHosted,
|
89
|
+
Rule,
|
90
|
+
RunsOnModel,
|
91
|
+
Strategy,
|
92
|
+
docker_execution,
|
93
|
+
local_execute,
|
94
|
+
local_execute_strategy,
|
95
|
+
self_hosted_execute,
|
96
|
+
)
|
97
|
+
from .params import (
|
98
|
+
ArrayParam,
|
99
|
+
DateParam,
|
100
|
+
DatetimeParam,
|
101
|
+
DecimalParam,
|
102
|
+
FloatParam,
|
103
|
+
IntParam,
|
104
|
+
MapParam,
|
105
|
+
Param,
|
106
|
+
StrParam,
|
107
|
+
)
|
108
|
+
from .result import (
|
109
|
+
CANCEL,
|
110
|
+
FAILED,
|
111
|
+
SKIP,
|
112
|
+
SUCCESS,
|
113
|
+
WAIT,
|
114
|
+
Result,
|
115
|
+
Status,
|
116
|
+
)
|
117
|
+
from .reusables import *
|
118
|
+
from .stages import (
|
119
|
+
BashStage,
|
120
|
+
CallStage,
|
121
|
+
CaseStage,
|
122
|
+
DockerStage,
|
123
|
+
EmptyStage,
|
124
|
+
ForEachStage,
|
125
|
+
ParallelStage,
|
126
|
+
PyStage,
|
127
|
+
RaiseStage,
|
128
|
+
Stage,
|
129
|
+
TriggerStage,
|
130
|
+
UntilStage,
|
131
|
+
VirtualPyStage,
|
132
|
+
)
|
133
|
+
from .traces import (
|
134
|
+
ConsoleTrace,
|
135
|
+
FileTrace,
|
136
|
+
Trace,
|
137
|
+
TraceData,
|
138
|
+
TraceMeta,
|
139
|
+
get_trace,
|
140
|
+
)
|
141
|
+
from .utils import *
|
142
|
+
from .workflow import (
|
143
|
+
EVENT,
|
144
|
+
FORCE,
|
145
|
+
NORMAL,
|
146
|
+
RERUN,
|
147
|
+
ReleaseType,
|
148
|
+
Workflow,
|
149
|
+
)
|
@@ -27,6 +27,47 @@ DictData = dict[str, Any]
|
|
27
27
|
DictStr = dict[str, str]
|
28
28
|
Matrix = dict[str, Union[list[str], list[int]]]
|
29
29
|
|
30
|
+
# Pre-compile regex patterns for better performance
|
31
|
+
_RE_CALLER_PATTERN = r"""
|
32
|
+
\$ # start with $
|
33
|
+
{{ # value open with {{
|
34
|
+
\s* # whitespace or not
|
35
|
+
(?P<caller>
|
36
|
+
(?P<caller_prefix>(?:[a-zA-Z_-]+\??\.)*)
|
37
|
+
(?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+\??)
|
38
|
+
)
|
39
|
+
\s* # whitespace or not
|
40
|
+
(?P<post_filters>
|
41
|
+
(?:
|
42
|
+
\|\s*
|
43
|
+
(?:
|
44
|
+
[a-zA-Z0-9_]{3,}
|
45
|
+
[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]*
|
46
|
+
)\s*
|
47
|
+
)*
|
48
|
+
)
|
49
|
+
}} # value close with }}
|
50
|
+
"""
|
51
|
+
|
52
|
+
_RE_TASK_FMT_PATTERN = r"""
|
53
|
+
^ # start task format
|
54
|
+
(?P<path>[^/@]+)
|
55
|
+
/ # start get function with /
|
56
|
+
(?P<func>[^@]+)
|
57
|
+
@ # start tag with @
|
58
|
+
(?P<tag>.+)
|
59
|
+
$ # end task format
|
60
|
+
"""
|
61
|
+
|
62
|
+
# Compile patterns at module level for better performance
|
63
|
+
RE_CALLER: Pattern = re.compile(
|
64
|
+
_RE_CALLER_PATTERN, MULTILINE | IGNORECASE | UNICODE | VERBOSE
|
65
|
+
)
|
66
|
+
|
67
|
+
RE_TASK_FMT: Pattern = re.compile(
|
68
|
+
_RE_TASK_FMT_PATTERN, MULTILINE | IGNORECASE | UNICODE | VERBOSE
|
69
|
+
)
|
70
|
+
|
30
71
|
|
31
72
|
class Context(TypedDict):
|
32
73
|
"""TypeDict support the Context."""
|
@@ -51,10 +92,12 @@ class CallerRe:
|
|
51
92
|
def from_regex(cls, match: Match[str]) -> Self:
|
52
93
|
"""Class construct from matching result.
|
53
94
|
|
54
|
-
:
|
55
|
-
|
95
|
+
Args:
|
96
|
+
match: A match string object for contract this Caller regex data
|
97
|
+
class.
|
56
98
|
|
57
|
-
:
|
99
|
+
Returns:
|
100
|
+
Self: The constructed CallerRe instance from regex match.
|
58
101
|
"""
|
59
102
|
return cls(full=match.group(0), **match.groupdict())
|
60
103
|
|
@@ -79,29 +122,7 @@ class Re:
|
|
79
122
|
# - ${{ params.datetime | fmt('%Y-%m-%d') }}
|
80
123
|
# - ${{ params.source?.schema }}
|
81
124
|
#
|
82
|
-
|
83
|
-
\$ # start with $
|
84
|
-
{{ # value open with {{
|
85
|
-
\s* # whitespace or not
|
86
|
-
(?P<caller>
|
87
|
-
(?P<caller_prefix>(?:[a-zA-Z_-]+\??\.)*)
|
88
|
-
(?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+\??)
|
89
|
-
)
|
90
|
-
\s* # whitespace or not
|
91
|
-
(?P<post_filters>
|
92
|
-
(?:
|
93
|
-
\|\s*
|
94
|
-
(?:
|
95
|
-
[a-zA-Z0-9_]{3,}
|
96
|
-
[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]*
|
97
|
-
)\s*
|
98
|
-
)*
|
99
|
-
)
|
100
|
-
}} # value close with }}
|
101
|
-
"""
|
102
|
-
RE_CALLER: Pattern = re.compile(
|
103
|
-
__re_caller, MULTILINE | IGNORECASE | UNICODE | VERBOSE
|
104
|
-
)
|
125
|
+
RE_CALLER: Pattern = RE_CALLER
|
105
126
|
|
106
127
|
# NOTE:
|
107
128
|
# Regular expression:
|
@@ -111,28 +132,19 @@ class Re:
|
|
111
132
|
# Examples:
|
112
133
|
# - tasks/function@dummy
|
113
134
|
#
|
114
|
-
|
115
|
-
^ # start task format
|
116
|
-
(?P<path>[^/@]+)
|
117
|
-
/ # start get function with /
|
118
|
-
(?P<func>[^@]+)
|
119
|
-
@ # start tag with @
|
120
|
-
(?P<tag>.+)
|
121
|
-
$ # end task format
|
122
|
-
"""
|
123
|
-
RE_TASK_FMT: Pattern = re.compile(
|
124
|
-
__re_task_fmt, MULTILINE | IGNORECASE | UNICODE | VERBOSE
|
125
|
-
)
|
135
|
+
RE_TASK_FMT: Pattern = RE_TASK_FMT
|
126
136
|
|
127
137
|
@classmethod
|
128
138
|
def finditer_caller(cls, value: str) -> Iterator[CallerRe]:
|
129
139
|
"""Generate CallerRe object that create from matching object that
|
130
140
|
extract with re.finditer function.
|
131
141
|
|
132
|
-
:
|
133
|
-
|
142
|
+
Args:
|
143
|
+
value: A string value that want to finditer with the caller
|
144
|
+
regular expression.
|
134
145
|
|
135
|
-
:
|
146
|
+
Yields:
|
147
|
+
CallerRe: CallerRe objects created from regex matches.
|
136
148
|
"""
|
137
149
|
for found in cls.RE_CALLER.finditer(value):
|
138
150
|
yield CallerRe.from_regex(found)
|
@@ -1,3 +1,30 @@
|
|
1
|
+
"""FastAPI Web Application for Workflow Management.
|
2
|
+
|
3
|
+
This module provides a RESTful API interface for workflow orchestration using
|
4
|
+
FastAPI. It enables remote workflow management, execution monitoring, and
|
5
|
+
provides endpoints for workflow operations.
|
6
|
+
|
7
|
+
The API supports:
|
8
|
+
- Workflow execution and management
|
9
|
+
- Job status monitoring
|
10
|
+
- Log streaming and access
|
11
|
+
- Result retrieval and analysis
|
12
|
+
|
13
|
+
Example:
|
14
|
+
```python
|
15
|
+
from ddeutil.workflow.api import app
|
16
|
+
|
17
|
+
# Run the API server
|
18
|
+
import uvicorn
|
19
|
+
uvicorn.run(app, host="0.0.0.0", port=8000)
|
20
|
+
```
|
21
|
+
|
22
|
+
Routes:
|
23
|
+
- /workflows: Workflow management endpoints
|
24
|
+
- /jobs: Job execution and monitoring
|
25
|
+
- /logs: Log access and streaming
|
26
|
+
"""
|
27
|
+
|
1
28
|
# ------------------------------------------------------------------------------
|
2
29
|
# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
|
3
30
|
# Licensed under the MIT License. See LICENSE in the project root for
|
@@ -28,7 +55,17 @@ logger = logging.getLogger("uvicorn.error")
|
|
28
55
|
|
29
56
|
@contextlib.asynccontextmanager
|
30
57
|
async def lifespan(_: FastAPI) -> AsyncIterator[dict[str, list]]:
|
31
|
-
"""
|
58
|
+
"""FastAPI application lifespan management.
|
59
|
+
|
60
|
+
Manages the startup and shutdown lifecycle of the FastAPI application.
|
61
|
+
Currently yields an empty dictionary for future extension.
|
62
|
+
|
63
|
+
Args:
|
64
|
+
_: FastAPI application instance (unused)
|
65
|
+
|
66
|
+
Yields:
|
67
|
+
dict: Empty dictionary for future lifespan data
|
68
|
+
"""
|
32
69
|
yield {}
|
33
70
|
|
34
71
|
|
@@ -59,7 +96,20 @@ app.add_middleware(
|
|
59
96
|
|
60
97
|
@app.get(path="/", response_class=UJSONResponse)
|
61
98
|
async def health() -> UJSONResponse:
|
62
|
-
"""
|
99
|
+
"""Health check endpoint for API status monitoring.
|
100
|
+
|
101
|
+
Provides a simple health check endpoint to verify the API is running
|
102
|
+
and responding correctly. Returns a JSON response with health status.
|
103
|
+
|
104
|
+
Returns:
|
105
|
+
UJSONResponse: JSON response confirming healthy API status
|
106
|
+
|
107
|
+
Example:
|
108
|
+
```bash
|
109
|
+
curl http://localhost:8000/
|
110
|
+
# Returns: {"message": "Workflow already start up with healthy status."}
|
111
|
+
```
|
112
|
+
"""
|
63
113
|
logger.info("[API]: Workflow API Application already running ...")
|
64
114
|
return UJSONResponse(
|
65
115
|
content={"message": "Workflow already start up with healthy status."},
|
@@ -78,7 +128,28 @@ async def validation_exception_handler(
|
|
78
128
|
request: Request,
|
79
129
|
exc: RequestValidationError,
|
80
130
|
) -> UJSONResponse:
|
81
|
-
"""
|
131
|
+
"""Handle request validation errors from Pydantic models.
|
132
|
+
|
133
|
+
Provides standardized error responses for request validation failures,
|
134
|
+
including detailed error information for debugging and client feedback.
|
135
|
+
|
136
|
+
Args:
|
137
|
+
request: The FastAPI request object (unused)
|
138
|
+
exc: The validation exception containing error details
|
139
|
+
|
140
|
+
Returns:
|
141
|
+
UJSONResponse: Standardized error response with validation details
|
142
|
+
|
143
|
+
Example:
|
144
|
+
When a request fails validation:
|
145
|
+
```json
|
146
|
+
{
|
147
|
+
"message": "Body does not parsing with model.",
|
148
|
+
"detail": [...],
|
149
|
+
"body": {...}
|
150
|
+
}
|
151
|
+
```
|
152
|
+
"""
|
82
153
|
_ = request
|
83
154
|
return UJSONResponse(
|
84
155
|
status_code=st.HTTP_422_UNPROCESSABLE_ENTITY,
|