ddeutil-workflow 0.0.15__tar.gz → 0.0.17__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ddeutil_workflow-0.0.15/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.17}/PKG-INFO +44 -25
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/README.md +41 -22
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/pyproject.toml +9 -7
- ddeutil_workflow-0.0.17/src/ddeutil/workflow/__about__.py +1 -0
- ddeutil_workflow-0.0.15/src/ddeutil/workflow/cron.py → ddeutil_workflow-0.0.17/src/ddeutil/workflow/__cron.py +12 -6
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/src/ddeutil/workflow/__init__.py +1 -0
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/src/ddeutil/workflow/__types.py +18 -6
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/src/ddeutil/workflow/api.py +3 -5
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/src/ddeutil/workflow/cli.py +2 -6
- ddeutil_workflow-0.0.17/src/ddeutil/workflow/conf.py +483 -0
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/src/ddeutil/workflow/job.py +119 -62
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/src/ddeutil/workflow/on.py +11 -8
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/src/ddeutil/workflow/repeat.py +2 -6
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/src/ddeutil/workflow/route.py +4 -12
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/src/ddeutil/workflow/scheduler.py +71 -54
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/src/ddeutil/workflow/stage.py +79 -43
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/src/ddeutil/workflow/utils.py +96 -283
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17/src/ddeutil_workflow.egg-info}/PKG-INFO +44 -25
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/src/ddeutil_workflow.egg-info/SOURCES.txt +8 -9
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/src/ddeutil_workflow.egg-info/requires.txt +2 -2
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/tests/test__conf_exist.py +1 -1
- ddeutil_workflow-0.0.15/tests/test_cron.py → ddeutil_workflow-0.0.17/tests/test__cron.py +35 -9
- ddeutil_workflow-0.0.17/tests/test_conf.py +100 -0
- ddeutil_workflow-0.0.17/tests/test_conf_log.py +74 -0
- ddeutil_workflow-0.0.17/tests/test_job.py +70 -0
- ddeutil_workflow-0.0.17/tests/test_job_py.py +141 -0
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/tests/test_job_strategy.py +18 -9
- ddeutil_workflow-0.0.17/tests/test_job_strategy_run.py +75 -0
- ddeutil_workflow-0.0.17/tests/test_on.py +190 -0
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/tests/test_scheduler.py +12 -7
- ddeutil_workflow-0.0.17/tests/test_stage.py +58 -0
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/tests/test_stage_bash.py +6 -12
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/tests/test_stage_hook.py +5 -15
- ddeutil_workflow-0.0.17/tests/test_stage_py.py +85 -0
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/tests/test_stage_trigger.py +0 -2
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/tests/test_utils.py +12 -12
- ddeutil_workflow-0.0.17/tests/test_utils_filter.py +80 -0
- ddeutil_workflow-0.0.17/tests/test_utils_params.py +56 -0
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/tests/test_utils_result.py +13 -18
- ddeutil_workflow-0.0.17/tests/test_utils_tag.py +42 -0
- ddeutil_workflow-0.0.17/tests/test_workflow.py +156 -0
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/tests/test_workflow_matrix.py +1 -4
- ddeutil_workflow-0.0.17/tests/test_workflow_params.py +23 -0
- ddeutil_workflow-0.0.15/tests/test_poke.py → ddeutil_workflow-0.0.17/tests/test_workflow_poke.py +6 -1
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/tests/test_workflow_run.py +13 -2
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/tests/test_workflow_task.py +1 -4
- ddeutil_workflow-0.0.15/src/ddeutil/workflow/__about__.py +0 -1
- ddeutil_workflow-0.0.15/src/ddeutil/workflow/conf.py +0 -45
- ddeutil_workflow-0.0.15/src/ddeutil/workflow/log.py +0 -198
- ddeutil_workflow-0.0.15/tests/test_conf.py +0 -8
- ddeutil_workflow-0.0.15/tests/test_job.py +0 -18
- ddeutil_workflow-0.0.15/tests/test_job_py.py +0 -47
- ddeutil_workflow-0.0.15/tests/test_log.py +0 -24
- ddeutil_workflow-0.0.15/tests/test_on.py +0 -67
- ddeutil_workflow-0.0.15/tests/test_stage.py +0 -44
- ddeutil_workflow-0.0.15/tests/test_stage_condition.py +0 -14
- ddeutil_workflow-0.0.15/tests/test_stage_py.py +0 -75
- ddeutil_workflow-0.0.15/tests/test_workflow.py +0 -48
- ddeutil_workflow-0.0.15/tests/test_workflow_desc.py +0 -11
- ddeutil_workflow-0.0.15/tests/test_workflow_if.py +0 -28
- ddeutil_workflow-0.0.15/tests/test_workflow_params.py +0 -12
- ddeutil_workflow-0.0.15/tests/test_workflow_run_raise.py +0 -12
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/LICENSE +0 -0
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/setup.cfg +0 -0
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/src/ddeutil/workflow/exceptions.py +0 -0
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/src/ddeutil_workflow.egg-info/dependency_links.txt +0 -0
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/src/ddeutil_workflow.egg-info/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/src/ddeutil_workflow.egg-info/top_level.txt +0 -0
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/tests/test__regex.py +0 -0
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/tests/test_params.py +0 -0
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/tests/test_utils_template.py +0 -0
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/tests/test_workflow_depends.py +0 -0
- {ddeutil_workflow-0.0.15 → ddeutil_workflow-0.0.17}/tests/test_workflow_on.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: ddeutil-workflow
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.17
|
4
4
|
Summary: Lightweight workflow orchestration with less dependencies
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
6
6
|
License: MIT
|
@@ -22,8 +22,8 @@ Classifier: Programming Language :: Python :: 3.13
|
|
22
22
|
Requires-Python: >=3.9.13
|
23
23
|
Description-Content-Type: text/markdown
|
24
24
|
License-File: LICENSE
|
25
|
-
Requires-Dist: ddeutil>=0.4.
|
26
|
-
Requires-Dist: ddeutil-io>=0.
|
25
|
+
Requires-Dist: ddeutil>=0.4.3
|
26
|
+
Requires-Dist: ddeutil-io[toml,yaml]>=0.2.3
|
27
27
|
Requires-Dist: python-dotenv==1.0.1
|
28
28
|
Requires-Dist: typer<1.0.0,==0.12.5
|
29
29
|
Requires-Dist: schedule<2.0.0,==1.2.2
|
@@ -33,6 +33,7 @@ Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "api"
|
|
33
33
|
# Workflow
|
34
34
|
|
35
35
|
[](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml)
|
36
|
+
[](https://codecov.io/gh/ddeutils/ddeutil-workflow)
|
36
37
|
[](https://pypi.org/project/ddeutil-workflow/)
|
37
38
|
[](https://pypi.org/project/ddeutil-workflow/)
|
38
39
|
[](https://github.com/ddeutils/ddeutil-workflow)
|
@@ -74,8 +75,9 @@ configuration. It called **Metadata Driven Data Workflow**.
|
|
74
75
|
|
75
76
|
## :round_pushpin: Installation
|
76
77
|
|
77
|
-
This project need `ddeutil-io` extension namespace packages.
|
78
|
-
this package with application add-ons, you should add
|
78
|
+
This project need `ddeutil` and `ddeutil-io` extension namespace packages.
|
79
|
+
If you want to install this package with application add-ons, you should add
|
80
|
+
`app` in installation;
|
79
81
|
|
80
82
|
| Usecase | Install Optional | Support |
|
81
83
|
|-------------------|------------------------------------------|--------------------|
|
@@ -179,29 +181,32 @@ The main configuration that use to dynamic changing with your propose of this
|
|
179
181
|
application. If any configuration values do not set yet, it will use default value
|
180
182
|
and do not raise any error to you.
|
181
183
|
|
182
|
-
| Environment
|
183
|
-
|
184
|
-
| `WORKFLOW_ROOT_PATH`
|
185
|
-
| `WORKFLOW_CORE_REGISTRY`
|
186
|
-
| `WORKFLOW_CORE_REGISTRY_FILTER`
|
187
|
-
| `WORKFLOW_CORE_PATH_CONF`
|
188
|
-
| `WORKFLOW_CORE_TIMEZONE`
|
189
|
-
| `WORKFLOW_CORE_STAGE_DEFAULT_ID`
|
190
|
-
| `WORKFLOW_CORE_STAGE_RAISE_ERROR`
|
191
|
-
| `
|
192
|
-
| `
|
193
|
-
| `
|
194
|
-
| `
|
195
|
-
| `
|
196
|
-
| `
|
197
|
-
| `
|
184
|
+
| Environment | Component | Default | Description | Remark |
|
185
|
+
|:----------------------------------------|-----------|----------------------------------|--------------------------------------------------------------------------------------------------------------------|--------|
|
186
|
+
| `WORKFLOW_ROOT_PATH` | Core | . | The root path of the workflow application. | |
|
187
|
+
| `WORKFLOW_CORE_REGISTRY` | Core | src.ddeutil.workflow,tests.utils | List of importable string for the hook stage. | |
|
188
|
+
| `WORKFLOW_CORE_REGISTRY_FILTER` | Core | ddeutil.workflow.utils | List of importable string for the filter template. | |
|
189
|
+
| `WORKFLOW_CORE_PATH_CONF` | Core | conf | The config path that keep all template `.yaml` files. | |
|
190
|
+
| `WORKFLOW_CORE_TIMEZONE` | Core | Asia/Bangkok | A Timezone string value that will pass to `ZoneInfo` object. | |
|
191
|
+
| `WORKFLOW_CORE_STAGE_DEFAULT_ID` | Core | true | A flag that enable default stage ID that use for catch an execution output. | |
|
192
|
+
| `WORKFLOW_CORE_STAGE_RAISE_ERROR` | Core | false | A flag that all stage raise StageException from stage execution. | |
|
193
|
+
| `WORKFLOW_CORE_JOB_DEFAULT_ID` | Core | false | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. | |
|
194
|
+
| `WORKFLOW_CORE_JOB_RAISE_ERROR` | Core | true | A flag that all job raise JobException from job strategy execution. | |
|
195
|
+
| `WORKFLOW_CORE_MAX_NUM_POKING` | Core | 4 | . | |
|
196
|
+
| `WORKFLOW_CORE_MAX_JOB_PARALLEL` | Core | 2 | The maximum job number that able to run parallel in workflow executor. | |
|
197
|
+
| `WORKFLOW_CORE_WORKFLOW_ID_SIMPLE_MODE` | Core | true | . | |
|
198
|
+
| `WORKFLOW_LOG_DEBUG_MODE` | Log | true | A flag that enable logging with debug level mode. | |
|
199
|
+
| `WORKFLOW_LOG_ENABLE_WRITE` | Log | true | A flag that enable logging object saving log to its destination. | |
|
200
|
+
| `WORKFLOW_APP_MAX_PROCESS` | Schedule | 2 | The maximum process worker number that run in scheduler app module. | |
|
201
|
+
| `WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS` | Schedule | 100 | A schedule per process that run parallel. | |
|
202
|
+
| `WORKFLOW_APP_STOP_BOUNDARY_DELTA` | Schedule | '{"minutes": 5, "seconds": 20}' | A time delta value that use to stop scheduler app in json string format. | |
|
198
203
|
|
199
204
|
**API Application**:
|
200
205
|
|
201
|
-
| Environment
|
202
|
-
|
203
|
-
| `WORKFLOW_API_ENABLE_ROUTE_WORKFLOW`
|
204
|
-
| `WORKFLOW_API_ENABLE_ROUTE_SCHEDULE`
|
206
|
+
| Environment | Component | Default | Description | Remark |
|
207
|
+
|:--------------------------------------|-----------|---------|------------------------------------------------------------------------------------|--------|
|
208
|
+
| `WORKFLOW_API_ENABLE_ROUTE_WORKFLOW` | API | true | A flag that enable workflow route to manage execute manually and workflow logging. | |
|
209
|
+
| `WORKFLOW_API_ENABLE_ROUTE_SCHEDULE` | API | true | A flag that enable run scheduler. | |
|
205
210
|
|
206
211
|
## :rocket: Deployment
|
207
212
|
|
@@ -224,3 +229,17 @@ like crontab job but via Python API.
|
|
224
229
|
> [!NOTE]
|
225
230
|
> If this package already deploy, it able to use
|
226
231
|
> `uvicorn ddeutil.workflow.api:app --host 127.0.0.1 --port 80 --workers 4`
|
232
|
+
|
233
|
+
### Docker Container
|
234
|
+
|
235
|
+
Create Docker image;
|
236
|
+
|
237
|
+
```shell
|
238
|
+
$ docker build -t ddeutil-workflow:latest -f .container/Dockerfile .
|
239
|
+
```
|
240
|
+
|
241
|
+
Run the above Docker image;
|
242
|
+
|
243
|
+
```shell
|
244
|
+
$ docker run -i ddeutil-workflow:latest
|
245
|
+
```
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# Workflow
|
2
2
|
|
3
3
|
[](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml)
|
4
|
+
[](https://codecov.io/gh/ddeutils/ddeutil-workflow)
|
4
5
|
[](https://pypi.org/project/ddeutil-workflow/)
|
5
6
|
[](https://pypi.org/project/ddeutil-workflow/)
|
6
7
|
[](https://github.com/ddeutils/ddeutil-workflow)
|
@@ -42,8 +43,9 @@ configuration. It called **Metadata Driven Data Workflow**.
|
|
42
43
|
|
43
44
|
## :round_pushpin: Installation
|
44
45
|
|
45
|
-
This project need `ddeutil-io` extension namespace packages.
|
46
|
-
this package with application add-ons, you should add
|
46
|
+
This project need `ddeutil` and `ddeutil-io` extension namespace packages.
|
47
|
+
If you want to install this package with application add-ons, you should add
|
48
|
+
`app` in installation;
|
47
49
|
|
48
50
|
| Usecase | Install Optional | Support |
|
49
51
|
|-------------------|------------------------------------------|--------------------|
|
@@ -147,29 +149,32 @@ The main configuration that use to dynamic changing with your propose of this
|
|
147
149
|
application. If any configuration values do not set yet, it will use default value
|
148
150
|
and do not raise any error to you.
|
149
151
|
|
150
|
-
| Environment
|
151
|
-
|
152
|
-
| `WORKFLOW_ROOT_PATH`
|
153
|
-
| `WORKFLOW_CORE_REGISTRY`
|
154
|
-
| `WORKFLOW_CORE_REGISTRY_FILTER`
|
155
|
-
| `WORKFLOW_CORE_PATH_CONF`
|
156
|
-
| `WORKFLOW_CORE_TIMEZONE`
|
157
|
-
| `WORKFLOW_CORE_STAGE_DEFAULT_ID`
|
158
|
-
| `WORKFLOW_CORE_STAGE_RAISE_ERROR`
|
159
|
-
| `
|
160
|
-
| `
|
161
|
-
| `
|
162
|
-
| `
|
163
|
-
| `
|
164
|
-
| `
|
165
|
-
| `
|
152
|
+
| Environment | Component | Default | Description | Remark |
|
153
|
+
|:----------------------------------------|-----------|----------------------------------|--------------------------------------------------------------------------------------------------------------------|--------|
|
154
|
+
| `WORKFLOW_ROOT_PATH` | Core | . | The root path of the workflow application. | |
|
155
|
+
| `WORKFLOW_CORE_REGISTRY` | Core | src.ddeutil.workflow,tests.utils | List of importable string for the hook stage. | |
|
156
|
+
| `WORKFLOW_CORE_REGISTRY_FILTER` | Core | ddeutil.workflow.utils | List of importable string for the filter template. | |
|
157
|
+
| `WORKFLOW_CORE_PATH_CONF` | Core | conf | The config path that keep all template `.yaml` files. | |
|
158
|
+
| `WORKFLOW_CORE_TIMEZONE` | Core | Asia/Bangkok | A Timezone string value that will pass to `ZoneInfo` object. | |
|
159
|
+
| `WORKFLOW_CORE_STAGE_DEFAULT_ID` | Core | true | A flag that enable default stage ID that use for catch an execution output. | |
|
160
|
+
| `WORKFLOW_CORE_STAGE_RAISE_ERROR` | Core | false | A flag that all stage raise StageException from stage execution. | |
|
161
|
+
| `WORKFLOW_CORE_JOB_DEFAULT_ID` | Core | false | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. | |
|
162
|
+
| `WORKFLOW_CORE_JOB_RAISE_ERROR` | Core | true | A flag that all job raise JobException from job strategy execution. | |
|
163
|
+
| `WORKFLOW_CORE_MAX_NUM_POKING` | Core | 4 | . | |
|
164
|
+
| `WORKFLOW_CORE_MAX_JOB_PARALLEL` | Core | 2 | The maximum job number that able to run parallel in workflow executor. | |
|
165
|
+
| `WORKFLOW_CORE_WORKFLOW_ID_SIMPLE_MODE` | Core | true | . | |
|
166
|
+
| `WORKFLOW_LOG_DEBUG_MODE` | Log | true | A flag that enable logging with debug level mode. | |
|
167
|
+
| `WORKFLOW_LOG_ENABLE_WRITE` | Log | true | A flag that enable logging object saving log to its destination. | |
|
168
|
+
| `WORKFLOW_APP_MAX_PROCESS` | Schedule | 2 | The maximum process worker number that run in scheduler app module. | |
|
169
|
+
| `WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS` | Schedule | 100 | A schedule per process that run parallel. | |
|
170
|
+
| `WORKFLOW_APP_STOP_BOUNDARY_DELTA` | Schedule | '{"minutes": 5, "seconds": 20}' | A time delta value that use to stop scheduler app in json string format. | |
|
166
171
|
|
167
172
|
**API Application**:
|
168
173
|
|
169
|
-
| Environment
|
170
|
-
|
171
|
-
| `WORKFLOW_API_ENABLE_ROUTE_WORKFLOW`
|
172
|
-
| `WORKFLOW_API_ENABLE_ROUTE_SCHEDULE`
|
174
|
+
| Environment | Component | Default | Description | Remark |
|
175
|
+
|:--------------------------------------|-----------|---------|------------------------------------------------------------------------------------|--------|
|
176
|
+
| `WORKFLOW_API_ENABLE_ROUTE_WORKFLOW` | API | true | A flag that enable workflow route to manage execute manually and workflow logging. | |
|
177
|
+
| `WORKFLOW_API_ENABLE_ROUTE_SCHEDULE` | API | true | A flag that enable run scheduler. | |
|
173
178
|
|
174
179
|
## :rocket: Deployment
|
175
180
|
|
@@ -192,3 +197,17 @@ like crontab job but via Python API.
|
|
192
197
|
> [!NOTE]
|
193
198
|
> If this package already deploy, it able to use
|
194
199
|
> `uvicorn ddeutil.workflow.api:app --host 127.0.0.1 --port 80 --workers 4`
|
200
|
+
|
201
|
+
### Docker Container
|
202
|
+
|
203
|
+
Create Docker image;
|
204
|
+
|
205
|
+
```shell
|
206
|
+
$ docker build -t ddeutil-workflow:latest -f .container/Dockerfile .
|
207
|
+
```
|
208
|
+
|
209
|
+
Run the above Docker image;
|
210
|
+
|
211
|
+
```shell
|
212
|
+
$ docker run -i ddeutil-workflow:latest
|
213
|
+
```
|
@@ -26,8 +26,8 @@ classifiers = [
|
|
26
26
|
]
|
27
27
|
requires-python = ">=3.9.13"
|
28
28
|
dependencies = [
|
29
|
-
"ddeutil>=0.4.
|
30
|
-
"ddeutil-io>=0.
|
29
|
+
"ddeutil>=0.4.3",
|
30
|
+
"ddeutil-io[yaml,toml]>=0.2.3",
|
31
31
|
"python-dotenv==1.0.1",
|
32
32
|
"typer==0.12.5,<1.0.0",
|
33
33
|
"schedule==1.2.2,<2.0.0",
|
@@ -60,20 +60,22 @@ changelog = "CHANGELOG.md"
|
|
60
60
|
branch = true
|
61
61
|
relative_files = true
|
62
62
|
concurrency = ["thread", "multiprocessing"]
|
63
|
-
source = ["ddeutil.workflow"
|
63
|
+
source = ["ddeutil.workflow"]
|
64
64
|
omit = [
|
65
65
|
"src/ddeutil/workflow/__about__.py",
|
66
|
-
|
66
|
+
"src/ddeutil/workflow/__cron.py",
|
67
|
+
# NOTE: Omit this files because it does not ready to production.
|
67
68
|
"src/ddeutil/workflow/api.py",
|
68
69
|
"src/ddeutil/workflow/cli.py",
|
69
70
|
"src/ddeutil/workflow/repeat.py",
|
70
71
|
"src/ddeutil/workflow/route.py",
|
71
|
-
"tests/utils.py",
|
72
|
-
"tests/tasks/dummy.py",
|
73
72
|
]
|
74
73
|
|
75
74
|
[tool.coverage.report]
|
76
|
-
exclude_lines = [
|
75
|
+
exclude_lines = [
|
76
|
+
"raise NotImplementedError",
|
77
|
+
"pragma: no cov",
|
78
|
+
]
|
77
79
|
|
78
80
|
[tool.pytest.ini_options]
|
79
81
|
pythonpath = ["src"]
|
@@ -0,0 +1 @@
|
|
1
|
+
__version__: str = "0.0.17"
|
@@ -18,10 +18,7 @@ from ddeutil.core import (
|
|
18
18
|
isinstance_check,
|
19
19
|
must_split,
|
20
20
|
)
|
21
|
-
from ddeutil.core.dtutils import
|
22
|
-
next_date,
|
23
|
-
replace_date,
|
24
|
-
)
|
21
|
+
from ddeutil.core.dtutils import next_date, replace_date
|
25
22
|
|
26
23
|
WEEKDAYS: dict[str, int] = {
|
27
24
|
"Sun": 0,
|
@@ -37,7 +34,7 @@ WEEKDAYS: dict[str, int] = {
|
|
37
34
|
class CronYearLimit(Exception): ...
|
38
35
|
|
39
36
|
|
40
|
-
def str2cron(value: str) -> str:
|
37
|
+
def str2cron(value: str) -> str: # pragma: no cov
|
41
38
|
"""Convert Special String with the @ prefix to Crontab value.
|
42
39
|
|
43
40
|
:param value: A string value that want to convert to cron value.
|
@@ -69,6 +66,8 @@ def str2cron(value: str) -> str:
|
|
69
66
|
|
70
67
|
@dataclass(frozen=True)
|
71
68
|
class Unit:
|
69
|
+
"""Unit dataclass for CronPart object."""
|
70
|
+
|
72
71
|
name: str
|
73
72
|
range: partial
|
74
73
|
min: int
|
@@ -85,6 +84,8 @@ class Unit:
|
|
85
84
|
|
86
85
|
@dataclass
|
87
86
|
class Options:
|
87
|
+
"""Options dataclass for config CronPart object."""
|
88
|
+
|
88
89
|
output_weekday_names: bool = False
|
89
90
|
output_month_names: bool = False
|
90
91
|
output_hashes: bool = False
|
@@ -158,7 +159,12 @@ CRON_UNITS_YEAR: tuple[Unit, ...] = CRON_UNITS + (
|
|
158
159
|
|
159
160
|
@total_ordering
|
160
161
|
class CronPart:
|
161
|
-
"""Part of Cron object that represent a collection of positive integers.
|
162
|
+
"""Part of Cron object that represent a collection of positive integers.
|
163
|
+
|
164
|
+
:param unit: A Unit dataclass object.
|
165
|
+
:param values: A crontab values that want to validate
|
166
|
+
:param options: A Options dataclass object.
|
167
|
+
"""
|
162
168
|
|
163
169
|
__slots__: tuple[str, ...] = (
|
164
170
|
"unit",
|
@@ -3,6 +3,7 @@
|
|
3
3
|
# Licensed under the MIT License. See LICENSE in the project root for
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
|
+
from .conf import Config, FileLog, Loader
|
6
7
|
from .exceptions import (
|
7
8
|
JobException,
|
8
9
|
ParamValueException,
|
@@ -16,7 +16,7 @@ from re import (
|
|
16
16
|
Match,
|
17
17
|
Pattern,
|
18
18
|
)
|
19
|
-
from typing import Any, Optional, Union
|
19
|
+
from typing import Any, Optional, TypedDict, Union
|
20
20
|
|
21
21
|
from typing_extensions import Self
|
22
22
|
|
@@ -24,8 +24,11 @@ TupleStr = tuple[str, ...]
|
|
24
24
|
DictData = dict[str, Any]
|
25
25
|
DictStr = dict[str, str]
|
26
26
|
Matrix = dict[str, Union[list[str], list[int]]]
|
27
|
-
|
28
|
-
|
27
|
+
|
28
|
+
|
29
|
+
class Context(TypedDict):
|
30
|
+
params: dict[str, Any]
|
31
|
+
jobs: dict[str, Any]
|
29
32
|
|
30
33
|
|
31
34
|
@dataclass(frozen=True)
|
@@ -56,20 +59,24 @@ class Re:
|
|
56
59
|
# Regular expression:
|
57
60
|
# - Version 1:
|
58
61
|
# \${{\s*(?P<caller>[a-zA-Z0-9_.\s'\"\[\]\(\)\-\{}]+?)\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
|
59
|
-
# - Version 2 (2024-09-30):
|
62
|
+
# - Version 2: (2024-09-30):
|
60
63
|
# \${{\s*(?P<caller>(?P<caller_prefix>(?:[a-zA-Z_-]+\.)*)(?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+))\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
|
64
|
+
# - Version 3: (2024-10-05):
|
65
|
+
# \${{\s*(?P<caller>(?P<caller_prefix>(?:[a-zA-Z_-]+\??\.)*)(?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+\??))\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
|
61
66
|
#
|
62
67
|
# Examples:
|
63
68
|
# - ${{ params.data_dt }}
|
64
69
|
# - ${{ params.source.table }}
|
70
|
+
# - ${{ params.datetime | fmt('%Y-%m-%d') }}
|
71
|
+
# - ${{ params.source?.schema }}
|
65
72
|
#
|
66
73
|
__re_caller: str = r"""
|
67
74
|
\$
|
68
75
|
{{
|
69
76
|
\s*
|
70
77
|
(?P<caller>
|
71
|
-
(?P<caller_prefix>(?:[a-zA-Z_-]
|
72
|
-
(?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]
|
78
|
+
(?P<caller_prefix>(?:[a-zA-Z_-]+\??\.)*)
|
79
|
+
(?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+\??)
|
73
80
|
)
|
74
81
|
\s*
|
75
82
|
(?P<post_filters>
|
@@ -109,5 +116,10 @@ class Re:
|
|
109
116
|
|
110
117
|
@classmethod
|
111
118
|
def finditer_caller(cls, value) -> Iterator[CallerRe]:
|
119
|
+
"""Generate CallerRe object that create from matching object that
|
120
|
+
extract with re.finditer function.
|
121
|
+
|
122
|
+
:rtype: Iterator[CallerRe]
|
123
|
+
"""
|
112
124
|
for found in cls.RE_CALLER.finditer(value):
|
113
125
|
yield CallerRe.from_regex(found)
|
@@ -7,7 +7,6 @@ from __future__ import annotations
|
|
7
7
|
|
8
8
|
import asyncio
|
9
9
|
import contextlib
|
10
|
-
import os
|
11
10
|
import uuid
|
12
11
|
from collections.abc import AsyncIterator
|
13
12
|
from datetime import datetime, timedelta
|
@@ -15,7 +14,6 @@ from queue import Empty, Queue
|
|
15
14
|
from threading import Thread
|
16
15
|
from typing import TypedDict
|
17
16
|
|
18
|
-
from ddeutil.core import str2bool
|
19
17
|
from dotenv import load_dotenv
|
20
18
|
from fastapi import FastAPI
|
21
19
|
from fastapi.middleware.gzip import GZipMiddleware
|
@@ -23,7 +21,7 @@ from fastapi.responses import UJSONResponse
|
|
23
21
|
from pydantic import BaseModel
|
24
22
|
|
25
23
|
from .__about__ import __version__
|
26
|
-
from .
|
24
|
+
from .conf import config, get_logger
|
27
25
|
from .repeat import repeat_at, repeat_every
|
28
26
|
from .scheduler import WorkflowTaskData
|
29
27
|
|
@@ -131,12 +129,12 @@ async def message_upper(payload: Payload):
|
|
131
129
|
return await get_result(request_id)
|
132
130
|
|
133
131
|
|
134
|
-
if
|
132
|
+
if config.enable_route_workflow:
|
135
133
|
from .route import workflow
|
136
134
|
|
137
135
|
app.include_router(workflow)
|
138
136
|
|
139
|
-
if
|
137
|
+
if config.enable_route_schedule:
|
140
138
|
from .route import schedule
|
141
139
|
from .scheduler import workflow_task
|
142
140
|
|
@@ -6,16 +6,14 @@
|
|
6
6
|
from __future__ import annotations
|
7
7
|
|
8
8
|
import json
|
9
|
-
import os
|
10
9
|
from datetime import datetime
|
11
10
|
from enum import Enum
|
12
11
|
from typing import Annotated, Optional
|
13
|
-
from zoneinfo import ZoneInfo
|
14
12
|
|
15
13
|
from ddeutil.core import str2list
|
16
14
|
from typer import Argument, Option, Typer
|
17
15
|
|
18
|
-
from .
|
16
|
+
from .conf import config, get_logger
|
19
17
|
|
20
18
|
logger = get_logger("ddeutil.workflow")
|
21
19
|
cli: Typer = Typer()
|
@@ -73,9 +71,7 @@ def schedule(
|
|
73
71
|
excluded: list[str] = str2list(excluded) if excluded else []
|
74
72
|
externals: str = externals or "{}"
|
75
73
|
if stop:
|
76
|
-
stop: datetime = stop.astimezone(
|
77
|
-
tz=ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
|
78
|
-
)
|
74
|
+
stop: datetime = stop.astimezone(tz=config.tz)
|
79
75
|
|
80
76
|
from .scheduler import workflow_runner
|
81
77
|
|