ddeutil-workflow 0.0.26.post1__py3-none-any.whl → 0.0.28__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +19 -14
- ddeutil/workflow/api/api.py +1 -53
- ddeutil/workflow/conf.py +44 -23
- ddeutil/workflow/cron.py +7 -7
- ddeutil/workflow/exceptions.py +1 -4
- ddeutil/workflow/hook.py +168 -0
- ddeutil/workflow/job.py +18 -17
- ddeutil/workflow/params.py +3 -3
- ddeutil/workflow/result.py +3 -3
- ddeutil/workflow/scheduler.py +9 -9
- ddeutil/workflow/stage.py +87 -170
- ddeutil/workflow/templates.py +336 -0
- ddeutil/workflow/utils.py +23 -404
- ddeutil/workflow/workflow.py +22 -23
- ddeutil_workflow-0.0.28.dist-info/METADATA +284 -0
- ddeutil_workflow-0.0.28.dist-info/RECORD +25 -0
- {ddeutil_workflow-0.0.26.post1.dist-info → ddeutil_workflow-0.0.28.dist-info}/WHEEL +1 -1
- ddeutil_workflow-0.0.26.post1.dist-info/METADATA +0 -230
- ddeutil_workflow-0.0.26.post1.dist-info/RECORD +0 -23
- {ddeutil_workflow-0.0.26.post1.dist-info → ddeutil_workflow-0.0.28.dist-info}/LICENSE +0 -0
- {ddeutil_workflow-0.0.26.post1.dist-info → ddeutil_workflow-0.0.28.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: ddeutil-workflow
|
|
3
|
+
Version: 0.0.28
|
|
4
|
+
Summary: Lightweight workflow orchestration
|
|
5
|
+
Author-email: ddeutils <korawich.anu@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ddeutils/ddeutil-workflow/
|
|
8
|
+
Project-URL: Source Code, https://github.com/ddeutils/ddeutil-workflow/
|
|
9
|
+
Keywords: orchestration,workflow
|
|
10
|
+
Classifier: Topic :: Utilities
|
|
11
|
+
Classifier: Natural Language :: English
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Requires-Python: >=3.9.13
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: ddeutil==0.4.6
|
|
26
|
+
Requires-Dist: ddeutil-io[toml,yaml]>=0.2.3
|
|
27
|
+
Requires-Dist: pydantic==2.10.6
|
|
28
|
+
Requires-Dist: python-dotenv==1.0.1
|
|
29
|
+
Requires-Dist: schedule<2.0.0,==1.2.2
|
|
30
|
+
Provides-Extra: api
|
|
31
|
+
Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "api"
|
|
32
|
+
|
|
33
|
+
# Workflow Orchestration
|
|
34
|
+
|
|
35
|
+
[](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml)
|
|
36
|
+
[](https://codecov.io/gh/ddeutils/ddeutil-workflow)
|
|
37
|
+
[](https://pypi.org/project/ddeutil-workflow/)
|
|
38
|
+
[](https://pypi.org/project/ddeutil-workflow/)
|
|
39
|
+
[](https://github.com/ddeutils/ddeutil-workflow)
|
|
40
|
+
[](https://github.com/ddeutils/ddeutil-workflow/blob/main/LICENSE)
|
|
41
|
+
[](https://github.com/psf/black)
|
|
42
|
+
|
|
43
|
+
The **Lightweight Workflow Orchestration** with fewer dependencies the was created
|
|
44
|
+
for easy to make a simple metadata driven data workflow. It can use for data operator
|
|
45
|
+
by a `.yaml` template.
|
|
46
|
+
|
|
47
|
+
> [!WARNING]
|
|
48
|
+
> This package provide only orchestration workload. That mean you should not
|
|
49
|
+
> use the workflow stage to process any large volume data which use a lot of compute
|
|
50
|
+
> resource :cold_sweat:.
|
|
51
|
+
|
|
52
|
+
In my opinion, I think it should not create duplicate workflow codes if I can
|
|
53
|
+
write with dynamic input parameters on the one template workflow that just change
|
|
54
|
+
the input parameters per use-case instead.
|
|
55
|
+
This way I can handle a lot of logical workflows in our orgs with only metadata
|
|
56
|
+
configuration. It called **Metadata Driven Data Workflow**.
|
|
57
|
+
|
|
58
|
+
**:pushpin: <u>Rules of This Workflow engine</u>**:
|
|
59
|
+
|
|
60
|
+
1. The Minimum frequency unit of scheduling is **1 minute** :warning:
|
|
61
|
+
2. Can not re-run only failed stage and its pending downstream :rotating_light:
|
|
62
|
+
3. All parallel tasks inside workflow engine use Multi-Threading
|
|
63
|
+
(Python 3.13 unlock GIL :unlock:)
|
|
64
|
+
|
|
65
|
+
**:memo: <u>Workflow Diagrams</u>**:
|
|
66
|
+
|
|
67
|
+
```mermaid
|
|
68
|
+
flowchart LR
|
|
69
|
+
subgraph Interface
|
|
70
|
+
A((User))
|
|
71
|
+
subgraph Docker Container
|
|
72
|
+
G@{ shape: rounded, label: "Observe<br>Application" }
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
A --->|action| B(Workflow<br>Application)
|
|
77
|
+
B ---> |response| A
|
|
78
|
+
B -..-> |response| G
|
|
79
|
+
G -..-> |request| B
|
|
80
|
+
|
|
81
|
+
subgraph Docker Container
|
|
82
|
+
B
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
subgraph Data Context
|
|
86
|
+
D@{ shape: processes, label: "Logs" }
|
|
87
|
+
E@{ shape: lin-cyl, label: "Metadata" }
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
subgraph Git Context
|
|
91
|
+
F@{ shape: tag-rect, label: "YAML<br>files" }
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
B --->|disable| F
|
|
95
|
+
F --->|read| B
|
|
96
|
+
|
|
97
|
+
B --->|write| E
|
|
98
|
+
E --->|read| B
|
|
99
|
+
B --->|write| D
|
|
100
|
+
|
|
101
|
+
D -.->|read| G
|
|
102
|
+
E -.->|read| G
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
> [!NOTE]
|
|
106
|
+
> _Disclaimer_: I inspire the dynamic statement from the [**GitHub Action**](https://github.com/features/actions)
|
|
107
|
+
> with `.yml` files and all configs file from several data orchestration framework
|
|
108
|
+
> tools from my experience on Data Engineer. :grimacing:
|
|
109
|
+
>
|
|
110
|
+
> Other workflow tools that I interest on them and pick some interested feature
|
|
111
|
+
> implement to this package:
|
|
112
|
+
>
|
|
113
|
+
> - [Google **Workflows**](https://cloud.google.com/workflows)
|
|
114
|
+
> - [AWS **Step Functions**](https://aws.amazon.com/step-functions/)
|
|
115
|
+
|
|
116
|
+
## :round_pushpin: Installation
|
|
117
|
+
|
|
118
|
+
This project need `ddeutil` and `ddeutil-io` extension namespace packages.
|
|
119
|
+
If you want to install this package with application add-ons, you should add
|
|
120
|
+
`app` in installation;
|
|
121
|
+
|
|
122
|
+
| Use-case | Install Optional | Support |
|
|
123
|
+
|----------------|--------------------------|--------------------|
|
|
124
|
+
| Python | `ddeutil-workflow` | :heavy_check_mark: |
|
|
125
|
+
| FastAPI Server | `ddeutil-workflow[api]` | :heavy_check_mark: |
|
|
126
|
+
|
|
127
|
+
## :beers: Usage
|
|
128
|
+
|
|
129
|
+
This is examples that use workflow file for running common Data Engineering
|
|
130
|
+
use-case.
|
|
131
|
+
|
|
132
|
+
> [!IMPORTANT]
|
|
133
|
+
> I recommend you to use the `hook` stage for all actions that you want to do
|
|
134
|
+
> with workflow activity that you want to orchestrate. Because it is able to
|
|
135
|
+
> dynamic an input argument with the same hook function that make you use less
|
|
136
|
+
> time to maintenance your data workflows.
|
|
137
|
+
|
|
138
|
+
```yaml
|
|
139
|
+
run-py-local:
|
|
140
|
+
|
|
141
|
+
# Validate model that use to parsing exists for template file
|
|
142
|
+
type: Workflow
|
|
143
|
+
on:
|
|
144
|
+
# If workflow deploy to schedule, it will run every 5 minutes
|
|
145
|
+
# with Asia/Bangkok timezone.
|
|
146
|
+
- cronjob: '*/5 * * * *'
|
|
147
|
+
timezone: "Asia/Bangkok"
|
|
148
|
+
params:
|
|
149
|
+
# Incoming execution parameters will validate with this type. It allows
|
|
150
|
+
# to set default value or templating.
|
|
151
|
+
source-extract: str
|
|
152
|
+
run-date: datetime
|
|
153
|
+
jobs:
|
|
154
|
+
getting-api-data:
|
|
155
|
+
stages:
|
|
156
|
+
- name: "Retrieve API Data"
|
|
157
|
+
id: retrieve-api
|
|
158
|
+
uses: tasks/get-api-with-oauth-to-s3@requests
|
|
159
|
+
with:
|
|
160
|
+
# Arguments of source data that want to retrieve.
|
|
161
|
+
method: post
|
|
162
|
+
url: https://finances/open-data/currency-pairs/
|
|
163
|
+
body:
|
|
164
|
+
resource: ${{ params.source-extract }}
|
|
165
|
+
|
|
166
|
+
# You can use filtering like Jinja template but this
|
|
167
|
+
# package does not use it.
|
|
168
|
+
filter: ${{ params.run-date | fmt(fmt='%Y%m%d') }}
|
|
169
|
+
auth:
|
|
170
|
+
type: bearer
|
|
171
|
+
keys: ${API_ACCESS_REFRESH_TOKEN}
|
|
172
|
+
|
|
173
|
+
# Arguments of target data that want to land.
|
|
174
|
+
writing_mode: flatten
|
|
175
|
+
aws_s3_path: my-data/open-data/${{ params.source-extract }}
|
|
176
|
+
|
|
177
|
+
# This Authentication code should implement with your custom hook
|
|
178
|
+
# function. The template allow you to use environment variable.
|
|
179
|
+
aws_access_client_id: ${AWS_ACCESS_CLIENT_ID}
|
|
180
|
+
aws_access_client_secret: ${AWS_ACCESS_CLIENT_SECRET}
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
The above workflow template is main executor pipeline that you want to do. If you
|
|
184
|
+
want to schedule this workflow, you want to dynamic its parameters change base on
|
|
185
|
+
execution time such as `run-date` should change base on that workflow running date.
|
|
186
|
+
|
|
187
|
+
So, this package provide the `Schedule` template for this action.
|
|
188
|
+
|
|
189
|
+
```yaml
|
|
190
|
+
schedule-run-local-wf:
|
|
191
|
+
|
|
192
|
+
# Validate model that use to parsing exists for template file
|
|
193
|
+
type: Schedule
|
|
194
|
+
workflows:
|
|
195
|
+
|
|
196
|
+
# Map existing workflow that want to deploy with scheduler application.
|
|
197
|
+
# It allows you to pass release parameter that dynamic change depend on the
|
|
198
|
+
# current context of this scheduler application releasing that time.
|
|
199
|
+
- name: run-py-local
|
|
200
|
+
params:
|
|
201
|
+
source-extract: "USD-THB"
|
|
202
|
+
asat-dt: "${{ release.logical_date }}"
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
## :cookie: Configuration
|
|
206
|
+
|
|
207
|
+
The main configuration that use to dynamic changing with your objective of this
|
|
208
|
+
application. If any configuration values do not set yet, it will use default value
|
|
209
|
+
and do not raise any error to you.
|
|
210
|
+
|
|
211
|
+
> [!IMPORTANT]
|
|
212
|
+
> The config value that you will set on the environment should combine with
|
|
213
|
+
> prefix, component, and name which is `WORKFLOW_{component}_{name}` (Upper case).
|
|
214
|
+
|
|
215
|
+
| Name | Component | Default | Description |
|
|
216
|
+
|:-----------------------------|:---------:|:----------------------------------|:-------------------------------------------------------------------------------------------------------------------|
|
|
217
|
+
| **ROOT_PATH** | Core | `.` | The root path of the workflow application. |
|
|
218
|
+
| **REGISTRY** | Core | `src` | List of importable string for the hook stage. |
|
|
219
|
+
| **REGISTRY_FILTER** | Core | `ddeutil.workflow.utils` | List of importable string for the filter template. |
|
|
220
|
+
| **CONF_PATH** | Core | `conf` | The config path that keep all template `.yaml` files. |
|
|
221
|
+
| **TIMEZONE** | Core | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
|
222
|
+
| **STAGE_DEFAULT_ID** | Core | `true` | A flag that enable default stage ID that use for catch an execution output. |
|
|
223
|
+
| **STAGE_RAISE_ERROR** | Core | `false` | A flag that all stage raise StageException from stage execution. |
|
|
224
|
+
| **JOB_DEFAULT_ID** | Core | `false` | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
|
|
225
|
+
| **JOB_RAISE_ERROR** | Core | `true` | A flag that all job raise JobException from job strategy execution. |
|
|
226
|
+
| **MAX_NUM_POKING** | Core | `4` | . |
|
|
227
|
+
| **MAX_JOB_PARALLEL** | Core | `2` | The maximum job number that able to run parallel in workflow executor. |
|
|
228
|
+
| **MAX_JOB_EXEC_TIMEOUT** | Core | `600` | |
|
|
229
|
+
| **MAX_CRON_PER_WORKFLOW** | Core | `5` | |
|
|
230
|
+
| **MAX_QUEUE_COMPLETE_HIST** | Core | `16` | |
|
|
231
|
+
| **GENERATE_ID_SIMPLE_MODE** | Core | `true` | A flog that enable generating ID with `md5` algorithm. |
|
|
232
|
+
| **PATH** | Log | `./logs` | The log path of the workflow saving log. |
|
|
233
|
+
| **DEBUG_MODE** | Log | `true` | A flag that enable logging with debug level mode. |
|
|
234
|
+
| **ENABLE_WRITE** | Log | `true` | A flag that enable logging object saving log to its destination. |
|
|
235
|
+
| **MAX_PROCESS** | App | `2` | The maximum process worker number that run in scheduler app module. |
|
|
236
|
+
| **MAX_SCHEDULE_PER_PROCESS** | App | `100` | A schedule per process that run parallel. |
|
|
237
|
+
| **STOP_BOUNDARY_DELTA** | App | `'{"minutes": 5, "seconds": 20}'` | A time delta value that use to stop scheduler app in json string format. |
|
|
238
|
+
|
|
239
|
+
**API Application**:
|
|
240
|
+
|
|
241
|
+
| Environment | Component | Default | Description |
|
|
242
|
+
|:---------------------------|:-----------:|---------|------------------------------------------------------------------------------------|
|
|
243
|
+
| **ENABLE_ROUTE_WORKFLOW** | API | `true` | A flag that enable workflow route to manage execute manually and workflow logging. |
|
|
244
|
+
| **ENABLE_ROUTE_SCHEDULE** | API | `true` | A flag that enable run scheduler. |
|
|
245
|
+
|
|
246
|
+
## :rocket: Deployment
|
|
247
|
+
|
|
248
|
+
This package able to run as an application service for receive manual trigger
|
|
249
|
+
from the master node via RestAPI or use to be Scheduler background service
|
|
250
|
+
like crontab job but via Python API.
|
|
251
|
+
|
|
252
|
+
### API Server
|
|
253
|
+
|
|
254
|
+
```shell
|
|
255
|
+
(venv) $ uvicorn src.ddeutil.workflow.api:app \
|
|
256
|
+
--host 127.0.0.1 \
|
|
257
|
+
--port 80 \
|
|
258
|
+
--no-access-log
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
> [!NOTE]
|
|
262
|
+
> If this package already deploy, it is able to use multiprocess;
|
|
263
|
+
> `uvicorn ddeutil.workflow.api:app --host 127.0.0.1 --port 80 --workers 4`
|
|
264
|
+
|
|
265
|
+
### Docker Container
|
|
266
|
+
|
|
267
|
+
Create Docker image;
|
|
268
|
+
|
|
269
|
+
```shell
|
|
270
|
+
$ docker build -t ddeutil-workflow:latest -f .container/Dockerfile .
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
Run the above Docker image;
|
|
274
|
+
|
|
275
|
+
```shell
|
|
276
|
+
$ docker run -i ddeutil-workflow:latest
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
## :speech_balloon: Contribute
|
|
280
|
+
|
|
281
|
+
I do not think this project will go around the world because it has specific propose,
|
|
282
|
+
and you can create by your coding without this project dependency for long term
|
|
283
|
+
solution. So, on this time, you can open [the GitHub issue on this project :raised_hands:](https://github.com/ddeutils/ddeutil-workflow/issues)
|
|
284
|
+
for fix bug or request new feature if you want it.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
ddeutil/workflow/__about__.py,sha256=Fb6dv9Oq3jpI1oIsn3w_zS2izPiywO6k7tph7IMZ9mA,28
|
|
2
|
+
ddeutil/workflow/__cron.py,sha256=uA8XcbY_GwA9rJSHaHUaXaJyGDObJN0ZeYlJSinL8y8,26880
|
|
3
|
+
ddeutil/workflow/__init__.py,sha256=ATYXzGtLyq4LWCtJ-Odz36QSrLL7dKymVs8ziThOVOk,1582
|
|
4
|
+
ddeutil/workflow/__types.py,sha256=Ia7f38kvL3NibwmRKi0wQ1ud_45Z-SojYGhNJwIqcu8,3713
|
|
5
|
+
ddeutil/workflow/conf.py,sha256=eoge4fNy2bb5qIunyVDDxSSp2l9rDKYPiqdxSuYtKcs,16954
|
|
6
|
+
ddeutil/workflow/cron.py,sha256=OLgniUxmrn65gzckk-uTmE2Pk1enJJyjYUKVeBbDQz0,7522
|
|
7
|
+
ddeutil/workflow/exceptions.py,sha256=XUnpJSuxOyataClP0w_gpYjzn-NIwZK2BHro-J7Yw24,895
|
|
8
|
+
ddeutil/workflow/hook.py,sha256=--4obgSCEImeRASSWIx-es3BctNmN3RVdhfKQ3O5AVk,5353
|
|
9
|
+
ddeutil/workflow/job.py,sha256=XcewyALsLYYq94ycF6mkj3Ydr6if683z7t1oBqEVInE,24290
|
|
10
|
+
ddeutil/workflow/params.py,sha256=svCjmFgEhim8yFJVjZhFmKP8JqTDHQ5EPhwJHVuDGno,5289
|
|
11
|
+
ddeutil/workflow/result.py,sha256=k4pcj5KjbEcEPymsEUXeGY4gyLMfPkMTO6YDrAtfk7Q,3408
|
|
12
|
+
ddeutil/workflow/scheduler.py,sha256=OlrnBZvVttoymeY1g-on9icEMU729OWISJReeX3jAKI,20452
|
|
13
|
+
ddeutil/workflow/stage.py,sha256=wn8CARTvFJY4ZK1SwjzH8sKoMRz_eIeSGUMgnDWNi6g,24031
|
|
14
|
+
ddeutil/workflow/templates.py,sha256=Ray7YFCxXHrYoMv_0WOsXqhthfU1hePrBDdnPfjtcqo,10818
|
|
15
|
+
ddeutil/workflow/utils.py,sha256=8LTqpvRPfrEYxsxhwszk6GKkyjrswxnwF3r_9vE8szw,6059
|
|
16
|
+
ddeutil/workflow/workflow.py,sha256=OHr-G57abR3OwynJiJb6DfoYpGeqXrifKcXOOroo8WI,42198
|
|
17
|
+
ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
|
|
18
|
+
ddeutil/workflow/api/api.py,sha256=hmH45GtpyZ-kbiqQNmnHgjwEiCiDDXLKTGvcNa5nFos,4041
|
|
19
|
+
ddeutil/workflow/api/repeat.py,sha256=zyvsrXKk-3-_N8ZRZSki0Mueshugum2jtqctEOp9QSc,4927
|
|
20
|
+
ddeutil/workflow/api/route.py,sha256=v96jNbgjM1cJ2MpVSRWs2kgRqF8DQElEBdRZrVFEpEw,8578
|
|
21
|
+
ddeutil_workflow-0.0.28.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
|
22
|
+
ddeutil_workflow-0.0.28.dist-info/METADATA,sha256=9YcDUz85c3ka_-AIkeas4hDvOqYrzlW78vBiYAzj8GA,14535
|
|
23
|
+
ddeutil_workflow-0.0.28.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
24
|
+
ddeutil_workflow-0.0.28.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
|
25
|
+
ddeutil_workflow-0.0.28.dist-info/RECORD,,
|
|
@@ -1,230 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: ddeutil-workflow
|
|
3
|
-
Version: 0.0.26.post1
|
|
4
|
-
Summary: Lightweight workflow orchestration with less dependencies
|
|
5
|
-
Author-email: ddeutils <korawich.anu@gmail.com>
|
|
6
|
-
License: MIT
|
|
7
|
-
Project-URL: Homepage, https://github.com/ddeutils/ddeutil-workflow/
|
|
8
|
-
Project-URL: Source Code, https://github.com/ddeutils/ddeutil-workflow/
|
|
9
|
-
Keywords: orchestration,workflow
|
|
10
|
-
Classifier: Topic :: Utilities
|
|
11
|
-
Classifier: Natural Language :: English
|
|
12
|
-
Classifier: Development Status :: 4 - Beta
|
|
13
|
-
Classifier: Intended Audience :: Developers
|
|
14
|
-
Classifier: Operating System :: OS Independent
|
|
15
|
-
Classifier: Programming Language :: Python
|
|
16
|
-
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
-
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
-
Requires-Python: >=3.9.13
|
|
23
|
-
Description-Content-Type: text/markdown
|
|
24
|
-
License-File: LICENSE
|
|
25
|
-
Requires-Dist: ddeutil>=0.4.3
|
|
26
|
-
Requires-Dist: ddeutil-io[toml,yaml]>=0.2.3
|
|
27
|
-
Requires-Dist: pydantic==2.10.4
|
|
28
|
-
Requires-Dist: python-dotenv==1.0.1
|
|
29
|
-
Requires-Dist: schedule<2.0.0,==1.2.2
|
|
30
|
-
Provides-Extra: api
|
|
31
|
-
Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "api"
|
|
32
|
-
|
|
33
|
-
# Workflow
|
|
34
|
-
|
|
35
|
-
[](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml)
|
|
36
|
-
[](https://codecov.io/gh/ddeutils/ddeutil-workflow)
|
|
37
|
-
[](https://pypi.org/project/ddeutil-workflow/)
|
|
38
|
-
[](https://pypi.org/project/ddeutil-workflow/)
|
|
39
|
-
[](https://github.com/ddeutils/ddeutil-workflow)
|
|
40
|
-
[](https://github.com/ddeutils/ddeutil-workflow/blob/main/LICENSE)
|
|
41
|
-
[](https://github.com/psf/black)
|
|
42
|
-
|
|
43
|
-
The **Lightweight workflow orchestration** with less dependencies the was created
|
|
44
|
-
for easy to make a simple metadata driven for data workflow orchestration.
|
|
45
|
-
It can to use for data operator by a `.yaml` template.
|
|
46
|
-
|
|
47
|
-
> [!WARNING]
|
|
48
|
-
> This package provide only orchestration workload. That mean you should not
|
|
49
|
-
> use the workflow stage to process any large volume data which use lot of compute
|
|
50
|
-
> resource. :cold_sweat:
|
|
51
|
-
|
|
52
|
-
In my opinion, I think it should not create duplicate workflow codes if I can
|
|
53
|
-
write with dynamic input parameters on the one template workflow that just change
|
|
54
|
-
the input parameters per use-case instead.
|
|
55
|
-
This way I can handle a lot of logical workflows in our orgs with only metadata
|
|
56
|
-
configuration. It called **Metadata Driven Data Workflow**.
|
|
57
|
-
|
|
58
|
-
**:pushpin: <u>Rules of This Workflow engine</u>**:
|
|
59
|
-
|
|
60
|
-
1. The Minimum frequency unit of scheduling is **1 minute** :warning:
|
|
61
|
-
2. Can not re-run only failed stage and its pending downstream :rotating_light:
|
|
62
|
-
3. All parallel tasks inside workflow engine use Multi-Threading
|
|
63
|
-
(Python 3.13 unlock GIL :unlock:)
|
|
64
|
-
|
|
65
|
-
> [!NOTE]
|
|
66
|
-
> _Disclaimer_: I inspire the dynamic statement from the [**GitHub Action**](https://github.com/features/actions)
|
|
67
|
-
> with `.yml` files and all of config file from several data orchestration framework
|
|
68
|
-
> tools from my experience on Data Engineer. :grimacing:
|
|
69
|
-
>
|
|
70
|
-
> Other workflow tools that I interest on them and pick some interested feature
|
|
71
|
-
> implement to this package:
|
|
72
|
-
>
|
|
73
|
-
> - [Google **Workflows**](https://cloud.google.com/workflows)
|
|
74
|
-
> - [AWS **Step Functions**](https://aws.amazon.com/step-functions/)
|
|
75
|
-
|
|
76
|
-
## :round_pushpin: Installation
|
|
77
|
-
|
|
78
|
-
This project need `ddeutil` and `ddeutil-io` extension namespace packages.
|
|
79
|
-
If you want to install this package with application add-ons, you should add
|
|
80
|
-
`app` in installation;
|
|
81
|
-
|
|
82
|
-
| Usecase | Install Optional | Support |
|
|
83
|
-
|----------------|------------------------------------------|--------------------|
|
|
84
|
-
| Python | `pip install ddeutil-workflow` | :heavy_check_mark: |
|
|
85
|
-
| FastAPI Server | `pip install ddeutil-workflow[api]` | :heavy_check_mark: |
|
|
86
|
-
|
|
87
|
-
## :beers: Usage
|
|
88
|
-
|
|
89
|
-
This is examples that use workflow file for running common Data Engineering
|
|
90
|
-
use-case.
|
|
91
|
-
|
|
92
|
-
> [!IMPORTANT]
|
|
93
|
-
> I recommend you to use the `hook` stage for all actions that you want to do
|
|
94
|
-
> with workflow activity that you want to orchestrate. Because it able to dynamic
|
|
95
|
-
> an input argument with the same hook function that make you use less time to
|
|
96
|
-
> maintenance your data workflows.
|
|
97
|
-
|
|
98
|
-
```yaml
|
|
99
|
-
run-py-local:
|
|
100
|
-
|
|
101
|
-
# Validate model that use to parsing exists for template file
|
|
102
|
-
type: Workflow
|
|
103
|
-
on:
|
|
104
|
-
# If workflow deploy to schedule, it will running every 5 minutes
|
|
105
|
-
# with Asia/Bangkok timezone.
|
|
106
|
-
- cronjob: '*/5 * * * *'
|
|
107
|
-
timezone: "Asia/Bangkok"
|
|
108
|
-
params:
|
|
109
|
-
# Incoming execution parameters will validate with this type. It allow
|
|
110
|
-
# to set default value or templating.
|
|
111
|
-
source-extract: str
|
|
112
|
-
run-date: datetime
|
|
113
|
-
jobs:
|
|
114
|
-
getting-api-data:
|
|
115
|
-
stages:
|
|
116
|
-
- name: "Retrieve API Data"
|
|
117
|
-
id: retrieve-api
|
|
118
|
-
uses: tasks/get-api-with-oauth-to-s3@requests
|
|
119
|
-
with:
|
|
120
|
-
# Arguments of source data that want to retrieve.
|
|
121
|
-
method: post
|
|
122
|
-
url: https://finances/open-data/currency-pairs/
|
|
123
|
-
body:
|
|
124
|
-
resource: ${{ params.source-extract }}
|
|
125
|
-
|
|
126
|
-
# You can able to use filtering like Jinja template but this
|
|
127
|
-
# package does not use it.
|
|
128
|
-
filter: ${{ params.run-date | fmt(fmt='%Y%m%d') }}
|
|
129
|
-
auth:
|
|
130
|
-
type: bearer
|
|
131
|
-
keys: ${API_ACCESS_REFRESH_TOKEN}
|
|
132
|
-
|
|
133
|
-
# Arguments of target data that want to landing.
|
|
134
|
-
writing_mode: flatten
|
|
135
|
-
aws_s3_path: my-data/open-data/${{ params.source-extract }}
|
|
136
|
-
|
|
137
|
-
# This Authentication code should implement with your custom hook
|
|
138
|
-
# function. The template allow you to use environment variable.
|
|
139
|
-
aws_access_client_id: ${AWS_ACCESS_CLIENT_ID}
|
|
140
|
-
aws_access_client_secret: ${AWS_ACCESS_CLIENT_SECRET}
|
|
141
|
-
```
|
|
142
|
-
|
|
143
|
-
The above workflow template is main executor pipeline that you want to do. If you
|
|
144
|
-
want to schedule this workflow, you want to dynamic its parameters change base on
|
|
145
|
-
execution time such as `run-date` should change base on that workflow running date.
|
|
146
|
-
|
|
147
|
-
So, this package provide the `Schedule` template for this action.
|
|
148
|
-
|
|
149
|
-
```yaml
|
|
150
|
-
schedule-run-local-wf:
|
|
151
|
-
|
|
152
|
-
# Validate model that use to parsing exists for template file
|
|
153
|
-
type: Schedule
|
|
154
|
-
workflows:
|
|
155
|
-
|
|
156
|
-
# Map existing workflow that want to deploy with scheduler application.
|
|
157
|
-
# It allow you to passing release parameter that dynamic change depend the
|
|
158
|
-
# current context of this scheduler application releasing that time.
|
|
159
|
-
- name: run-py-local
|
|
160
|
-
params:
|
|
161
|
-
source-extract: "USD-THB"
|
|
162
|
-
asat-dt: "${{ release.logical_date }}"
|
|
163
|
-
```
|
|
164
|
-
|
|
165
|
-
## :cookie: Configuration
|
|
166
|
-
|
|
167
|
-
The main configuration that use to dynamic changing with your propose of this
|
|
168
|
-
application. If any configuration values do not set yet, it will use default value
|
|
169
|
-
and do not raise any error to you.
|
|
170
|
-
|
|
171
|
-
| Environment | Component | Default | Description | Remark |
|
|
172
|
-
|:-------------------------------------------|:---------:|:-----------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------|--------|
|
|
173
|
-
| **WORKFLOW_ROOT_PATH** | Core | `.` | The root path of the workflow application. | |
|
|
174
|
-
| **WORKFLOW_CORE_REGISTRY** | Core | `src,src.ddeutil.workflow,tests,tests.utils` | List of importable string for the hook stage. | |
|
|
175
|
-
| **WORKFLOW_CORE_REGISTRY_FILTER** | Core | `src.ddeutil.workflow.utils,ddeutil.workflow.utils` | List of importable string for the filter template. | |
|
|
176
|
-
| **WORKFLOW_CORE_PATH_CONF** | Core | `conf` | The config path that keep all template `.yaml` files. | |
|
|
177
|
-
| **WORKFLOW_CORE_TIMEZONE** | Core | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. | |
|
|
178
|
-
| **WORKFLOW_CORE_STAGE_DEFAULT_ID** | Core | `true` | A flag that enable default stage ID that use for catch an execution output. | |
|
|
179
|
-
| **WORKFLOW_CORE_STAGE_RAISE_ERROR** | Core | `false` | A flag that all stage raise StageException from stage execution. | |
|
|
180
|
-
| **WORKFLOW_CORE_JOB_DEFAULT_ID** | Core | `false` | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. | |
|
|
181
|
-
| **WORKFLOW_CORE_JOB_RAISE_ERROR** | Core | `true` | A flag that all job raise JobException from job strategy execution. | |
|
|
182
|
-
| **WORKFLOW_CORE_MAX_NUM_POKING** | Core | `4` | . | |
|
|
183
|
-
| **WORKFLOW_CORE_MAX_JOB_PARALLEL** | Core | `2` | The maximum job number that able to run parallel in workflow executor. | |
|
|
184
|
-
| **WORKFLOW_CORE_MAX_JOB_EXEC_TIMEOUT** | Core | `600` | | |
|
|
185
|
-
| **WORKFLOW_CORE_MAX_CRON_PER_WORKFLOW** | Core | `5` | | |
|
|
186
|
-
| **WORKFLOW_CORE_MAX_QUEUE_COMPLETE_HIST** | Core | `16` | | |
|
|
187
|
-
| **WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE** | Core | `true` | A flog that enable generating ID with `md5` algorithm. | |
|
|
188
|
-
| **WORKFLOW_LOG_PATH** | Log | `./logs` | The log path of the workflow saving log. | |
|
|
189
|
-
| **WORKFLOW_LOG_DEBUG_MODE** | Log | `true` | A flag that enable logging with debug level mode. | |
|
|
190
|
-
| **WORKFLOW_LOG_ENABLE_WRITE** | Log | `true` | A flag that enable logging object saving log to its destination. | |
|
|
191
|
-
| **WORKFLOW_APP_MAX_PROCESS** | Schedule | `2` | The maximum process worker number that run in scheduler app module. | |
|
|
192
|
-
| **WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS** | Schedule | `100` | A schedule per process that run parallel. | |
|
|
193
|
-
| **WORKFLOW_APP_STOP_BOUNDARY_DELTA** | Schedule | `'{"minutes": 5, "seconds": 20}'` | A time delta value that use to stop scheduler app in json string format. | |
|
|
194
|
-
|
|
195
|
-
**API Application**:
|
|
196
|
-
|
|
197
|
-
| Environment | Component | Default | Description | Remark |
|
|
198
|
-
|:----------------------------------------|:-----------:|---------|------------------------------------------------------------------------------------|--------|
|
|
199
|
-
| **WORKFLOW_API_ENABLE_ROUTE_WORKFLOW** | API | `true` | A flag that enable workflow route to manage execute manually and workflow logging. | |
|
|
200
|
-
| **WORKFLOW_API_ENABLE_ROUTE_SCHEDULE** | API | `true` | A flag that enable run scheduler. | |
|
|
201
|
-
|
|
202
|
-
## :rocket: Deployment
|
|
203
|
-
|
|
204
|
-
This package able to run as a application service for receive manual trigger
|
|
205
|
-
from the master node via RestAPI or use to be Scheduler background service
|
|
206
|
-
like crontab job but via Python API.
|
|
207
|
-
|
|
208
|
-
### API Server
|
|
209
|
-
|
|
210
|
-
```shell
|
|
211
|
-
(venv) $ uvicorn src.ddeutil.workflow.api:app --host 127.0.0.1 --port 80
|
|
212
|
-
```
|
|
213
|
-
|
|
214
|
-
> [!NOTE]
|
|
215
|
-
> If this package already deploy, it able to use multiprocess;
|
|
216
|
-
> `uvicorn ddeutil.workflow.api:app --host 127.0.0.1 --port 80 --workers 4`
|
|
217
|
-
|
|
218
|
-
### Docker Container
|
|
219
|
-
|
|
220
|
-
Create Docker image;
|
|
221
|
-
|
|
222
|
-
```shell
|
|
223
|
-
$ docker build -t ddeutil-workflow:latest -f .container/Dockerfile .
|
|
224
|
-
```
|
|
225
|
-
|
|
226
|
-
Run the above Docker image;
|
|
227
|
-
|
|
228
|
-
```shell
|
|
229
|
-
$ docker run -i ddeutil-workflow:latest
|
|
230
|
-
```
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
ddeutil/workflow/__about__.py,sha256=jU_KFZf1uiZIWhuownbhRsjIL3oHGR_URL-jKTEnMKo,34
|
|
2
|
-
ddeutil/workflow/__cron.py,sha256=uA8XcbY_GwA9rJSHaHUaXaJyGDObJN0ZeYlJSinL8y8,26880
|
|
3
|
-
ddeutil/workflow/__init__.py,sha256=ozadVrqfqFRuukjv_zXUcgLANdiSrC6wrKkyVjdGg3w,1521
|
|
4
|
-
ddeutil/workflow/__types.py,sha256=Ia7f38kvL3NibwmRKi0wQ1ud_45Z-SojYGhNJwIqcu8,3713
|
|
5
|
-
ddeutil/workflow/conf.py,sha256=AU3GKTaxFFGDN-Sg8BGb08xj7vRBCTTjwk0FaORLJIk,16188
|
|
6
|
-
ddeutil/workflow/cron.py,sha256=75A0hqevvouziKoLALncLJspVAeki9qCH3zniAJaxzY,7513
|
|
7
|
-
ddeutil/workflow/exceptions.py,sha256=P56K7VD3etGm9y-k_GXrzEyqsTCaz9EJazTIshZDf9g,943
|
|
8
|
-
ddeutil/workflow/job.py,sha256=cvSLMdc1sMl1MeU7so7Oe2SdRYxQwt6hm55mLV1iP-Y,24219
|
|
9
|
-
ddeutil/workflow/params.py,sha256=uPGkZx18E-iZ8BteqQ2ONgg0frhF3ZmP5cOyfK2j59U,5280
|
|
10
|
-
ddeutil/workflow/result.py,sha256=WIC8MsnfLiWNpZomT6jS4YCdYhlbIVVBjtGGe2dkoKk,3404
|
|
11
|
-
ddeutil/workflow/scheduler.py,sha256=BbY_3Y3QOdNwDfdvnRa7grGC2_a0Hn1KJbZKAscchk8,20454
|
|
12
|
-
ddeutil/workflow/stage.py,sha256=a2sngzs9DkP6GU2pgAD3QvGoijyBQTR_pOhyJUIuWAo,26692
|
|
13
|
-
ddeutil/workflow/utils.py,sha256=pucRnCi9aLJDptXhzzReHZd5d-S0o5oZif5tr6H4iy8,18736
|
|
14
|
-
ddeutil/workflow/workflow.py,sha256=s6E-mKzSVQPTSV0biIAu5lFjslo6blKA-WTAjeOfLuw,42183
|
|
15
|
-
ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
|
|
16
|
-
ddeutil/workflow/api/api.py,sha256=Md1cz3Edc7_uz63s_L_i-R3IE4mkO3aTADrX8GOGU-Y,5644
|
|
17
|
-
ddeutil/workflow/api/repeat.py,sha256=zyvsrXKk-3-_N8ZRZSki0Mueshugum2jtqctEOp9QSc,4927
|
|
18
|
-
ddeutil/workflow/api/route.py,sha256=v96jNbgjM1cJ2MpVSRWs2kgRqF8DQElEBdRZrVFEpEw,8578
|
|
19
|
-
ddeutil_workflow-0.0.26.post1.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
|
20
|
-
ddeutil_workflow-0.0.26.post1.dist-info/METADATA,sha256=B95z9M1Z9DWiKXQr1VoRvtlYcB6eX11RGktlAwn4MvI,14364
|
|
21
|
-
ddeutil_workflow-0.0.26.post1.dist-info/WHEEL,sha256=A3WOREP4zgxI0fKrHUG8DC8013e3dK3n7a6HDbcEIwE,91
|
|
22
|
-
ddeutil_workflow-0.0.26.post1.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
|
23
|
-
ddeutil_workflow-0.0.26.post1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|