ddeutil-workflow 0.0.32__py3-none-any.whl → 0.0.34__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +20 -12
- ddeutil/workflow/api/api.py +2 -2
- ddeutil/workflow/api/route.py +4 -3
- ddeutil/workflow/audit.py +252 -0
- ddeutil/workflow/{hook.py → call.py} +27 -27
- ddeutil/workflow/conf.py +163 -271
- ddeutil/workflow/job.py +113 -144
- ddeutil/workflow/result.py +199 -46
- ddeutil/workflow/scheduler.py +167 -151
- ddeutil/workflow/{stage.py → stages.py} +174 -89
- ddeutil/workflow/utils.py +20 -2
- ddeutil/workflow/workflow.py +172 -148
- {ddeutil_workflow-0.0.32.dist-info → ddeutil_workflow-0.0.34.dist-info}/METADATA +43 -38
- ddeutil_workflow-0.0.34.dist-info/RECORD +26 -0
- ddeutil_workflow-0.0.32.dist-info/RECORD +0 -25
- {ddeutil_workflow-0.0.32.dist-info → ddeutil_workflow-0.0.34.dist-info}/LICENSE +0 -0
- {ddeutil_workflow-0.0.32.dist-info → ddeutil_workflow-0.0.34.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.32.dist-info → ddeutil_workflow-0.0.34.dist-info}/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: ddeutil-workflow
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.34
|
|
4
4
|
Summary: Lightweight workflow orchestration
|
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -74,25 +74,25 @@ you should to set the data layer separate this core program before run this appl
|
|
|
74
74
|
|
|
75
75
|
```mermaid
|
|
76
76
|
flowchart LR
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
77
|
+
A((fa:fa-user User))
|
|
78
|
+
|
|
79
|
+
subgraph Docker Container
|
|
80
|
+
direction TB
|
|
80
81
|
G@{ shape: rounded, label: "Observe<br>Application" }
|
|
81
|
-
end
|
|
82
82
|
end
|
|
83
83
|
|
|
84
|
-
A --->|action| B(Workflow<br>Application)
|
|
85
|
-
B ---> |response| A
|
|
86
|
-
B -..-> |response| G
|
|
87
|
-
G -..-> |request| B
|
|
88
|
-
|
|
89
84
|
subgraph Docker Container
|
|
90
|
-
|
|
85
|
+
direction TB
|
|
86
|
+
B@{ shape: rounded, label: "Workflow<br>Application" }
|
|
91
87
|
end
|
|
92
88
|
|
|
89
|
+
A <--->|action &<br>response| B
|
|
90
|
+
B -....-> |response| G
|
|
91
|
+
G -....-> |request| B
|
|
92
|
+
|
|
93
93
|
subgraph Data Context
|
|
94
94
|
D@{ shape: processes, label: "Logs" }
|
|
95
|
-
E@{ shape: lin-cyl, label: "
|
|
95
|
+
E@{ shape: lin-cyl, label: "Audit<br>Logs" }
|
|
96
96
|
end
|
|
97
97
|
|
|
98
98
|
subgraph Git Context
|
|
@@ -138,9 +138,9 @@ This is examples that use workflow file for running common Data Engineering
|
|
|
138
138
|
use-case.
|
|
139
139
|
|
|
140
140
|
> [!IMPORTANT]
|
|
141
|
-
> I recommend you to use the `
|
|
141
|
+
> I recommend you to use the `call` stage for all actions that you want to do
|
|
142
142
|
> with workflow activity that you want to orchestrate. Because it is able to
|
|
143
|
-
> dynamic an input argument with the same
|
|
143
|
+
> dynamic an input argument with the same call function that make you use less
|
|
144
144
|
> time to maintenance your data workflows.
|
|
145
145
|
|
|
146
146
|
```yaml
|
|
@@ -182,7 +182,7 @@ run-py-local:
|
|
|
182
182
|
writing_mode: flatten
|
|
183
183
|
aws_s3_path: my-data/open-data/${{ params.source-extract }}
|
|
184
184
|
|
|
185
|
-
# This Authentication code should implement with your custom
|
|
185
|
+
# This Authentication code should implement with your custom call
|
|
186
186
|
# function. The template allow you to use environment variable.
|
|
187
187
|
aws_access_client_id: ${AWS_ACCESS_CLIENT_ID}
|
|
188
188
|
aws_access_client_secret: ${AWS_ACCESS_CLIENT_SECRET}
|
|
@@ -244,29 +244,34 @@ it will use default value and do not raise any error to you.
|
|
|
244
244
|
> The config value that you will set on the environment should combine with
|
|
245
245
|
> prefix, component, and name which is `WORKFLOW_{component}_{name}` (Upper case).
|
|
246
246
|
|
|
247
|
-
| Name | Component | Default
|
|
248
|
-
|
|
249
|
-
| **ROOT_PATH** | Core | `.`
|
|
250
|
-
| **REGISTRY** | Core | `.`
|
|
251
|
-
| **REGISTRY_FILTER** | Core | `ddeutil.workflow.templates`
|
|
252
|
-
| **CONF_PATH** | Core | `conf`
|
|
253
|
-
| **TIMEZONE** | Core | `Asia/Bangkok`
|
|
254
|
-
| **STAGE_DEFAULT_ID** | Core | `true`
|
|
255
|
-
| **STAGE_RAISE_ERROR** | Core | `false`
|
|
256
|
-
| **JOB_DEFAULT_ID** | Core | `false`
|
|
257
|
-
| **JOB_RAISE_ERROR** | Core | `true`
|
|
258
|
-
| **MAX_NUM_POKING** | Core | `4`
|
|
259
|
-
| **MAX_JOB_PARALLEL** | Core | `2`
|
|
260
|
-
| **MAX_JOB_EXEC_TIMEOUT** | Core | `600`
|
|
261
|
-
| **MAX_CRON_PER_WORKFLOW** | Core | `5`
|
|
262
|
-
| **MAX_QUEUE_COMPLETE_HIST** | Core | `16`
|
|
263
|
-
| **GENERATE_ID_SIMPLE_MODE** | Core | `true`
|
|
264
|
-
| **PATH** | Log | `./logs`
|
|
265
|
-
| **DEBUG_MODE** | Log | `true`
|
|
266
|
-
| **
|
|
267
|
-
| **
|
|
268
|
-
| **
|
|
269
|
-
| **
|
|
247
|
+
| Name | Component | Default | Description |
|
|
248
|
+
|:-----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------|
|
|
249
|
+
| **ROOT_PATH** | Core | `.` | The root path of the workflow application. |
|
|
250
|
+
| **REGISTRY** | Core | `.` | List of importable string for the call stage. |
|
|
251
|
+
| **REGISTRY_FILTER** | Core | `ddeutil.workflow.templates` | List of importable string for the filter template. |
|
|
252
|
+
| **CONF_PATH** | Core | `conf` | The config path that keep all template `.yaml` files. |
|
|
253
|
+
| **TIMEZONE** | Core | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
|
254
|
+
| **STAGE_DEFAULT_ID** | Core | `true` | A flag that enable default stage ID that use for catch an execution output. |
|
|
255
|
+
| **STAGE_RAISE_ERROR** | Core | `false` | A flag that all stage raise StageException from stage execution. |
|
|
256
|
+
| **JOB_DEFAULT_ID** | Core | `false` | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
|
|
257
|
+
| **JOB_RAISE_ERROR** | Core | `true` | A flag that all job raise JobException from job strategy execution. |
|
|
258
|
+
| **MAX_NUM_POKING** | Core | `4` | . |
|
|
259
|
+
| **MAX_JOB_PARALLEL** | Core | `2` | The maximum job number that able to run parallel in workflow executor. |
|
|
260
|
+
| **MAX_JOB_EXEC_TIMEOUT** | Core | `600` | |
|
|
261
|
+
| **MAX_CRON_PER_WORKFLOW** | Core | `5` | |
|
|
262
|
+
| **MAX_QUEUE_COMPLETE_HIST** | Core | `16` | |
|
|
263
|
+
| **GENERATE_ID_SIMPLE_MODE** | Core | `true` | A flog that enable generating ID with `md5` algorithm. |
|
|
264
|
+
| **PATH** | Log | `./logs` | The log path of the workflow saving log. |
|
|
265
|
+
| **DEBUG_MODE** | Log | `true` | A flag that enable logging with debug level mode. |
|
|
266
|
+
| **FORMAT** | Log | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | |
|
|
267
|
+
| **FORMAT_FILE** | Log | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})` | |
|
|
268
|
+
| **DATETIME_FORMAT** | Log | `%Y-%m-%d %H:%M:%S` | |
|
|
269
|
+
| **ENABLE_WRITE** | Log | `false` | |
|
|
270
|
+
| **PATH** | Audit | `./audits` | |
|
|
271
|
+
| **ENABLE_WRITE** | Audit | `true` | A flag that enable logging object saving log to its destination. |
|
|
272
|
+
| **MAX_PROCESS** | App | `2` | The maximum process worker number that run in scheduler app module. |
|
|
273
|
+
| **MAX_SCHEDULE_PER_PROCESS** | App | `100` | A schedule per process that run parallel. |
|
|
274
|
+
| **STOP_BOUNDARY_DELTA** | App | `'{"minutes": 5, "seconds": 20}'` | A time delta value that use to stop scheduler app in json string format. |
|
|
270
275
|
|
|
271
276
|
**API Application**:
|
|
272
277
|
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
ddeutil/workflow/__about__.py,sha256=lXPz0LJuQJgNzG0V-1UIfdYsebVctya1i8YqUhNgkXU,28
|
|
2
|
+
ddeutil/workflow/__cron.py,sha256=3i-wmjTlh0ADCzN9pLKaWHzJkXzC72aIBmVEQSbyCCE,26895
|
|
3
|
+
ddeutil/workflow/__init__.py,sha256=ucsWC4fWdD3ZJu3mDLwe1BCS7wHR0-43jtxm8eZe53o,1779
|
|
4
|
+
ddeutil/workflow/__types.py,sha256=CK1jfzyHP9P-MB0ElhpJZ59ZFGJC9MkQuAop5739_9k,4304
|
|
5
|
+
ddeutil/workflow/audit.py,sha256=LZDgZbWioy0ctEWBmTUXJY_cAdFEMxvkvamJbkVAhdc,8066
|
|
6
|
+
ddeutil/workflow/call.py,sha256=gkEmrYZT1KFVMVy6uyXcl2wBPQXyg1onPLkXN3q9JJk,5469
|
|
7
|
+
ddeutil/workflow/conf.py,sha256=cFc2cd_SGXg9PMrkvCT7WWE85a5UN-DdH53_JIbFyzs,14031
|
|
8
|
+
ddeutil/workflow/cron.py,sha256=j8EeoHst70toRfnD_frix41vrI-eLYVJkZ9yeJtpfnI,8871
|
|
9
|
+
ddeutil/workflow/exceptions.py,sha256=5ghT443VLq0IeU87loHNEqqrrrctklP7YfxwJ51ImWU,949
|
|
10
|
+
ddeutil/workflow/job.py,sha256=cm4u_l3fopL6tgUZYy-apZMGektaFyB5qu9uja3Nups,23776
|
|
11
|
+
ddeutil/workflow/params.py,sha256=LKR7jXyxTb5NVrFav_fl2y9xo3p7qL1S9h-i6CtvNwE,5851
|
|
12
|
+
ddeutil/workflow/result.py,sha256=z3C1V9C8D736gqTefBH0uUFr40pNU5Z2TwV2Pndoxmw,7800
|
|
13
|
+
ddeutil/workflow/scheduler.py,sha256=5xlgzunC-eDOc0uSxJZhphywVOwrtmuF0mT5YYc83BY,24943
|
|
14
|
+
ddeutil/workflow/stages.py,sha256=5UhmcDgJ214NSNyTwgYcX2hvSW1Rcgk_Qq71n1VMdZ4,26555
|
|
15
|
+
ddeutil/workflow/templates.py,sha256=A0JgZFGkBv-AX-EskZj656nG5zFd3j1PpLpyXihf6Xg,10967
|
|
16
|
+
ddeutil/workflow/utils.py,sha256=MctJmvklTYtiqZ-nZ7fazQeDoe77UvU0YUEqQZSlbCs,7225
|
|
17
|
+
ddeutil/workflow/workflow.py,sha256=7SvF80y4GrrwL-0X46wlct76qPp_fttZpEgxu5xjcBY,45148
|
|
18
|
+
ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
|
|
19
|
+
ddeutil/workflow/api/api.py,sha256=TfgJtu-yREsrRveLcqTjxoJszuhq21qKkl4oyQpSzvQ,3959
|
|
20
|
+
ddeutil/workflow/api/repeat.py,sha256=zyvsrXKk-3-_N8ZRZSki0Mueshugum2jtqctEOp9QSc,4927
|
|
21
|
+
ddeutil/workflow/api/route.py,sha256=fKs5zlG4prwL8HplXYjYsyuxelOKgV-54VBgBufIBcQ,8629
|
|
22
|
+
ddeutil_workflow-0.0.34.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
|
23
|
+
ddeutil_workflow-0.0.34.dist-info/METADATA,sha256=J-Y3CGyv6r39SVYlasXC6-fVNFYhAtmWjM46CDoYz_0,19221
|
|
24
|
+
ddeutil_workflow-0.0.34.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
25
|
+
ddeutil_workflow-0.0.34.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
|
26
|
+
ddeutil_workflow-0.0.34.dist-info/RECORD,,
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
ddeutil/workflow/__about__.py,sha256=xnsfT_lwFz-qKzFIHbOUGMgvgpOAAANig-X3brDri44,28
|
|
2
|
-
ddeutil/workflow/__cron.py,sha256=3i-wmjTlh0ADCzN9pLKaWHzJkXzC72aIBmVEQSbyCCE,26895
|
|
3
|
-
ddeutil/workflow/__init__.py,sha256=pRIZIGwC7Xs8Ur7-jHPIAMLriD5If9zOPc-ZmKZS2XQ,1678
|
|
4
|
-
ddeutil/workflow/__types.py,sha256=CK1jfzyHP9P-MB0ElhpJZ59ZFGJC9MkQuAop5739_9k,4304
|
|
5
|
-
ddeutil/workflow/conf.py,sha256=6yGbSi69lsccYgnrwTzdjdPhU54hUop2e1GjBNres08,17663
|
|
6
|
-
ddeutil/workflow/cron.py,sha256=j8EeoHst70toRfnD_frix41vrI-eLYVJkZ9yeJtpfnI,8871
|
|
7
|
-
ddeutil/workflow/exceptions.py,sha256=5ghT443VLq0IeU87loHNEqqrrrctklP7YfxwJ51ImWU,949
|
|
8
|
-
ddeutil/workflow/hook.py,sha256=MgZFlTGvaRSBrTouZGlxwYpKQoKDOT26PNhESeL3LY0,5469
|
|
9
|
-
ddeutil/workflow/job.py,sha256=siph5FBU5Ah2CHWZyXotHJKMemFWQXSBqSdwk8b_6p8,24366
|
|
10
|
-
ddeutil/workflow/params.py,sha256=LKR7jXyxTb5NVrFav_fl2y9xo3p7qL1S9h-i6CtvNwE,5851
|
|
11
|
-
ddeutil/workflow/result.py,sha256=8LItqF-Xe6pAAWkAsY_QFkKBOA0fEBh97I2og3CZsPc,3409
|
|
12
|
-
ddeutil/workflow/scheduler.py,sha256=r26ZKpx0kuaL0hMJ39tIyZaeFMwhcwitUzBh-ge2ZJg,24394
|
|
13
|
-
ddeutil/workflow/stage.py,sha256=vUne9aUZX7G8K5BkBwirAjXSVwVvAoBt8HXbqUMEgns,24329
|
|
14
|
-
ddeutil/workflow/templates.py,sha256=A0JgZFGkBv-AX-EskZj656nG5zFd3j1PpLpyXihf6Xg,10967
|
|
15
|
-
ddeutil/workflow/utils.py,sha256=rTDQKaaber7cRqzJjWpCP9OTbarti1UMKdLgH6VRjFM,6709
|
|
16
|
-
ddeutil/workflow/workflow.py,sha256=ET1otR5VcfnOMoNiW7EMb1_wIaxNw9yWsBXS5kVWG9s,43428
|
|
17
|
-
ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
|
|
18
|
-
ddeutil/workflow/api/api.py,sha256=AwcUkE6S2Txz4hZOD10TsUkuFiKQ47rpMYMYpl7ssis,3954
|
|
19
|
-
ddeutil/workflow/api/repeat.py,sha256=zyvsrXKk-3-_N8ZRZSki0Mueshugum2jtqctEOp9QSc,4927
|
|
20
|
-
ddeutil/workflow/api/route.py,sha256=A9YPfchPNRyN6ev5sg1Z12zSi8JEQ6yZEea0EFPQp8k,8591
|
|
21
|
-
ddeutil_workflow-0.0.32.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
|
22
|
-
ddeutil_workflow-0.0.32.dist-info/METADATA,sha256=FYn_uM88ehiyBBZto8ECxmWHwBPtFGHAOcBPkaRBNFU,15552
|
|
23
|
-
ddeutil_workflow-0.0.32.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
24
|
-
ddeutil_workflow-0.0.32.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
|
25
|
-
ddeutil_workflow-0.0.32.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|