ddeutil-workflow 0.0.32__tar.gz → 0.0.33__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ddeutil_workflow-0.0.32/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.33}/PKG-INFO +29 -25
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/README.md +28 -24
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/pyproject.toml +1 -0
- ddeutil_workflow-0.0.33/src/ddeutil/workflow/__about__.py +1 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil/workflow/__init__.py +4 -2
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil/workflow/api/api.py +2 -2
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil/workflow/api/route.py +4 -3
- ddeutil_workflow-0.0.33/src/ddeutil/workflow/audit.py +261 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil/workflow/conf.py +122 -265
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil/workflow/job.py +59 -52
- ddeutil_workflow-0.0.33/src/ddeutil/workflow/result.py +155 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil/workflow/scheduler.py +7 -6
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil/workflow/stage.py +73 -54
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil/workflow/workflow.py +63 -64
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33/src/ddeutil_workflow.egg-info}/PKG-INFO +29 -25
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil_workflow.egg-info/SOURCES.txt +4 -3
- ddeutil_workflow-0.0.32/tests/test_conf_log.py → ddeutil_workflow-0.0.33/tests/test_audit.py +14 -13
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_conf.py +7 -6
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_hook_tag.py +22 -0
- ddeutil_workflow-0.0.32/tests/test_job_exec_py.py → ddeutil_workflow-0.0.33/tests/test_job_exec.py +88 -12
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_result.py +14 -21
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_workflow_task.py +3 -2
- ddeutil_workflow-0.0.32/src/ddeutil/workflow/__about__.py +0 -1
- ddeutil_workflow-0.0.32/src/ddeutil/workflow/result.py +0 -103
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/LICENSE +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/setup.cfg +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil/workflow/__cron.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil/workflow/__types.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil/workflow/api/__init__.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil/workflow/api/repeat.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil/workflow/cron.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil/workflow/exceptions.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil/workflow/hook.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil/workflow/params.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil/workflow/templates.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil/workflow/utils.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil_workflow.egg-info/dependency_links.txt +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil_workflow.egg-info/requires.txt +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/src/ddeutil_workflow.egg-info/top_level.txt +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test__cron.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test__regex.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_cron_on.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_job.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_job_exec_strategy.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_job_strategy.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_params.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_release.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_release_queue.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_schedule.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_schedule_control.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_schedule_pending.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_schedule_tasks.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_schedule_workflow.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_stage.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_stage_handler_exec.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_templates.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_templates_filter.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_utils.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_workflow.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_workflow_exec.py +0 -0
- /ddeutil_workflow-0.0.32/tests/test_workflow_job_exec.py → /ddeutil_workflow-0.0.33/tests/test_workflow_exec_job.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_workflow_exec_poke.py +0 -0
- {ddeutil_workflow-0.0.32 → ddeutil_workflow-0.0.33}/tests/test_workflow_exec_release.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.2
|
2
2
|
Name: ddeutil-workflow
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.33
|
4
4
|
Summary: Lightweight workflow orchestration
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
6
6
|
License: MIT
|
@@ -92,7 +92,7 @@ flowchart LR
|
|
92
92
|
|
93
93
|
subgraph Data Context
|
94
94
|
D@{ shape: processes, label: "Logs" }
|
95
|
-
E@{ shape: lin-cyl, label: "
|
95
|
+
E@{ shape: lin-cyl, label: "Audit<br>Logs" }
|
96
96
|
end
|
97
97
|
|
98
98
|
subgraph Git Context
|
@@ -244,29 +244,33 @@ it will use default value and do not raise any error to you.
|
|
244
244
|
> The config value that you will set on the environment should combine with
|
245
245
|
> prefix, component, and name which is `WORKFLOW_{component}_{name}` (Upper case).
|
246
246
|
|
247
|
-
| Name | Component | Default
|
248
|
-
|
249
|
-
| **ROOT_PATH** | Core | `.`
|
250
|
-
| **REGISTRY** | Core | `.`
|
251
|
-
| **REGISTRY_FILTER** | Core | `ddeutil.workflow.templates`
|
252
|
-
| **CONF_PATH** | Core | `conf`
|
253
|
-
| **TIMEZONE** | Core | `Asia/Bangkok`
|
254
|
-
| **STAGE_DEFAULT_ID** | Core | `true`
|
255
|
-
| **STAGE_RAISE_ERROR** | Core | `false`
|
256
|
-
| **JOB_DEFAULT_ID** | Core | `false`
|
257
|
-
| **JOB_RAISE_ERROR** | Core | `true`
|
258
|
-
| **MAX_NUM_POKING** | Core | `4`
|
259
|
-
| **MAX_JOB_PARALLEL** | Core | `2`
|
260
|
-
| **MAX_JOB_EXEC_TIMEOUT** | Core | `600`
|
261
|
-
| **MAX_CRON_PER_WORKFLOW** | Core | `5`
|
262
|
-
| **MAX_QUEUE_COMPLETE_HIST** | Core | `16`
|
263
|
-
| **GENERATE_ID_SIMPLE_MODE** | Core | `true`
|
264
|
-
| **PATH** | Log | `./logs`
|
265
|
-
| **DEBUG_MODE** | Log | `true`
|
266
|
-
| **
|
267
|
-
| **
|
268
|
-
| **
|
269
|
-
| **
|
247
|
+
| Name | Component | Default | Description |
|
248
|
+
|:-----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------|
|
249
|
+
| **ROOT_PATH** | Core | `.` | The root path of the workflow application. |
|
250
|
+
| **REGISTRY** | Core | `.` | List of importable string for the hook stage. |
|
251
|
+
| **REGISTRY_FILTER** | Core | `ddeutil.workflow.templates` | List of importable string for the filter template. |
|
252
|
+
| **CONF_PATH** | Core | `conf` | The config path that keep all template `.yaml` files. |
|
253
|
+
| **TIMEZONE** | Core | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
254
|
+
| **STAGE_DEFAULT_ID** | Core | `true` | A flag that enable default stage ID that use for catch an execution output. |
|
255
|
+
| **STAGE_RAISE_ERROR** | Core | `false` | A flag that all stage raise StageException from stage execution. |
|
256
|
+
| **JOB_DEFAULT_ID** | Core | `false` | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
|
257
|
+
| **JOB_RAISE_ERROR** | Core | `true` | A flag that all job raise JobException from job strategy execution. |
|
258
|
+
| **MAX_NUM_POKING** | Core | `4` | . |
|
259
|
+
| **MAX_JOB_PARALLEL** | Core | `2` | The maximum job number that able to run parallel in workflow executor. |
|
260
|
+
| **MAX_JOB_EXEC_TIMEOUT** | Core | `600` | |
|
261
|
+
| **MAX_CRON_PER_WORKFLOW** | Core | `5` | |
|
262
|
+
| **MAX_QUEUE_COMPLETE_HIST** | Core | `16` | |
|
263
|
+
| **GENERATE_ID_SIMPLE_MODE** | Core | `true` | A flog that enable generating ID with `md5` algorithm. |
|
264
|
+
| **PATH** | Log | `./logs` | The log path of the workflow saving log. |
|
265
|
+
| **DEBUG_MODE** | Log | `true` | A flag that enable logging with debug level mode. |
|
266
|
+
| **FORMAT** | Log | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | |
|
267
|
+
| **DATETIME_FORMAT** | Log | `%Y-%m-%d %H:%M:%S` | |
|
268
|
+
| **ENABLE_ROTATED_FILE** | Log | `false` | |
|
269
|
+
| **PATH** | Audit | `./logs` | |
|
270
|
+
| **ENABLE_WRITE** | Audit | `true` | A flag that enable logging object saving log to its destination. |
|
271
|
+
| **MAX_PROCESS** | App | `2` | The maximum process worker number that run in scheduler app module. |
|
272
|
+
| **MAX_SCHEDULE_PER_PROCESS** | App | `100` | A schedule per process that run parallel. |
|
273
|
+
| **STOP_BOUNDARY_DELTA** | App | `'{"minutes": 5, "seconds": 20}'` | A time delta value that use to stop scheduler app in json string format. |
|
270
274
|
|
271
275
|
**API Application**:
|
272
276
|
|
@@ -60,7 +60,7 @@ flowchart LR
|
|
60
60
|
|
61
61
|
subgraph Data Context
|
62
62
|
D@{ shape: processes, label: "Logs" }
|
63
|
-
E@{ shape: lin-cyl, label: "
|
63
|
+
E@{ shape: lin-cyl, label: "Audit<br>Logs" }
|
64
64
|
end
|
65
65
|
|
66
66
|
subgraph Git Context
|
@@ -212,29 +212,33 @@ it will use default value and do not raise any error to you.
|
|
212
212
|
> The config value that you will set on the environment should combine with
|
213
213
|
> prefix, component, and name which is `WORKFLOW_{component}_{name}` (Upper case).
|
214
214
|
|
215
|
-
| Name | Component | Default
|
216
|
-
|
217
|
-
| **ROOT_PATH** | Core | `.`
|
218
|
-
| **REGISTRY** | Core | `.`
|
219
|
-
| **REGISTRY_FILTER** | Core | `ddeutil.workflow.templates`
|
220
|
-
| **CONF_PATH** | Core | `conf`
|
221
|
-
| **TIMEZONE** | Core | `Asia/Bangkok`
|
222
|
-
| **STAGE_DEFAULT_ID** | Core | `true`
|
223
|
-
| **STAGE_RAISE_ERROR** | Core | `false`
|
224
|
-
| **JOB_DEFAULT_ID** | Core | `false`
|
225
|
-
| **JOB_RAISE_ERROR** | Core | `true`
|
226
|
-
| **MAX_NUM_POKING** | Core | `4`
|
227
|
-
| **MAX_JOB_PARALLEL** | Core | `2`
|
228
|
-
| **MAX_JOB_EXEC_TIMEOUT** | Core | `600`
|
229
|
-
| **MAX_CRON_PER_WORKFLOW** | Core | `5`
|
230
|
-
| **MAX_QUEUE_COMPLETE_HIST** | Core | `16`
|
231
|
-
| **GENERATE_ID_SIMPLE_MODE** | Core | `true`
|
232
|
-
| **PATH** | Log | `./logs`
|
233
|
-
| **DEBUG_MODE** | Log | `true`
|
234
|
-
| **
|
235
|
-
| **
|
236
|
-
| **
|
237
|
-
| **
|
215
|
+
| Name | Component | Default | Description |
|
216
|
+
|:-----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------|
|
217
|
+
| **ROOT_PATH** | Core | `.` | The root path of the workflow application. |
|
218
|
+
| **REGISTRY** | Core | `.` | List of importable string for the hook stage. |
|
219
|
+
| **REGISTRY_FILTER** | Core | `ddeutil.workflow.templates` | List of importable string for the filter template. |
|
220
|
+
| **CONF_PATH** | Core | `conf` | The config path that keep all template `.yaml` files. |
|
221
|
+
| **TIMEZONE** | Core | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
222
|
+
| **STAGE_DEFAULT_ID** | Core | `true` | A flag that enable default stage ID that use for catch an execution output. |
|
223
|
+
| **STAGE_RAISE_ERROR** | Core | `false` | A flag that all stage raise StageException from stage execution. |
|
224
|
+
| **JOB_DEFAULT_ID** | Core | `false` | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
|
225
|
+
| **JOB_RAISE_ERROR** | Core | `true` | A flag that all job raise JobException from job strategy execution. |
|
226
|
+
| **MAX_NUM_POKING** | Core | `4` | . |
|
227
|
+
| **MAX_JOB_PARALLEL** | Core | `2` | The maximum job number that able to run parallel in workflow executor. |
|
228
|
+
| **MAX_JOB_EXEC_TIMEOUT** | Core | `600` | |
|
229
|
+
| **MAX_CRON_PER_WORKFLOW** | Core | `5` | |
|
230
|
+
| **MAX_QUEUE_COMPLETE_HIST** | Core | `16` | |
|
231
|
+
| **GENERATE_ID_SIMPLE_MODE** | Core | `true` | A flog that enable generating ID with `md5` algorithm. |
|
232
|
+
| **PATH** | Log | `./logs` | The log path of the workflow saving log. |
|
233
|
+
| **DEBUG_MODE** | Log | `true` | A flag that enable logging with debug level mode. |
|
234
|
+
| **FORMAT** | Log | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | |
|
235
|
+
| **DATETIME_FORMAT** | Log | `%Y-%m-%d %H:%M:%S` | |
|
236
|
+
| **ENABLE_ROTATED_FILE** | Log | `false` | |
|
237
|
+
| **PATH** | Audit | `./logs` | |
|
238
|
+
| **ENABLE_WRITE** | Audit | `true` | A flag that enable logging object saving log to its destination. |
|
239
|
+
| **MAX_PROCESS** | App | `2` | The maximum process worker number that run in scheduler app module. |
|
240
|
+
| **MAX_SCHEDULE_PER_PROCESS** | App | `100` | A schedule per process that run parallel. |
|
241
|
+
| **STOP_BOUNDARY_DELTA** | App | `'{"minutes": 5, "seconds": 20}'` | A time delta value that use to stop scheduler app in json string format. |
|
238
242
|
|
239
243
|
**API Application**:
|
240
244
|
|
@@ -0,0 +1 @@
|
|
1
|
+
__version__: str = "0.0.33"
|
@@ -5,13 +5,15 @@
|
|
5
5
|
# ------------------------------------------------------------------------------
|
6
6
|
from .__cron import CronJob, CronRunner
|
7
7
|
from .__types import Re
|
8
|
+
from .audit import (
|
9
|
+
Audit,
|
10
|
+
get_audit,
|
11
|
+
)
|
8
12
|
from .conf import (
|
9
13
|
Config,
|
10
14
|
Loader,
|
11
|
-
Log,
|
12
15
|
config,
|
13
16
|
env,
|
14
|
-
get_log,
|
15
17
|
get_logger,
|
16
18
|
)
|
17
19
|
from .cron import (
|
@@ -86,7 +86,7 @@ if config.enable_route_workflow:
|
|
86
86
|
|
87
87
|
# NOTE: Enable the schedule route.
|
88
88
|
if config.enable_route_schedule:
|
89
|
-
from ..
|
89
|
+
from ..audit import get_audit
|
90
90
|
from ..scheduler import schedule_task
|
91
91
|
from .route import schedule_route
|
92
92
|
|
@@ -106,7 +106,7 @@ if config.enable_route_schedule:
|
|
106
106
|
stop=datetime.now(config.tz) + timedelta(minutes=1),
|
107
107
|
queue=app.state.workflow_queue,
|
108
108
|
threads=app.state.workflow_threads,
|
109
|
-
log=
|
109
|
+
log=get_audit(),
|
110
110
|
)
|
111
111
|
|
112
112
|
@schedule_route.on_event("startup")
|
@@ -16,7 +16,8 @@ from fastapi.responses import UJSONResponse
|
|
16
16
|
from pydantic import BaseModel
|
17
17
|
|
18
18
|
from ..__types import DictData
|
19
|
-
from ..
|
19
|
+
from ..audit import Audit, get_audit
|
20
|
+
from ..conf import Loader, config, get_logger
|
20
21
|
from ..result import Result
|
21
22
|
from ..scheduler import Schedule
|
22
23
|
from ..workflow import Workflow
|
@@ -109,7 +110,7 @@ async def get_workflow_logs(name: str):
|
|
109
110
|
exclude_unset=True,
|
110
111
|
exclude_defaults=True,
|
111
112
|
)
|
112
|
-
for log in
|
113
|
+
for log in get_audit().find_logs(name=name)
|
113
114
|
],
|
114
115
|
}
|
115
116
|
except FileNotFoundError:
|
@@ -122,7 +123,7 @@ async def get_workflow_logs(name: str):
|
|
122
123
|
@workflow_route.get(path="/{name}/logs/{release}")
|
123
124
|
async def get_workflow_release_log(name: str, release: str):
|
124
125
|
try:
|
125
|
-
log:
|
126
|
+
log: Audit = get_audit().find_log_with_release(
|
126
127
|
name=name, release=datetime.strptime(release, "%Y%m%d%H%M%S")
|
127
128
|
)
|
128
129
|
except FileNotFoundError:
|
@@ -0,0 +1,261 @@
|
|
1
|
+
# ------------------------------------------------------------------------------
|
2
|
+
# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
|
3
|
+
# Licensed under the MIT License. See LICENSE in the project root for
|
4
|
+
# license information.
|
5
|
+
# ------------------------------------------------------------------------------
|
6
|
+
"""Audit Log module."""
|
7
|
+
from __future__ import annotations
|
8
|
+
|
9
|
+
import json
|
10
|
+
import os
|
11
|
+
from abc import ABC, abstractmethod
|
12
|
+
from collections.abc import Iterator
|
13
|
+
from datetime import datetime
|
14
|
+
from pathlib import Path
|
15
|
+
from typing import Any, ClassVar, Optional, Union
|
16
|
+
|
17
|
+
from pydantic import BaseModel, Field
|
18
|
+
from pydantic.functional_validators import model_validator
|
19
|
+
from typing_extensions import Self
|
20
|
+
|
21
|
+
from .__types import DictData, TupleStr
|
22
|
+
from .conf import config, get_logger
|
23
|
+
|
24
|
+
logger = get_logger("ddeutil.workflow")
|
25
|
+
|
26
|
+
__all__: TupleStr = (
|
27
|
+
"get_audit",
|
28
|
+
"FileAudit",
|
29
|
+
"SQLiteAudit",
|
30
|
+
"Audit",
|
31
|
+
)
|
32
|
+
|
33
|
+
|
34
|
+
class BaseAudit(BaseModel, ABC):
|
35
|
+
"""Base Audit Pydantic Model with abstraction class property that implement
|
36
|
+
only model fields. This model should to use with inherit to logging
|
37
|
+
subclass like file, sqlite, etc.
|
38
|
+
"""
|
39
|
+
|
40
|
+
name: str = Field(description="A workflow name.")
|
41
|
+
release: datetime = Field(description="A release datetime.")
|
42
|
+
type: str = Field(description="A running type before logging.")
|
43
|
+
context: DictData = Field(
|
44
|
+
default_factory=dict,
|
45
|
+
description="A context that receive from a workflow execution result.",
|
46
|
+
)
|
47
|
+
parent_run_id: Optional[str] = Field(default=None)
|
48
|
+
run_id: str
|
49
|
+
update: datetime = Field(default_factory=datetime.now)
|
50
|
+
execution_time: float = Field(default=0)
|
51
|
+
|
52
|
+
@model_validator(mode="after")
|
53
|
+
def __model_action(self) -> Self:
|
54
|
+
"""Do before the Audit action with WORKFLOW_AUDIT_ENABLE_WRITE env variable.
|
55
|
+
|
56
|
+
:rtype: Self
|
57
|
+
"""
|
58
|
+
if config.enable_write_log:
|
59
|
+
self.do_before()
|
60
|
+
return self
|
61
|
+
|
62
|
+
def do_before(self) -> None: # pragma: no cov
|
63
|
+
"""To something before end up of initial log model."""
|
64
|
+
|
65
|
+
@abstractmethod
|
66
|
+
def save(self, excluded: list[str] | None) -> None: # pragma: no cov
|
67
|
+
"""Save this model logging to target logging store."""
|
68
|
+
raise NotImplementedError("Audit should implement ``save`` method.")
|
69
|
+
|
70
|
+
|
71
|
+
class FileAudit(BaseAudit):
|
72
|
+
"""File Audit Pydantic Model that use to saving log data from result of
|
73
|
+
workflow execution. It inherits from BaseAudit model that implement the
|
74
|
+
``self.save`` method for file.
|
75
|
+
"""
|
76
|
+
|
77
|
+
filename_fmt: ClassVar[str] = (
|
78
|
+
"workflow={name}/release={release:%Y%m%d%H%M%S}"
|
79
|
+
)
|
80
|
+
|
81
|
+
def do_before(self) -> None:
|
82
|
+
"""Create directory of release before saving log file."""
|
83
|
+
self.pointer().mkdir(parents=True, exist_ok=True)
|
84
|
+
|
85
|
+
@classmethod
|
86
|
+
def find_logs(cls, name: str) -> Iterator[Self]:
|
87
|
+
"""Generate the logging data that found from logs path with specific a
|
88
|
+
workflow name.
|
89
|
+
|
90
|
+
:param name: A workflow name that want to search release logging data.
|
91
|
+
|
92
|
+
:rtype: Iterator[Self]
|
93
|
+
"""
|
94
|
+
pointer: Path = config.audit_path / f"workflow={name}"
|
95
|
+
if not pointer.exists():
|
96
|
+
raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
|
97
|
+
|
98
|
+
for file in pointer.glob("./release=*/*.log"):
|
99
|
+
with file.open(mode="r", encoding="utf-8") as f:
|
100
|
+
yield cls.model_validate(obj=json.load(f))
|
101
|
+
|
102
|
+
@classmethod
|
103
|
+
def find_log_with_release(
|
104
|
+
cls,
|
105
|
+
name: str,
|
106
|
+
release: datetime | None = None,
|
107
|
+
) -> Self:
|
108
|
+
"""Return the logging data that found from logs path with specific
|
109
|
+
workflow name and release values. If a release does not pass to an input
|
110
|
+
argument, it will return the latest release from the current log path.
|
111
|
+
|
112
|
+
:param name: A workflow name that want to search log.
|
113
|
+
:param release: A release datetime that want to search log.
|
114
|
+
|
115
|
+
:raise FileNotFoundError:
|
116
|
+
:raise NotImplementedError:
|
117
|
+
|
118
|
+
:rtype: Self
|
119
|
+
"""
|
120
|
+
if release is None:
|
121
|
+
raise NotImplementedError("Find latest log does not implement yet.")
|
122
|
+
|
123
|
+
pointer: Path = (
|
124
|
+
config.audit_path
|
125
|
+
/ f"workflow={name}/release={release:%Y%m%d%H%M%S}"
|
126
|
+
)
|
127
|
+
if not pointer.exists():
|
128
|
+
raise FileNotFoundError(
|
129
|
+
f"Pointer: ./logs/workflow={name}/"
|
130
|
+
f"release={release:%Y%m%d%H%M%S} does not found."
|
131
|
+
)
|
132
|
+
|
133
|
+
with max(pointer.glob("./*.log"), key=os.path.getctime).open(
|
134
|
+
mode="r", encoding="utf-8"
|
135
|
+
) as f:
|
136
|
+
return cls.model_validate(obj=json.load(f))
|
137
|
+
|
138
|
+
@classmethod
|
139
|
+
def is_pointed(cls, name: str, release: datetime) -> bool:
|
140
|
+
"""Check the release log already pointed or created at the destination
|
141
|
+
log path.
|
142
|
+
|
143
|
+
:param name: A workflow name.
|
144
|
+
:param release: A release datetime.
|
145
|
+
|
146
|
+
:rtype: bool
|
147
|
+
:return: Return False if the release log was not pointed or created.
|
148
|
+
"""
|
149
|
+
# NOTE: Return False if enable writing log flag does not set.
|
150
|
+
if not config.enable_write_log:
|
151
|
+
return False
|
152
|
+
|
153
|
+
# NOTE: create pointer path that use the same logic of pointer method.
|
154
|
+
pointer: Path = config.audit_path / cls.filename_fmt.format(
|
155
|
+
name=name, release=release
|
156
|
+
)
|
157
|
+
|
158
|
+
return pointer.exists()
|
159
|
+
|
160
|
+
def pointer(self) -> Path:
|
161
|
+
"""Return release directory path that was generated from model data.
|
162
|
+
|
163
|
+
:rtype: Path
|
164
|
+
"""
|
165
|
+
return config.audit_path / self.filename_fmt.format(
|
166
|
+
name=self.name, release=self.release
|
167
|
+
)
|
168
|
+
|
169
|
+
def save(self, excluded: list[str] | None) -> Self:
|
170
|
+
"""Save logging data that receive a context data from a workflow
|
171
|
+
execution result.
|
172
|
+
|
173
|
+
:param excluded: An excluded list of key name that want to pass in the
|
174
|
+
model_dump method.
|
175
|
+
|
176
|
+
:rtype: Self
|
177
|
+
"""
|
178
|
+
from .utils import cut_id
|
179
|
+
|
180
|
+
# NOTE: Check environ variable was set for real writing.
|
181
|
+
if not config.enable_write_log:
|
182
|
+
logger.debug(
|
183
|
+
f"({cut_id(self.run_id)}) [LOG]: Skip writing log cause "
|
184
|
+
f"config was set"
|
185
|
+
)
|
186
|
+
return self
|
187
|
+
|
188
|
+
log_file: Path = self.pointer() / f"{self.run_id}.log"
|
189
|
+
log_file.write_text(
|
190
|
+
json.dumps(
|
191
|
+
self.model_dump(exclude=excluded),
|
192
|
+
default=str,
|
193
|
+
indent=2,
|
194
|
+
),
|
195
|
+
encoding="utf-8",
|
196
|
+
)
|
197
|
+
return self
|
198
|
+
|
199
|
+
|
200
|
+
class SQLiteAudit(BaseAudit): # pragma: no cov
|
201
|
+
"""SQLite Audit Pydantic Model."""
|
202
|
+
|
203
|
+
@staticmethod
|
204
|
+
def meta() -> dict[str, Any]:
|
205
|
+
return {
|
206
|
+
"table": "workflow_log",
|
207
|
+
"ddl": """
|
208
|
+
workflow str,
|
209
|
+
release int,
|
210
|
+
type str,
|
211
|
+
context json,
|
212
|
+
parent_run_id int,
|
213
|
+
run_id int,
|
214
|
+
update datetime
|
215
|
+
primary key ( run_id )
|
216
|
+
""",
|
217
|
+
}
|
218
|
+
|
219
|
+
def save(self, excluded: list[str] | None) -> SQLiteAudit:
|
220
|
+
"""Save logging data that receive a context data from a workflow
|
221
|
+
execution result.
|
222
|
+
"""
|
223
|
+
from .utils import cut_id
|
224
|
+
|
225
|
+
# NOTE: Check environ variable was set for real writing.
|
226
|
+
if not config.enable_write_log:
|
227
|
+
logger.debug(
|
228
|
+
f"({cut_id(self.run_id)}) [LOG]: Skip writing log cause "
|
229
|
+
f"config was set"
|
230
|
+
)
|
231
|
+
return self
|
232
|
+
|
233
|
+
raise NotImplementedError("SQLiteAudit does not implement yet.")
|
234
|
+
|
235
|
+
|
236
|
+
class RemoteFileAudit(FileAudit): # pragma: no cov
|
237
|
+
"""Remote File Audit Pydantic Model."""
|
238
|
+
|
239
|
+
def save(self, excluded: list[str] | None) -> RemoteFileAudit: ...
|
240
|
+
|
241
|
+
|
242
|
+
class RedisAudit(BaseAudit): # pragma: no cov
|
243
|
+
"""Redis Audit Pydantic Model."""
|
244
|
+
|
245
|
+
def save(self, excluded: list[str] | None) -> RedisAudit: ...
|
246
|
+
|
247
|
+
|
248
|
+
Audit = Union[
|
249
|
+
FileAudit,
|
250
|
+
SQLiteAudit,
|
251
|
+
]
|
252
|
+
|
253
|
+
|
254
|
+
def get_audit() -> type[Audit]: # pragma: no cov
|
255
|
+
"""Get an audit class that dynamic base on the config audit path value.
|
256
|
+
|
257
|
+
:rtype: type[Audit]
|
258
|
+
"""
|
259
|
+
if config.audit_path.is_file():
|
260
|
+
return SQLiteAudit
|
261
|
+
return FileAudit
|