ddeutil-workflow 0.0.78__py3-none-any.whl → 0.0.79__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +1 -5
- ddeutil/workflow/api/routes/job.py +2 -2
- ddeutil/workflow/audits.py +554 -112
- ddeutil/workflow/cli.py +19 -1
- ddeutil/workflow/conf.py +9 -21
- ddeutil/workflow/event.py +15 -6
- ddeutil/workflow/job.py +147 -73
- ddeutil/workflow/params.py +172 -58
- ddeutil/workflow/plugins/__init__.py +0 -0
- ddeutil/workflow/plugins/providers/__init__.py +0 -0
- ddeutil/workflow/plugins/providers/aws.py +908 -0
- ddeutil/workflow/plugins/providers/az.py +1003 -0
- ddeutil/workflow/plugins/providers/container.py +703 -0
- ddeutil/workflow/plugins/providers/gcs.py +826 -0
- ddeutil/workflow/result.py +6 -4
- ddeutil/workflow/reusables.py +151 -95
- ddeutil/workflow/stages.py +28 -28
- ddeutil/workflow/traces.py +1678 -540
- ddeutil/workflow/utils.py +109 -67
- ddeutil/workflow/workflow.py +20 -11
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.79.dist-info}/METADATA +52 -19
- ddeutil_workflow-0.0.79.dist-info/RECORD +36 -0
- ddeutil_workflow-0.0.78.dist-info/RECORD +0 -30
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.79.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.79.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.79.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.79.dist-info}/top_level.txt +0 -0
ddeutil/workflow/audits.py
CHANGED
@@ -9,26 +9,33 @@ This module provides comprehensive audit capabilities for workflow execution
|
|
9
9
|
tracking and monitoring. It supports multiple audit backends for capturing
|
10
10
|
execution metadata, status information, and detailed logging.
|
11
11
|
|
12
|
+
Be noted that, you can set only one audit backend setting for the current
|
13
|
+
run-time because it will conflinct audit data if it set more than one audit
|
14
|
+
backend pointer.
|
15
|
+
|
12
16
|
The audit system tracks workflow, job, and stage executions with configurable
|
13
|
-
storage backends including file-based JSON storage
|
17
|
+
storage backends including file-based JSON storage, database persistence, and
|
18
|
+
more (Up to this package already implement).
|
19
|
+
|
20
|
+
That is mean if you release the workflow with the same release date with force mode,
|
21
|
+
it will overwrite the previous release log. By the way, if you do not pass any
|
22
|
+
release mode, it will not overwrite the previous release log and return the skip
|
23
|
+
status to you because it already releases.
|
14
24
|
|
15
25
|
Classes:
|
16
|
-
|
26
|
+
BaseAudit: Abstract base class for audit implementations
|
17
27
|
FileAudit: File-based audit storage implementation
|
28
|
+
SQLiteAudit: SQLite database audit storage implementation
|
18
29
|
|
19
30
|
Functions:
|
20
31
|
get_audit_model: Factory function for creating audit instances
|
21
32
|
|
22
33
|
Example:
|
23
34
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
audit = get_audit_model(run_id="run-123")
|
29
|
-
audit.info("Workflow execution started")
|
30
|
-
audit.success("Workflow completed successfully")
|
31
|
-
```
|
35
|
+
>>> from ddeutil.workflow.audits import get_audit_model
|
36
|
+
>>> audit = get_audit_model(run_id="run-123")
|
37
|
+
>>> audit.info("Workflow execution started")
|
38
|
+
>>> audit.success("Workflow completed successfully")
|
32
39
|
|
33
40
|
Note:
|
34
41
|
Audit instances are automatically configured based on the workflow
|
@@ -39,29 +46,32 @@ from __future__ import annotations
|
|
39
46
|
import json
|
40
47
|
import logging
|
41
48
|
import os
|
49
|
+
import sqlite3
|
50
|
+
import zlib
|
42
51
|
from abc import ABC, abstractmethod
|
43
52
|
from collections.abc import Iterator
|
44
|
-
from datetime import datetime
|
53
|
+
from datetime import datetime, timedelta
|
45
54
|
from pathlib import Path
|
46
|
-
from typing import ClassVar, Optional, Union
|
47
|
-
from urllib.parse import ParseResult
|
55
|
+
from typing import Any, ClassVar, Optional, TypeVar, Union
|
56
|
+
from urllib.parse import ParseResult, urlparse
|
48
57
|
|
49
58
|
from pydantic import BaseModel, Field
|
50
59
|
from pydantic.functional_serializers import field_serializer
|
51
|
-
from pydantic.functional_validators import model_validator
|
60
|
+
from pydantic.functional_validators import field_validator, model_validator
|
52
61
|
from typing_extensions import Self
|
53
62
|
|
54
63
|
from .__types import DictData
|
55
64
|
from .conf import dynamic
|
56
|
-
from .traces import
|
65
|
+
from .traces import TraceManager, get_trace, set_logging
|
57
66
|
|
58
67
|
logger = logging.getLogger("ddeutil.workflow")
|
59
68
|
|
60
69
|
|
61
70
|
class BaseAudit(BaseModel, ABC):
|
62
|
-
"""Base Audit Pydantic Model with abstraction class property
|
63
|
-
|
64
|
-
|
71
|
+
"""Base Audit Pydantic Model with abstraction class property.
|
72
|
+
|
73
|
+
This model implements only model fields and should be used as a base class
|
74
|
+
for logging subclasses like file, sqlite, etc.
|
65
75
|
"""
|
66
76
|
|
67
77
|
extras: DictData = Field(
|
@@ -84,11 +94,22 @@ class BaseAudit(BaseModel, ABC):
|
|
84
94
|
description="A runs metadata that will use to tracking this audit log.",
|
85
95
|
)
|
86
96
|
|
97
|
+
@field_validator("extras", mode="before")
|
98
|
+
def validate_extras(cls, v: Any) -> DictData:
|
99
|
+
"""Validate extras field to ensure it's a dictionary."""
|
100
|
+
if v is None:
|
101
|
+
return {}
|
102
|
+
return v
|
103
|
+
|
87
104
|
@model_validator(mode="after")
|
88
105
|
def __model_action(self) -> Self:
|
89
|
-
"""
|
106
|
+
"""Perform actions before Audit initialization.
|
107
|
+
|
108
|
+
This method checks the WORKFLOW_AUDIT_ENABLE_WRITE environment variable
|
109
|
+
and performs necessary setup actions.
|
90
110
|
|
91
|
-
:
|
111
|
+
Returns:
|
112
|
+
Self: The validated model instance.
|
92
113
|
"""
|
93
114
|
if dynamic("enable_write_audit", extras=self.extras):
|
94
115
|
self.do_before()
|
@@ -106,6 +127,19 @@ class BaseAudit(BaseModel, ABC):
|
|
106
127
|
*,
|
107
128
|
extras: Optional[DictData] = None,
|
108
129
|
) -> bool:
|
130
|
+
"""Check if audit data exists for the given workflow and release.
|
131
|
+
|
132
|
+
Args:
|
133
|
+
name: The workflow name to check.
|
134
|
+
release: The release datetime to check.
|
135
|
+
extras: Optional extra parameters to override core config.
|
136
|
+
|
137
|
+
Returns:
|
138
|
+
bool: True if audit data exists, False otherwise.
|
139
|
+
|
140
|
+
Raises:
|
141
|
+
NotImplementedError: If the method is not implemented by subclass.
|
142
|
+
"""
|
109
143
|
raise NotImplementedError(
|
110
144
|
"Audit should implement `is_pointed` class-method"
|
111
145
|
)
|
@@ -118,6 +152,18 @@ class BaseAudit(BaseModel, ABC):
|
|
118
152
|
*,
|
119
153
|
extras: Optional[DictData] = None,
|
120
154
|
) -> Iterator[Self]:
|
155
|
+
"""Find all audit data for a given workflow name.
|
156
|
+
|
157
|
+
Args:
|
158
|
+
name: The workflow name to search for.
|
159
|
+
extras: Optional extra parameters to override core config.
|
160
|
+
|
161
|
+
Returns:
|
162
|
+
Iterator[Self]: Iterator of audit instances.
|
163
|
+
|
164
|
+
Raises:
|
165
|
+
NotImplementedError: If the method is not implemented by subclass.
|
166
|
+
"""
|
121
167
|
raise NotImplementedError(
|
122
168
|
"Audit should implement `find_audits` class-method"
|
123
169
|
)
|
@@ -131,23 +177,57 @@ class BaseAudit(BaseModel, ABC):
|
|
131
177
|
*,
|
132
178
|
extras: Optional[DictData] = None,
|
133
179
|
) -> Self:
|
180
|
+
"""Find audit data for a specific workflow and release.
|
181
|
+
|
182
|
+
Args:
|
183
|
+
name: The workflow name to search for.
|
184
|
+
release: Optional release datetime. If None, returns latest release.
|
185
|
+
extras: Optional extra parameters to override core config.
|
186
|
+
|
187
|
+
Returns:
|
188
|
+
Self: The audit instance for the specified workflow and release.
|
189
|
+
|
190
|
+
Raises:
|
191
|
+
NotImplementedError: If the method is not implemented by subclass.
|
192
|
+
"""
|
134
193
|
raise NotImplementedError(
|
135
194
|
"Audit should implement `find_audit_with_release` class-method"
|
136
195
|
)
|
137
196
|
|
138
|
-
def do_before(self) -> None:
|
139
|
-
"""
|
197
|
+
def do_before(self) -> None:
|
198
|
+
"""Perform actions before the end of initial log model setup.
|
199
|
+
|
200
|
+
This method is called during model validation and can be overridden
|
201
|
+
by subclasses to perform custom initialization actions.
|
202
|
+
"""
|
140
203
|
|
141
204
|
@abstractmethod
|
142
|
-
def save(
|
143
|
-
|
205
|
+
def save(
|
206
|
+
self, excluded: Optional[list[str]] = None
|
207
|
+
) -> Self: # pragma: no cov
|
208
|
+
"""Save this model logging to target logging store.
|
209
|
+
|
210
|
+
Args:
|
211
|
+
excluded: Optional list of field names to exclude from saving.
|
212
|
+
|
213
|
+
Returns:
|
214
|
+
Self: The audit instance after saving.
|
215
|
+
|
216
|
+
Raises:
|
217
|
+
NotImplementedError: If the method is not implemented by subclass.
|
218
|
+
"""
|
144
219
|
raise NotImplementedError("Audit should implement `save` method.")
|
145
220
|
|
146
221
|
|
147
222
|
class FileAudit(BaseAudit):
|
148
|
-
"""File Audit Pydantic Model
|
149
|
-
|
150
|
-
|
223
|
+
"""File Audit Pydantic Model for saving log data from workflow execution.
|
224
|
+
|
225
|
+
This class inherits from BaseAudit and implements file-based storage
|
226
|
+
for audit logs. It saves workflow execution results to JSON files
|
227
|
+
in a structured directory hierarchy.
|
228
|
+
|
229
|
+
Attributes:
|
230
|
+
filename_fmt: Class variable defining the filename format for audit files.
|
151
231
|
"""
|
152
232
|
|
153
233
|
filename_fmt: ClassVar[str] = (
|
@@ -156,30 +236,49 @@ class FileAudit(BaseAudit):
|
|
156
236
|
|
157
237
|
@field_serializer("extras")
|
158
238
|
def __serialize_extras(self, value: DictData) -> DictData:
|
239
|
+
"""Serialize extras field, converting ParseResult objects to URLs.
|
240
|
+
|
241
|
+
Args:
|
242
|
+
value: The extras dictionary to serialize.
|
243
|
+
|
244
|
+
Returns:
|
245
|
+
DictData: Serialized extras with ParseResult objects converted to URLs.
|
246
|
+
"""
|
159
247
|
return {
|
160
248
|
k: (v.geturl() if isinstance(v, ParseResult) else v)
|
161
249
|
for k, v in value.items()
|
162
250
|
}
|
163
251
|
|
164
252
|
def do_before(self) -> None:
|
165
|
-
"""Create directory of release before saving log file.
|
253
|
+
"""Create directory of release before saving log file.
|
254
|
+
|
255
|
+
This method ensures the target directory exists before attempting
|
256
|
+
to save audit log files.
|
257
|
+
"""
|
166
258
|
self.pointer().mkdir(parents=True, exist_ok=True)
|
167
259
|
|
168
260
|
@classmethod
|
169
261
|
def find_audits(
|
170
262
|
cls, name: str, *, extras: Optional[DictData] = None
|
171
263
|
) -> Iterator[Self]:
|
172
|
-
"""Generate
|
173
|
-
workflow name.
|
264
|
+
"""Generate audit data found from logs path for a specific workflow name.
|
174
265
|
|
175
|
-
:
|
176
|
-
|
266
|
+
Args:
|
267
|
+
name: The workflow name to search for release logging data.
|
268
|
+
extras: Optional extra parameters to override core config.
|
177
269
|
|
178
|
-
:
|
270
|
+
Returns:
|
271
|
+
Iterator[Self]: Iterator of audit instances found for the workflow.
|
272
|
+
|
273
|
+
Raises:
|
274
|
+
FileNotFoundError: If the workflow directory does not exist.
|
179
275
|
"""
|
180
|
-
|
181
|
-
|
182
|
-
|
276
|
+
audit_url = dynamic("audit_url", extras=extras)
|
277
|
+
if audit_url is None:
|
278
|
+
raise ValueError("audit_url configuration is not set")
|
279
|
+
|
280
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
281
|
+
pointer: Path = Path(audit_url_parse.path) / f"workflow={name}"
|
183
282
|
if not pointer.exists():
|
184
283
|
raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
|
185
284
|
|
@@ -195,34 +294,66 @@ class FileAudit(BaseAudit):
|
|
195
294
|
*,
|
196
295
|
extras: Optional[DictData] = None,
|
197
296
|
) -> Self:
|
198
|
-
"""Return
|
199
|
-
|
200
|
-
|
297
|
+
"""Return audit data found from logs path for specific workflow and release.
|
298
|
+
|
299
|
+
If a release is not provided, it will return the latest release from
|
300
|
+
the current log path.
|
201
301
|
|
202
|
-
:
|
203
|
-
|
204
|
-
|
302
|
+
Args:
|
303
|
+
name: The workflow name to search for.
|
304
|
+
release: Optional release datetime to search for.
|
305
|
+
extras: Optional extra parameters to override core config.
|
205
306
|
|
206
|
-
:
|
207
|
-
|
208
|
-
method. Because this method does not implement latest log.
|
307
|
+
Returns:
|
308
|
+
Self: The audit instance for the specified workflow and release.
|
209
309
|
|
210
|
-
:
|
310
|
+
Raises:
|
311
|
+
FileNotFoundError: If the specified workflow/release directory does not exist.
|
312
|
+
ValueError: If no releases found when release is None.
|
211
313
|
"""
|
212
314
|
if release is None:
|
213
|
-
|
315
|
+
# Find the latest release
|
316
|
+
audit_url = dynamic("audit_url", extras=extras)
|
317
|
+
if audit_url is None:
|
318
|
+
raise ValueError("audit_url configuration is not set")
|
319
|
+
|
320
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
321
|
+
pointer: Path = Path(audit_url_parse.path) / f"workflow={name}"
|
322
|
+
if not pointer.exists():
|
323
|
+
raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
|
324
|
+
|
325
|
+
if not any(pointer.glob("./release=*")):
|
326
|
+
raise FileNotFoundError(
|
327
|
+
f"No releases found for workflow: {name}"
|
328
|
+
)
|
329
|
+
|
330
|
+
# NOTE: Get the latest release directory
|
331
|
+
release_pointer = max(
|
332
|
+
pointer.glob("./release=*"), key=os.path.getctime
|
333
|
+
)
|
334
|
+
else:
|
335
|
+
audit_url = dynamic("audit_url", extras=extras)
|
336
|
+
if audit_url is None:
|
337
|
+
raise ValueError("audit_url configuration is not set")
|
338
|
+
|
339
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
340
|
+
release_pointer: Path = (
|
341
|
+
Path(audit_url_parse.path)
|
342
|
+
/ f"workflow={name}/release={release:%Y%m%d%H%M%S}"
|
343
|
+
)
|
344
|
+
if not release_pointer.exists():
|
345
|
+
raise FileNotFoundError(
|
346
|
+
f"Pointer: {release_pointer} does not found."
|
347
|
+
)
|
214
348
|
|
215
|
-
|
216
|
-
Path(dynamic("audit_url", extras=extras).path)
|
217
|
-
/ f"workflow={name}/release={release:%Y%m%d%H%M%S}"
|
218
|
-
)
|
219
|
-
if not pointer.exists():
|
349
|
+
if not any(release_pointer.glob("./*.log")):
|
220
350
|
raise FileNotFoundError(
|
221
|
-
f"Pointer:
|
222
|
-
f"release={release:%Y%m%d%H%M%S} does not found."
|
351
|
+
f"Pointer: {release_pointer} does not contain any log."
|
223
352
|
)
|
224
353
|
|
225
|
-
latest_file: Path = max(
|
354
|
+
latest_file: Path = max(
|
355
|
+
release_pointer.glob("./*.log"), key=os.path.getctime
|
356
|
+
)
|
226
357
|
with latest_file.open(mode="r", encoding="utf-8") as f:
|
227
358
|
return cls.model_validate(obj=json.load(f))
|
228
359
|
|
@@ -234,46 +365,57 @@ class FileAudit(BaseAudit):
|
|
234
365
|
*,
|
235
366
|
extras: Optional[DictData] = None,
|
236
367
|
) -> bool:
|
237
|
-
"""Check the release log already
|
238
|
-
log path.
|
368
|
+
"""Check if the release log already exists at the destination log path.
|
239
369
|
|
240
|
-
:
|
241
|
-
|
242
|
-
|
370
|
+
Args:
|
371
|
+
name: The workflow name.
|
372
|
+
release: The release datetime.
|
373
|
+
extras: Optional extra parameters to override core config.
|
243
374
|
|
244
|
-
:
|
245
|
-
|
375
|
+
Returns:
|
376
|
+
bool: True if the release log exists, False otherwise.
|
246
377
|
"""
|
247
378
|
# NOTE: Return False if enable writing log flag does not set.
|
248
379
|
if not dynamic("enable_write_audit", extras=extras):
|
249
380
|
return False
|
250
381
|
|
251
382
|
# NOTE: create pointer path that use the same logic of pointer method.
|
252
|
-
|
253
|
-
|
254
|
-
|
383
|
+
audit_url: Optional[str] = dynamic("audit_url", extras=extras)
|
384
|
+
if audit_url is None:
|
385
|
+
return False
|
386
|
+
|
387
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
388
|
+
pointer: Path = Path(audit_url_parse.path) / cls.filename_fmt.format(
|
389
|
+
name=name, release=release
|
390
|
+
)
|
255
391
|
|
256
392
|
return pointer.exists()
|
257
393
|
|
258
394
|
def pointer(self) -> Path:
|
259
|
-
"""Return release directory path
|
395
|
+
"""Return release directory path generated from model data.
|
260
396
|
|
261
|
-
:
|
397
|
+
Returns:
|
398
|
+
Path: The directory path for the current workflow and release.
|
262
399
|
"""
|
263
|
-
|
264
|
-
|
265
|
-
|
400
|
+
audit_url = dynamic("audit_url", extras=self.extras)
|
401
|
+
if audit_url is None:
|
402
|
+
raise ValueError("audit_url configuration is not set")
|
403
|
+
|
404
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
405
|
+
return Path(audit_url_parse.path) / self.filename_fmt.format(
|
406
|
+
name=self.name, release=self.release
|
407
|
+
)
|
266
408
|
|
267
409
|
def save(self, excluded: Optional[list[str]] = None) -> Self:
|
268
|
-
"""Save logging data
|
269
|
-
execution result.
|
410
|
+
"""Save logging data received from workflow execution result.
|
270
411
|
|
271
|
-
:
|
272
|
-
|
412
|
+
Args:
|
413
|
+
excluded: Optional list of field names to exclude from saving.
|
273
414
|
|
274
|
-
:
|
415
|
+
Returns:
|
416
|
+
Self: The audit instance after saving.
|
275
417
|
"""
|
276
|
-
trace:
|
418
|
+
trace: TraceManager = get_trace(
|
277
419
|
self.run_id,
|
278
420
|
parent_run_id=self.parent_run_id,
|
279
421
|
extras=self.extras,
|
@@ -281,15 +423,22 @@ class FileAudit(BaseAudit):
|
|
281
423
|
|
282
424
|
# NOTE: Check environ variable was set for real writing.
|
283
425
|
if not dynamic("enable_write_audit", extras=self.extras):
|
284
|
-
trace.debug("[AUDIT]: Skip writing log cause config was set")
|
426
|
+
trace.debug("[AUDIT]: Skip writing audit log cause config was set.")
|
285
427
|
return self
|
286
428
|
|
287
429
|
log_file: Path = (
|
288
430
|
self.pointer() / f"{self.parent_run_id or self.run_id}.log"
|
289
431
|
)
|
432
|
+
|
433
|
+
# NOTE: Convert excluded list to set for pydantic compatibility
|
434
|
+
exclude_set = set(excluded) if excluded else None
|
435
|
+
trace.info(
|
436
|
+
f"[AUDIT]: Start writing audit log with "
|
437
|
+
f"release: {self.release:%Y%m%d%H%M%S}"
|
438
|
+
)
|
290
439
|
log_file.write_text(
|
291
440
|
json.dumps(
|
292
|
-
self.model_dump(exclude=
|
441
|
+
self.model_dump(exclude=exclude_set),
|
293
442
|
default=str,
|
294
443
|
indent=2,
|
295
444
|
),
|
@@ -297,26 +446,80 @@ class FileAudit(BaseAudit):
|
|
297
446
|
)
|
298
447
|
return self
|
299
448
|
|
449
|
+
def cleanup(self, max_age_days: int = 180) -> int: # pragma: no cov
|
450
|
+
"""Clean up old audit files based on its age.
|
451
|
+
|
452
|
+
Args:
|
453
|
+
max_age_days: Maximum age in days for audit files to keep.
|
454
|
+
|
455
|
+
Returns:
|
456
|
+
int: Number of files cleaned up.
|
457
|
+
"""
|
458
|
+
audit_url = dynamic("audit_url", extras=self.extras)
|
459
|
+
if audit_url is None:
|
460
|
+
return 0
|
461
|
+
|
462
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
463
|
+
base_path = Path(audit_url_parse.path)
|
464
|
+
cutoff_time = datetime.now().timestamp() - (max_age_days * 24 * 3600)
|
465
|
+
cleaned_count: int = 0
|
466
|
+
|
467
|
+
for workflow_dir in base_path.glob("workflow=*"):
|
468
|
+
for release_dir in workflow_dir.glob("release=*"):
|
469
|
+
if release_dir.stat().st_mtime < cutoff_time:
|
470
|
+
import shutil
|
471
|
+
|
472
|
+
shutil.rmtree(release_dir)
|
473
|
+
cleaned_count += 1
|
474
|
+
|
475
|
+
return cleaned_count
|
476
|
+
|
300
477
|
|
301
478
|
class SQLiteAudit(BaseAudit): # pragma: no cov
|
302
|
-
"""SQLite Audit model.
|
479
|
+
"""SQLite Audit model for database-based audit storage.
|
480
|
+
|
481
|
+
This class inherits from BaseAudit and implements SQLite database storage
|
482
|
+
for audit logs with compression support.
|
483
|
+
|
484
|
+
Attributes:
|
485
|
+
table_name: Class variable defining the database table name.
|
486
|
+
schemas: Class variable defining the database schema.
|
487
|
+
"""
|
303
488
|
|
304
489
|
table_name: ClassVar[str] = "audits"
|
305
490
|
schemas: ClassVar[
|
306
491
|
str
|
307
492
|
] = """
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
493
|
+
CREATE TABLE IF NOT EXISTS audits (
|
494
|
+
workflow TEXT NOT NULL
|
495
|
+
, release TEXT NOT NULL
|
496
|
+
, type TEXT NOT NULL
|
497
|
+
, context BLOB NOT NULL
|
498
|
+
, parent_run_id TEXT
|
499
|
+
, run_id TEXT NOT NULL
|
500
|
+
, metadata BLOB NOT NULL
|
501
|
+
, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
502
|
+
, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
503
|
+
, PRIMARY KEY ( workflow, release )
|
504
|
+
)
|
318
505
|
"""
|
319
506
|
|
507
|
+
def _ensure_table_exists(self) -> None:
|
508
|
+
"""Ensure the audit table exists in the database."""
|
509
|
+
audit_url = dynamic("audit_url", extras=self.extras)
|
510
|
+
if audit_url is None or not audit_url.path:
|
511
|
+
raise ValueError(
|
512
|
+
"SQLite audit_url must specify a database file path"
|
513
|
+
)
|
514
|
+
|
515
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
516
|
+
db_path = Path(audit_url_parse.path)
|
517
|
+
db_path.parent.mkdir(parents=True, exist_ok=True)
|
518
|
+
|
519
|
+
with sqlite3.connect(db_path) as conn:
|
520
|
+
conn.execute(self.schemas)
|
521
|
+
conn.commit()
|
522
|
+
|
320
523
|
@classmethod
|
321
524
|
def is_pointed(
|
322
525
|
cls,
|
@@ -324,7 +527,35 @@ class SQLiteAudit(BaseAudit): # pragma: no cov
|
|
324
527
|
release: datetime,
|
325
528
|
*,
|
326
529
|
extras: Optional[DictData] = None,
|
327
|
-
) -> bool:
|
530
|
+
) -> bool:
|
531
|
+
"""Check if audit data exists for the given workflow and release.
|
532
|
+
|
533
|
+
Args:
|
534
|
+
name: The workflow name to check.
|
535
|
+
release: The release datetime to check.
|
536
|
+
extras: Optional extra parameters to override core config.
|
537
|
+
|
538
|
+
Returns:
|
539
|
+
bool: True if audit data exists, False otherwise.
|
540
|
+
"""
|
541
|
+
if not dynamic("enable_write_audit", extras=extras):
|
542
|
+
return False
|
543
|
+
|
544
|
+
audit_url = dynamic("audit_url", extras=extras)
|
545
|
+
if audit_url is None or not audit_url.path:
|
546
|
+
return False
|
547
|
+
|
548
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
549
|
+
db_path = Path(audit_url_parse.path)
|
550
|
+
if not db_path.exists():
|
551
|
+
return False
|
552
|
+
|
553
|
+
with sqlite3.connect(db_path) as conn:
|
554
|
+
cursor = conn.execute(
|
555
|
+
"SELECT COUNT(*) FROM audits WHERE workflow = ? AND release = ?",
|
556
|
+
(name, release.isoformat()),
|
557
|
+
)
|
558
|
+
return cursor.fetchone()[0] > 0
|
328
559
|
|
329
560
|
@classmethod
|
330
561
|
def find_audits(
|
@@ -332,7 +563,45 @@ class SQLiteAudit(BaseAudit): # pragma: no cov
|
|
332
563
|
name: str,
|
333
564
|
*,
|
334
565
|
extras: Optional[DictData] = None,
|
335
|
-
) -> Iterator[Self]:
|
566
|
+
) -> Iterator[Self]:
|
567
|
+
"""Find all audit data for a given workflow name.
|
568
|
+
|
569
|
+
Args:
|
570
|
+
name: The workflow name to search for.
|
571
|
+
extras: Optional extra parameters to override core config.
|
572
|
+
|
573
|
+
Returns:
|
574
|
+
Iterator[Self]: Iterator of audit instances.
|
575
|
+
"""
|
576
|
+
audit_url = dynamic("audit_url", extras=extras)
|
577
|
+
if audit_url is None or not audit_url.path:
|
578
|
+
return
|
579
|
+
|
580
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
581
|
+
db_path = Path(audit_url_parse.path)
|
582
|
+
if not db_path.exists():
|
583
|
+
return
|
584
|
+
|
585
|
+
with sqlite3.connect(db_path) as conn:
|
586
|
+
cursor = conn.execute(
|
587
|
+
"SELECT * FROM audits WHERE workflow = ? ORDER BY release DESC",
|
588
|
+
(name,),
|
589
|
+
)
|
590
|
+
for row in cursor.fetchall():
|
591
|
+
# Decompress context and metadata
|
592
|
+
context = json.loads(cls._decompress_data(row[3]))
|
593
|
+
metadata = json.loads(cls._decompress_data(row[6]))
|
594
|
+
|
595
|
+
yield cls(
|
596
|
+
name=row[0],
|
597
|
+
release=datetime.fromisoformat(row[1]),
|
598
|
+
type=row[2],
|
599
|
+
context=context,
|
600
|
+
parent_run_id=row[4],
|
601
|
+
run_id=row[5],
|
602
|
+
runs_metadata=metadata,
|
603
|
+
extras=extras or {},
|
604
|
+
)
|
336
605
|
|
337
606
|
@classmethod
|
338
607
|
def find_audit_with_release(
|
@@ -341,13 +610,100 @@ class SQLiteAudit(BaseAudit): # pragma: no cov
|
|
341
610
|
release: Optional[datetime] = None,
|
342
611
|
*,
|
343
612
|
extras: Optional[DictData] = None,
|
344
|
-
) -> Self:
|
613
|
+
) -> Self:
|
614
|
+
"""Find audit data for a specific workflow and release.
|
615
|
+
|
616
|
+
Args:
|
617
|
+
name: The workflow name to search for.
|
618
|
+
release: Optional release datetime. If None, returns latest release.
|
619
|
+
extras: Optional extra parameters to override core config.
|
620
|
+
|
621
|
+
Returns:
|
622
|
+
Self: The audit instance for the specified workflow and release.
|
623
|
+
|
624
|
+
Raises:
|
625
|
+
FileNotFoundError: If the specified workflow/release is not found.
|
626
|
+
"""
|
627
|
+
audit_url = dynamic("audit_url", extras=extras)
|
628
|
+
if audit_url is None or not audit_url.path:
|
629
|
+
raise FileNotFoundError("SQLite database not configured")
|
630
|
+
|
631
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
632
|
+
db_path = Path(audit_url_parse.path)
|
633
|
+
if not db_path.exists():
|
634
|
+
raise FileNotFoundError(f"Database file not found: {db_path}")
|
635
|
+
|
636
|
+
with sqlite3.connect(db_path) as conn:
|
637
|
+
if release is None:
|
638
|
+
# Get latest release
|
639
|
+
cursor = conn.execute(
|
640
|
+
"SELECT * FROM audits WHERE workflow = ? ORDER BY release DESC LIMIT 1",
|
641
|
+
(name,),
|
642
|
+
)
|
643
|
+
else:
|
644
|
+
cursor = conn.execute(
|
645
|
+
"SELECT * FROM audits WHERE workflow = ? AND release = ?",
|
646
|
+
(name, release.isoformat()),
|
647
|
+
)
|
648
|
+
|
649
|
+
row = cursor.fetchone()
|
650
|
+
if not row:
|
651
|
+
raise FileNotFoundError(
|
652
|
+
f"Audit not found for workflow: {name}, release: {release}"
|
653
|
+
)
|
654
|
+
|
655
|
+
# Decompress context and metadata
|
656
|
+
context = json.loads(cls._decompress_data(row[3]))
|
657
|
+
metadata = json.loads(cls._decompress_data(row[6]))
|
658
|
+
|
659
|
+
return cls(
|
660
|
+
name=row[0],
|
661
|
+
release=datetime.fromisoformat(row[1]),
|
662
|
+
type=row[2],
|
663
|
+
context=context,
|
664
|
+
parent_run_id=row[4],
|
665
|
+
run_id=row[5],
|
666
|
+
runs_metadata=metadata,
|
667
|
+
extras=extras or {},
|
668
|
+
)
|
669
|
+
|
670
|
+
@staticmethod
|
671
|
+
def _compress_data(data: str) -> bytes:
|
672
|
+
"""Compress audit data for storage efficiency.
|
673
|
+
|
674
|
+
Args:
|
675
|
+
data: JSON string data to compress.
|
345
676
|
|
346
|
-
|
347
|
-
|
348
|
-
execution result.
|
677
|
+
Returns:
|
678
|
+
bytes: Compressed data.
|
349
679
|
"""
|
350
|
-
|
680
|
+
return zlib.compress(data.encode("utf-8"))
|
681
|
+
|
682
|
+
@staticmethod
|
683
|
+
def _decompress_data(data: bytes) -> str:
|
684
|
+
"""Decompress audit data.
|
685
|
+
|
686
|
+
Args:
|
687
|
+
data: Compressed data to decompress.
|
688
|
+
|
689
|
+
Returns:
|
690
|
+
str: Decompressed JSON string.
|
691
|
+
"""
|
692
|
+
return zlib.decompress(data).decode("utf-8")
|
693
|
+
|
694
|
+
def save(self, excluded: Optional[list[str]] = None) -> Self:
|
695
|
+
"""Save logging data received from workflow execution result.
|
696
|
+
|
697
|
+
Args:
|
698
|
+
excluded: Optional list of field names to exclude from saving.
|
699
|
+
|
700
|
+
Returns:
|
701
|
+
Self: The audit instance after saving.
|
702
|
+
|
703
|
+
Raises:
|
704
|
+
ValueError: If SQLite database is not properly configured.
|
705
|
+
"""
|
706
|
+
trace: TraceManager = get_trace(
|
351
707
|
self.run_id,
|
352
708
|
parent_run_id=self.parent_run_id,
|
353
709
|
extras=self.extras,
|
@@ -355,40 +711,126 @@ class SQLiteAudit(BaseAudit): # pragma: no cov
|
|
355
711
|
|
356
712
|
# NOTE: Check environ variable was set for real writing.
|
357
713
|
if not dynamic("enable_write_audit", extras=self.extras):
|
358
|
-
trace.debug("[AUDIT]: Skip writing log cause config was set")
|
714
|
+
trace.debug("[AUDIT]: Skip writing audit log cause config was set.")
|
359
715
|
return self
|
360
716
|
|
361
|
-
|
717
|
+
audit_url = dynamic("audit_url", extras=self.extras)
|
718
|
+
if audit_url is None or not audit_url.path:
|
719
|
+
raise ValueError(
|
720
|
+
"SQLite audit_url must specify a database file path"
|
721
|
+
)
|
722
|
+
|
723
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
724
|
+
db_path = Path(audit_url_parse.path)
|
725
|
+
db_path.parent.mkdir(parents=True, exist_ok=True)
|
726
|
+
|
727
|
+
# Prepare data for storage
|
728
|
+
exclude_set = set(excluded) if excluded else None
|
729
|
+
model_data = self.model_dump(exclude=exclude_set)
|
730
|
+
|
731
|
+
# Compress context and metadata
|
732
|
+
context_blob = self._compress_data(
|
733
|
+
json.dumps(model_data.get("context", {}))
|
734
|
+
)
|
735
|
+
metadata_blob = self._compress_data(
|
736
|
+
json.dumps(model_data.get("runs_metadata", {}))
|
737
|
+
)
|
738
|
+
|
739
|
+
with sqlite3.connect(db_path) as conn:
|
740
|
+
conn.execute(
|
741
|
+
"""
|
742
|
+
INSERT OR REPLACE INTO audits
|
743
|
+
(workflow, release, type, context, parent_run_id, run_id, metadata, updated_at)
|
744
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
745
|
+
""",
|
746
|
+
(
|
747
|
+
self.name,
|
748
|
+
self.release.isoformat(),
|
749
|
+
self.type,
|
750
|
+
context_blob,
|
751
|
+
self.parent_run_id,
|
752
|
+
self.run_id,
|
753
|
+
metadata_blob,
|
754
|
+
),
|
755
|
+
)
|
756
|
+
conn.commit()
|
757
|
+
|
758
|
+
return self
|
759
|
+
|
760
|
+
def cleanup(self, max_age_days: int = 180) -> int:
|
761
|
+
"""Clean up old audit records based on age.
|
762
|
+
|
763
|
+
Args:
|
764
|
+
max_age_days: Maximum age in days for audit records to keep.
|
765
|
+
|
766
|
+
Returns:
|
767
|
+
int: Number of records cleaned up.
|
768
|
+
"""
|
769
|
+
audit_url = dynamic("audit_url", extras=self.extras)
|
770
|
+
if audit_url is None or not audit_url.path:
|
771
|
+
return 0
|
772
|
+
|
773
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
774
|
+
db_path = Path(audit_url_parse.path)
|
775
|
+
if not db_path.exists():
|
776
|
+
return 0
|
777
|
+
|
778
|
+
cutoff_date = (
|
779
|
+
datetime.now() - timedelta(days=max_age_days)
|
780
|
+
).isoformat()
|
781
|
+
|
782
|
+
with sqlite3.connect(db_path) as conn:
|
783
|
+
cursor = conn.execute(
|
784
|
+
"DELETE FROM audits WHERE release < ?", (cutoff_date,)
|
785
|
+
)
|
786
|
+
conn.commit()
|
787
|
+
return cursor.rowcount
|
362
788
|
|
363
789
|
|
364
790
|
Audit = Union[
|
365
791
|
FileAudit,
|
366
792
|
SQLiteAudit,
|
367
|
-
BaseAudit,
|
368
793
|
]
|
794
|
+
AuditType = TypeVar("AuditType", bound=BaseAudit)
|
369
795
|
|
370
796
|
|
371
797
|
def get_audit_model(
|
798
|
+
*,
|
372
799
|
extras: Optional[DictData] = None,
|
373
|
-
) -> type[
|
374
|
-
"""Get an audit model
|
800
|
+
) -> type[AuditType]: # pragma: no cov
|
801
|
+
"""Get an audit model dynamically based on the config audit path value.
|
802
|
+
|
803
|
+
Args:
|
804
|
+
extras: Optional extra parameters to override the core config.
|
805
|
+
This function allow you to pass `audit_model_mapping` for override
|
806
|
+
the audit model object with your custom model.
|
375
807
|
|
376
|
-
:
|
808
|
+
Returns:
|
809
|
+
type[Audit]: The appropriate audit model class based on configuration.
|
377
810
|
|
378
|
-
:
|
811
|
+
Raises:
|
812
|
+
NotImplementedError: If the audit URL scheme is not supported.
|
379
813
|
"""
|
380
|
-
# NOTE: Allow you to override
|
381
|
-
map_audit_models: dict[str, type[
|
814
|
+
# NOTE: Allow you to override audit model by the extra parameter.
|
815
|
+
map_audit_models: dict[str, type[AuditType]] = (extras or {}).get(
|
382
816
|
"audit_model_mapping", {}
|
383
817
|
)
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
|
818
|
+
|
819
|
+
audit_url = dynamic("audit_url", extras=extras)
|
820
|
+
if audit_url is None:
|
821
|
+
return map_audit_models.get("file", FileAudit)
|
822
|
+
|
823
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
824
|
+
if not audit_url_parse.scheme:
|
825
|
+
return map_audit_models.get("file", FileAudit)
|
826
|
+
|
827
|
+
if audit_url_parse.scheme == "sqlite" or (
|
828
|
+
audit_url_parse.scheme == "file"
|
829
|
+
and Path(audit_url_parse.path).is_file()
|
388
830
|
):
|
389
|
-
return map_audit_models.get("sqlite",
|
390
|
-
elif
|
831
|
+
return map_audit_models.get("sqlite", SQLiteAudit)
|
832
|
+
elif audit_url_parse.scheme != "file":
|
391
833
|
raise NotImplementedError(
|
392
|
-
f"Does not implement the audit model support for URL: {
|
834
|
+
f"Does not implement the audit model support for URL: {audit_url_parse}"
|
393
835
|
)
|
394
836
|
return map_audit_models.get("file", FileAudit)
|