ddeutil-workflow 0.0.78__py3-none-any.whl → 0.0.80__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +2 -6
- ddeutil/workflow/api/routes/job.py +2 -2
- ddeutil/workflow/api/routes/logs.py +5 -5
- ddeutil/workflow/api/routes/workflows.py +3 -3
- ddeutil/workflow/audits.py +547 -176
- ddeutil/workflow/cli.py +19 -1
- ddeutil/workflow/conf.py +10 -20
- ddeutil/workflow/event.py +15 -6
- ddeutil/workflow/job.py +147 -74
- ddeutil/workflow/params.py +172 -58
- ddeutil/workflow/plugins/__init__.py +0 -0
- ddeutil/workflow/plugins/providers/__init__.py +0 -0
- ddeutil/workflow/plugins/providers/aws.py +908 -0
- ddeutil/workflow/plugins/providers/az.py +1003 -0
- ddeutil/workflow/plugins/providers/container.py +703 -0
- ddeutil/workflow/plugins/providers/gcs.py +826 -0
- ddeutil/workflow/result.py +6 -4
- ddeutil/workflow/reusables.py +151 -95
- ddeutil/workflow/stages.py +28 -28
- ddeutil/workflow/traces.py +1697 -541
- ddeutil/workflow/utils.py +109 -67
- ddeutil/workflow/workflow.py +42 -30
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.80.dist-info}/METADATA +39 -19
- ddeutil_workflow-0.0.80.dist-info/RECORD +36 -0
- ddeutil_workflow-0.0.78.dist-info/RECORD +0 -30
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.80.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.80.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.80.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.80.dist-info}/top_level.txt +0 -0
ddeutil/workflow/audits.py
CHANGED
@@ -9,26 +9,31 @@ This module provides comprehensive audit capabilities for workflow execution
|
|
9
9
|
tracking and monitoring. It supports multiple audit backends for capturing
|
10
10
|
execution metadata, status information, and detailed logging.
|
11
11
|
|
12
|
+
Be noted that, you can set only one audit backend setting for the current
|
13
|
+
run-time because it will conflinct audit data if it set more than one audit
|
14
|
+
backend pointer.
|
15
|
+
|
12
16
|
The audit system tracks workflow, job, and stage executions with configurable
|
13
|
-
storage backends including file-based JSON storage
|
17
|
+
storage backends including file-based JSON storage, database persistence, and
|
18
|
+
more (Up to this package already implement).
|
19
|
+
|
20
|
+
That is mean if you release the workflow with the same release date with force mode,
|
21
|
+
it will overwrite the previous release log. By the way, if you do not pass any
|
22
|
+
release mode, it will not overwrite the previous release log and return the skip
|
23
|
+
status to you because it already releases.
|
14
24
|
|
15
25
|
Classes:
|
16
|
-
|
26
|
+
BaseAudit: Abstract base class for audit implementations
|
17
27
|
FileAudit: File-based audit storage implementation
|
28
|
+
SQLiteAudit: SQLite database audit storage implementation
|
18
29
|
|
19
30
|
Functions:
|
20
31
|
get_audit_model: Factory function for creating audit instances
|
21
32
|
|
22
33
|
Example:
|
23
34
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
# NOTE: Create file-based Audit.
|
28
|
-
audit = get_audit_model(run_id="run-123")
|
29
|
-
audit.info("Workflow execution started")
|
30
|
-
audit.success("Workflow completed successfully")
|
31
|
-
```
|
35
|
+
>>> from ddeutil.workflow.audits import get_audit
|
36
|
+
>>> audit = get_audit(run_id="run-123")
|
32
37
|
|
33
38
|
Note:
|
34
39
|
Audit instances are automatically configured based on the workflow
|
@@ -39,35 +44,27 @@ from __future__ import annotations
|
|
39
44
|
import json
|
40
45
|
import logging
|
41
46
|
import os
|
47
|
+
import sqlite3
|
48
|
+
import zlib
|
42
49
|
from abc import ABC, abstractmethod
|
43
50
|
from collections.abc import Iterator
|
44
|
-
from datetime import datetime
|
51
|
+
from datetime import datetime, timedelta
|
45
52
|
from pathlib import Path
|
46
|
-
from typing import ClassVar, Optional, Union
|
47
|
-
from urllib.parse import ParseResult
|
53
|
+
from typing import Annotated, Any, ClassVar, Literal, Optional, Union
|
54
|
+
from urllib.parse import ParseResult, urlparse
|
48
55
|
|
49
|
-
from pydantic import BaseModel, Field
|
50
|
-
from pydantic.
|
51
|
-
from pydantic.functional_validators import model_validator
|
56
|
+
from pydantic import BaseModel, Field, TypeAdapter
|
57
|
+
from pydantic.functional_validators import field_validator, model_validator
|
52
58
|
from typing_extensions import Self
|
53
59
|
|
54
60
|
from .__types import DictData
|
55
61
|
from .conf import dynamic
|
56
|
-
from .traces import
|
62
|
+
from .traces import TraceManager, get_trace, set_logging
|
57
63
|
|
58
64
|
logger = logging.getLogger("ddeutil.workflow")
|
59
65
|
|
60
66
|
|
61
|
-
class
|
62
|
-
"""Base Audit Pydantic Model with abstraction class property that implement
|
63
|
-
only model fields. This model should to use with inherit to logging
|
64
|
-
subclass like file, sqlite, etc.
|
65
|
-
"""
|
66
|
-
|
67
|
-
extras: DictData = Field(
|
68
|
-
default_factory=dict,
|
69
|
-
description="An extras parameter that want to override core config",
|
70
|
-
)
|
67
|
+
class AuditData(BaseModel):
|
71
68
|
name: str = Field(description="A workflow name.")
|
72
69
|
release: datetime = Field(description="A release datetime.")
|
73
70
|
type: str = Field(description="A running type before logging.")
|
@@ -84,11 +81,36 @@ class BaseAudit(BaseModel, ABC):
|
|
84
81
|
description="A runs metadata that will use to tracking this audit log.",
|
85
82
|
)
|
86
83
|
|
84
|
+
|
85
|
+
class BaseAudit(BaseModel, ABC):
|
86
|
+
"""Base Audit Pydantic Model with abstraction class property.
|
87
|
+
|
88
|
+
This model implements only model fields and should be used as a base class
|
89
|
+
for logging subclasses like file, sqlite, etc.
|
90
|
+
"""
|
91
|
+
|
92
|
+
type: str
|
93
|
+
extras: DictData = Field(
|
94
|
+
default_factory=dict,
|
95
|
+
description="An extras parameter that want to override core config",
|
96
|
+
)
|
97
|
+
|
98
|
+
@field_validator("extras", mode="before")
|
99
|
+
def validate_extras(cls, v: Any) -> DictData:
|
100
|
+
"""Validate extras field to ensure it's a dictionary."""
|
101
|
+
if v is None:
|
102
|
+
return {}
|
103
|
+
return v
|
104
|
+
|
87
105
|
@model_validator(mode="after")
|
88
106
|
def __model_action(self) -> Self:
|
89
|
-
"""
|
107
|
+
"""Perform actions before Audit initialization.
|
108
|
+
|
109
|
+
This method checks the WORKFLOW_AUDIT_ENABLE_WRITE environment variable
|
110
|
+
and performs necessary setup actions.
|
90
111
|
|
91
|
-
:
|
112
|
+
Returns:
|
113
|
+
Self: The validated model instance.
|
92
114
|
"""
|
93
115
|
if dynamic("enable_write_audit", extras=self.extras):
|
94
116
|
self.do_before()
|
@@ -97,199 +119,278 @@ class BaseAudit(BaseModel, ABC):
|
|
97
119
|
set_logging("ddeutil.workflow")
|
98
120
|
return self
|
99
121
|
|
100
|
-
@classmethod
|
101
122
|
@abstractmethod
|
102
123
|
def is_pointed(
|
103
|
-
|
104
|
-
|
105
|
-
release: datetime,
|
124
|
+
self,
|
125
|
+
data: AuditData,
|
106
126
|
*,
|
107
127
|
extras: Optional[DictData] = None,
|
108
128
|
) -> bool:
|
129
|
+
"""Check if audit data exists for the given workflow and release.
|
130
|
+
|
131
|
+
Args:
|
132
|
+
data:
|
133
|
+
extras: Optional extra parameters to override core config.
|
134
|
+
|
135
|
+
Returns:
|
136
|
+
bool: True if audit data exists, False otherwise.
|
137
|
+
|
138
|
+
Raises:
|
139
|
+
NotImplementedError: If the method is not implemented by subclass.
|
140
|
+
"""
|
109
141
|
raise NotImplementedError(
|
110
142
|
"Audit should implement `is_pointed` class-method"
|
111
143
|
)
|
112
144
|
|
113
|
-
@classmethod
|
114
145
|
@abstractmethod
|
115
146
|
def find_audits(
|
116
|
-
|
147
|
+
self,
|
117
148
|
name: str,
|
118
149
|
*,
|
119
150
|
extras: Optional[DictData] = None,
|
120
151
|
) -> Iterator[Self]:
|
152
|
+
"""Find all audit data for a given workflow name.
|
153
|
+
|
154
|
+
Args:
|
155
|
+
name: The workflow name to search for.
|
156
|
+
extras: Optional extra parameters to override core config.
|
157
|
+
|
158
|
+
Returns:
|
159
|
+
Iterator[Self]: Iterator of audit instances.
|
160
|
+
|
161
|
+
Raises:
|
162
|
+
NotImplementedError: If the method is not implemented by subclass.
|
163
|
+
"""
|
121
164
|
raise NotImplementedError(
|
122
165
|
"Audit should implement `find_audits` class-method"
|
123
166
|
)
|
124
167
|
|
125
|
-
@classmethod
|
126
168
|
@abstractmethod
|
127
169
|
def find_audit_with_release(
|
128
|
-
|
170
|
+
self,
|
129
171
|
name: str,
|
130
172
|
release: Optional[datetime] = None,
|
131
173
|
*,
|
132
174
|
extras: Optional[DictData] = None,
|
133
175
|
) -> Self:
|
176
|
+
"""Find audit data for a specific workflow and release.
|
177
|
+
|
178
|
+
Args:
|
179
|
+
name: The workflow name to search for.
|
180
|
+
release: Optional release datetime. If None, returns latest release.
|
181
|
+
extras: Optional extra parameters to override core config.
|
182
|
+
|
183
|
+
Returns:
|
184
|
+
Self: The audit instance for the specified workflow and release.
|
185
|
+
|
186
|
+
Raises:
|
187
|
+
NotImplementedError: If the method is not implemented by subclass.
|
188
|
+
"""
|
134
189
|
raise NotImplementedError(
|
135
190
|
"Audit should implement `find_audit_with_release` class-method"
|
136
191
|
)
|
137
192
|
|
138
|
-
def do_before(self) -> None:
|
139
|
-
"""
|
193
|
+
def do_before(self) -> None:
|
194
|
+
"""Perform actions before the end of initial log model setup.
|
195
|
+
|
196
|
+
This method is called during model validation and can be overridden
|
197
|
+
by subclasses to perform custom initialization actions.
|
198
|
+
"""
|
140
199
|
|
141
200
|
@abstractmethod
|
142
|
-
def save(
|
143
|
-
|
201
|
+
def save(
|
202
|
+
self, data: Any, excluded: Optional[list[str]] = None
|
203
|
+
) -> Self: # pragma: no cov
|
204
|
+
"""Save this model logging to target logging store.
|
205
|
+
|
206
|
+
Args:
|
207
|
+
data:
|
208
|
+
excluded: Optional list of field names to exclude from saving.
|
209
|
+
|
210
|
+
Returns:
|
211
|
+
Self: The audit instance after saving.
|
212
|
+
|
213
|
+
Raises:
|
214
|
+
NotImplementedError: If the method is not implemented by subclass.
|
215
|
+
"""
|
144
216
|
raise NotImplementedError("Audit should implement `save` method.")
|
145
217
|
|
146
218
|
|
147
219
|
class FileAudit(BaseAudit):
|
148
|
-
"""File Audit Pydantic Model
|
149
|
-
|
150
|
-
|
220
|
+
"""File Audit Pydantic Model for saving log data from workflow execution.
|
221
|
+
|
222
|
+
This class inherits from BaseAudit and implements file-based storage
|
223
|
+
for audit logs. It saves workflow execution results to JSON files
|
224
|
+
in a structured directory hierarchy.
|
225
|
+
|
226
|
+
Attributes:
|
227
|
+
filename_fmt: Class variable defining the filename format for audit files.
|
151
228
|
"""
|
152
229
|
|
153
230
|
filename_fmt: ClassVar[str] = (
|
154
231
|
"workflow={name}/release={release:%Y%m%d%H%M%S}"
|
155
232
|
)
|
156
233
|
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
}
|
234
|
+
type: Literal["file"] = "file"
|
235
|
+
path: str = Field(
|
236
|
+
default="./audits",
|
237
|
+
description="A file path that use to manage audit logs.",
|
238
|
+
)
|
163
239
|
|
164
240
|
def do_before(self) -> None:
|
165
|
-
"""Create directory of release before saving log file.
|
166
|
-
|
241
|
+
"""Create directory of release before saving log file.
|
242
|
+
|
243
|
+
This method ensures the target directory exists before attempting
|
244
|
+
to save audit log files.
|
245
|
+
"""
|
246
|
+
Path(self.path).mkdir(parents=True, exist_ok=True)
|
167
247
|
|
168
|
-
@classmethod
|
169
248
|
def find_audits(
|
170
|
-
|
171
|
-
) -> Iterator[
|
172
|
-
"""Generate
|
173
|
-
|
249
|
+
self, name: str, *, extras: Optional[DictData] = None
|
250
|
+
) -> Iterator[AuditData]:
|
251
|
+
"""Generate audit data found from logs path for a specific workflow name.
|
252
|
+
|
253
|
+
Args:
|
254
|
+
name: The workflow name to search for release logging data.
|
255
|
+
extras: Optional extra parameters to override core config.
|
174
256
|
|
175
|
-
:
|
176
|
-
|
257
|
+
Returns:
|
258
|
+
Iterator[Self]: Iterator of audit instances found for the workflow.
|
177
259
|
|
178
|
-
:
|
260
|
+
Raises:
|
261
|
+
FileNotFoundError: If the workflow directory does not exist.
|
179
262
|
"""
|
180
|
-
pointer: Path = (
|
181
|
-
Path(dynamic("audit_url", extras=extras).path) / f"workflow={name}"
|
182
|
-
)
|
263
|
+
pointer: Path = Path(self.path) / f"workflow={name}"
|
183
264
|
if not pointer.exists():
|
184
265
|
raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
|
185
266
|
|
186
267
|
for file in pointer.glob("./release=*/*.log"):
|
187
268
|
with file.open(mode="r", encoding="utf-8") as f:
|
188
|
-
yield
|
269
|
+
yield AuditData.model_validate(obj=json.load(f))
|
189
270
|
|
190
|
-
@classmethod
|
191
271
|
def find_audit_with_release(
|
192
|
-
|
272
|
+
self,
|
193
273
|
name: str,
|
194
274
|
release: Optional[datetime] = None,
|
195
275
|
*,
|
196
276
|
extras: Optional[DictData] = None,
|
197
|
-
) ->
|
198
|
-
"""Return
|
199
|
-
|
200
|
-
|
277
|
+
) -> AuditData:
|
278
|
+
"""Return audit data found from logs path for specific workflow and release.
|
279
|
+
|
280
|
+
If a release is not provided, it will return the latest release from
|
281
|
+
the current log path.
|
201
282
|
|
202
|
-
:
|
203
|
-
|
204
|
-
|
283
|
+
Args:
|
284
|
+
name: The workflow name to search for.
|
285
|
+
release: Optional release datetime to search for.
|
286
|
+
extras: Optional extra parameters to override core config.
|
205
287
|
|
206
|
-
:
|
207
|
-
|
208
|
-
method. Because this method does not implement latest log.
|
288
|
+
Returns:
|
289
|
+
AuditData: The audit instance for the specified workflow and release.
|
209
290
|
|
210
|
-
:
|
291
|
+
Raises:
|
292
|
+
FileNotFoundError: If the specified workflow/release directory does not exist.
|
293
|
+
ValueError: If no releases found when release is None.
|
211
294
|
"""
|
212
295
|
if release is None:
|
213
|
-
|
296
|
+
pointer: Path = Path(self.path) / f"workflow={name}"
|
297
|
+
if not pointer.exists():
|
298
|
+
raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
|
299
|
+
|
300
|
+
if not any(pointer.glob("./release=*")):
|
301
|
+
raise FileNotFoundError(
|
302
|
+
f"No releases found for workflow: {name}"
|
303
|
+
)
|
304
|
+
|
305
|
+
# NOTE: Get the latest release directory
|
306
|
+
release_pointer = max(
|
307
|
+
pointer.glob("./release=*"), key=os.path.getctime
|
308
|
+
)
|
309
|
+
else:
|
310
|
+
release_pointer: Path = (
|
311
|
+
Path(self.path)
|
312
|
+
/ f"workflow={name}/release={release:%Y%m%d%H%M%S}"
|
313
|
+
)
|
314
|
+
if not release_pointer.exists():
|
315
|
+
raise FileNotFoundError(
|
316
|
+
f"Pointer: {release_pointer} does not found."
|
317
|
+
)
|
214
318
|
|
215
|
-
|
216
|
-
Path(dynamic("audit_url", extras=extras).path)
|
217
|
-
/ f"workflow={name}/release={release:%Y%m%d%H%M%S}"
|
218
|
-
)
|
219
|
-
if not pointer.exists():
|
319
|
+
if not any(release_pointer.glob("./*.log")):
|
220
320
|
raise FileNotFoundError(
|
221
|
-
f"Pointer:
|
222
|
-
f"release={release:%Y%m%d%H%M%S} does not found."
|
321
|
+
f"Pointer: {release_pointer} does not contain any log."
|
223
322
|
)
|
224
323
|
|
225
|
-
latest_file: Path = max(
|
324
|
+
latest_file: Path = max(
|
325
|
+
release_pointer.glob("./*.log"), key=os.path.getctime
|
326
|
+
)
|
226
327
|
with latest_file.open(mode="r", encoding="utf-8") as f:
|
227
|
-
return
|
328
|
+
return AuditData.model_validate(obj=json.load(f))
|
228
329
|
|
229
|
-
@classmethod
|
230
330
|
def is_pointed(
|
231
|
-
|
232
|
-
name: str,
|
233
|
-
release: datetime,
|
234
|
-
*,
|
235
|
-
extras: Optional[DictData] = None,
|
331
|
+
self, data: AuditData, *, extras: Optional[DictData] = None
|
236
332
|
) -> bool:
|
237
|
-
"""Check the release log already
|
238
|
-
log path.
|
333
|
+
"""Check if the release log already exists at the destination log path.
|
239
334
|
|
240
|
-
:
|
241
|
-
|
242
|
-
|
335
|
+
Args:
|
336
|
+
data: The workflow name.
|
337
|
+
extras: Optional extra parameters to override core config.
|
243
338
|
|
244
|
-
:
|
245
|
-
|
339
|
+
Returns:
|
340
|
+
bool: True if the release log exists, False otherwise.
|
246
341
|
"""
|
247
342
|
# NOTE: Return False if enable writing log flag does not set.
|
248
343
|
if not dynamic("enable_write_audit", extras=extras):
|
249
344
|
return False
|
345
|
+
return self.pointer(data).exists()
|
250
346
|
|
251
|
-
|
252
|
-
|
253
|
-
dynamic("audit_url", extras=extras).path
|
254
|
-
) / cls.filename_fmt.format(name=name, release=release)
|
255
|
-
|
256
|
-
return pointer.exists()
|
347
|
+
def pointer(self, data: AuditData) -> Path:
|
348
|
+
"""Return release directory path generated from model data.
|
257
349
|
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
:rtype: Path
|
350
|
+
Returns:
|
351
|
+
Path: The directory path for the current workflow and release.
|
262
352
|
"""
|
263
|
-
return Path(
|
264
|
-
|
265
|
-
)
|
353
|
+
return Path(self.path) / self.filename_fmt.format(
|
354
|
+
name=data.name, release=data.release
|
355
|
+
)
|
266
356
|
|
267
|
-
def save(self, excluded: Optional[list[str]] = None) -> Self:
|
268
|
-
"""Save logging data
|
269
|
-
execution result.
|
357
|
+
def save(self, data: Any, excluded: Optional[list[str]] = None) -> Self:
|
358
|
+
"""Save logging data received from workflow execution result.
|
270
359
|
|
271
|
-
:
|
272
|
-
|
360
|
+
Args:
|
361
|
+
data:
|
362
|
+
excluded: Optional list of field names to exclude from saving.
|
273
363
|
|
274
|
-
:
|
364
|
+
Returns:
|
365
|
+
Self: The audit instance after saving.
|
275
366
|
"""
|
276
|
-
|
277
|
-
|
278
|
-
|
367
|
+
audit = AuditData.model_validate(data)
|
368
|
+
trace: TraceManager = get_trace(
|
369
|
+
audit.run_id,
|
370
|
+
parent_run_id=audit.parent_run_id,
|
279
371
|
extras=self.extras,
|
280
372
|
)
|
281
373
|
|
282
374
|
# NOTE: Check environ variable was set for real writing.
|
283
375
|
if not dynamic("enable_write_audit", extras=self.extras):
|
284
|
-
trace.debug("[AUDIT]: Skip writing log cause config was set")
|
376
|
+
trace.debug("[AUDIT]: Skip writing audit log cause config was set.")
|
285
377
|
return self
|
286
378
|
|
287
|
-
|
288
|
-
|
379
|
+
pointer: Path = self.pointer(data=audit)
|
380
|
+
if not pointer.exists():
|
381
|
+
pointer.mkdir(parents=True)
|
382
|
+
|
383
|
+
log_file: Path = pointer / f"{audit.parent_run_id or audit.run_id}.log"
|
384
|
+
|
385
|
+
# NOTE: Convert excluded list to set for pydantic compatibility
|
386
|
+
exclude_set = set(excluded) if excluded else None
|
387
|
+
trace.info(
|
388
|
+
f"[AUDIT]: Start writing audit log with "
|
389
|
+
f"release: {audit.release:%Y%m%d%H%M%S}"
|
289
390
|
)
|
290
391
|
log_file.write_text(
|
291
392
|
json.dumps(
|
292
|
-
|
393
|
+
audit.model_dump(exclude=exclude_set),
|
293
394
|
default=str,
|
294
395
|
indent=2,
|
295
396
|
),
|
@@ -297,34 +398,116 @@ class FileAudit(BaseAudit):
|
|
297
398
|
)
|
298
399
|
return self
|
299
400
|
|
401
|
+
def cleanup(self, max_age_days: int = 180) -> int: # pragma: no cov
|
402
|
+
"""Clean up old audit files based on its age.
|
403
|
+
|
404
|
+
Args:
|
405
|
+
max_age_days: Maximum age in days for audit files to keep.
|
406
|
+
|
407
|
+
Returns:
|
408
|
+
int: Number of files cleaned up.
|
409
|
+
"""
|
410
|
+
audit_url = dynamic("audit_url", extras=self.extras)
|
411
|
+
if audit_url is None:
|
412
|
+
return 0
|
413
|
+
|
414
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
415
|
+
base_path = Path(audit_url_parse.path)
|
416
|
+
cutoff_time = datetime.now().timestamp() - (max_age_days * 24 * 3600)
|
417
|
+
cleaned_count: int = 0
|
418
|
+
|
419
|
+
for workflow_dir in base_path.glob("workflow=*"):
|
420
|
+
for release_dir in workflow_dir.glob("release=*"):
|
421
|
+
if release_dir.stat().st_mtime < cutoff_time:
|
422
|
+
import shutil
|
423
|
+
|
424
|
+
shutil.rmtree(release_dir)
|
425
|
+
cleaned_count += 1
|
426
|
+
|
427
|
+
return cleaned_count
|
428
|
+
|
300
429
|
|
301
430
|
class SQLiteAudit(BaseAudit): # pragma: no cov
|
302
|
-
"""SQLite Audit model.
|
431
|
+
"""SQLite Audit model for database-based audit storage.
|
432
|
+
|
433
|
+
This class inherits from BaseAudit and implements SQLite database storage
|
434
|
+
for audit logs with compression support.
|
435
|
+
|
436
|
+
Attributes:
|
437
|
+
table_name: Class variable defining the database table name.
|
438
|
+
schemas: Class variable defining the database schema.
|
439
|
+
"""
|
303
440
|
|
304
441
|
table_name: ClassVar[str] = "audits"
|
305
442
|
schemas: ClassVar[
|
306
443
|
str
|
307
444
|
] = """
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
445
|
+
CREATE TABLE IF NOT EXISTS audits (
|
446
|
+
workflow TEXT NOT NULL
|
447
|
+
, release TEXT NOT NULL
|
448
|
+
, type TEXT NOT NULL
|
449
|
+
, context BLOB NOT NULL
|
450
|
+
, parent_run_id TEXT
|
451
|
+
, run_id TEXT NOT NULL
|
452
|
+
, metadata BLOB NOT NULL
|
453
|
+
, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
454
|
+
, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
455
|
+
, PRIMARY KEY ( workflow, release )
|
456
|
+
)
|
318
457
|
"""
|
319
458
|
|
320
|
-
|
459
|
+
type: Literal["sqlite"] = "sqlite"
|
460
|
+
path: str
|
461
|
+
|
462
|
+
def _ensure_table_exists(self) -> None:
|
463
|
+
"""Ensure the audit table exists in the database."""
|
464
|
+
audit_url = dynamic("audit_url", extras=self.extras)
|
465
|
+
if audit_url is None or not audit_url.path:
|
466
|
+
raise ValueError(
|
467
|
+
"SQLite audit_url must specify a database file path"
|
468
|
+
)
|
469
|
+
|
470
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
471
|
+
db_path = Path(audit_url_parse.path)
|
472
|
+
db_path.parent.mkdir(parents=True, exist_ok=True)
|
473
|
+
|
474
|
+
with sqlite3.connect(db_path) as conn:
|
475
|
+
conn.execute(self.schemas)
|
476
|
+
conn.commit()
|
477
|
+
|
321
478
|
def is_pointed(
|
322
|
-
|
323
|
-
|
324
|
-
release: datetime,
|
479
|
+
self,
|
480
|
+
data: AuditData,
|
325
481
|
*,
|
326
482
|
extras: Optional[DictData] = None,
|
327
|
-
) -> bool:
|
483
|
+
) -> bool:
|
484
|
+
"""Check if audit data exists for the given workflow and release.
|
485
|
+
|
486
|
+
Args:
|
487
|
+
data:
|
488
|
+
extras: Optional extra parameters to override core config.
|
489
|
+
|
490
|
+
Returns:
|
491
|
+
bool: True if audit data exists, False otherwise.
|
492
|
+
"""
|
493
|
+
if not dynamic("enable_write_audit", extras=extras):
|
494
|
+
return False
|
495
|
+
|
496
|
+
audit_url = dynamic("audit_url", extras=extras)
|
497
|
+
if audit_url is None or not audit_url.path:
|
498
|
+
return False
|
499
|
+
|
500
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
501
|
+
db_path = Path(audit_url_parse.path)
|
502
|
+
if not db_path.exists():
|
503
|
+
return False
|
504
|
+
|
505
|
+
with sqlite3.connect(db_path) as conn:
|
506
|
+
cursor = conn.execute(
|
507
|
+
"SELECT COUNT(*) FROM audits WHERE workflow = ? AND release = ?",
|
508
|
+
(data.name, data.release.isoformat()),
|
509
|
+
)
|
510
|
+
return cursor.fetchone()[0] > 0
|
328
511
|
|
329
512
|
@classmethod
|
330
513
|
def find_audits(
|
@@ -332,7 +515,44 @@ class SQLiteAudit(BaseAudit): # pragma: no cov
|
|
332
515
|
name: str,
|
333
516
|
*,
|
334
517
|
extras: Optional[DictData] = None,
|
335
|
-
) -> Iterator[Self]:
|
518
|
+
) -> Iterator[Self]:
|
519
|
+
"""Find all audit data for a given workflow name.
|
520
|
+
|
521
|
+
Args:
|
522
|
+
name: The workflow name to search for.
|
523
|
+
extras: Optional extra parameters to override core config.
|
524
|
+
|
525
|
+
Returns:
|
526
|
+
Iterator[Self]: Iterator of audit instances.
|
527
|
+
"""
|
528
|
+
audit_url = dynamic("audit_url", extras=extras)
|
529
|
+
if audit_url is None or not audit_url.path:
|
530
|
+
return
|
531
|
+
|
532
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
533
|
+
db_path = Path(audit_url_parse.path)
|
534
|
+
if not db_path.exists():
|
535
|
+
return
|
536
|
+
|
537
|
+
with sqlite3.connect(db_path) as conn:
|
538
|
+
cursor = conn.execute(
|
539
|
+
"SELECT * FROM audits WHERE workflow = ? ORDER BY release DESC",
|
540
|
+
(name,),
|
541
|
+
)
|
542
|
+
for row in cursor.fetchall():
|
543
|
+
# Decompress context and metadata
|
544
|
+
context = json.loads(cls._decompress_data(row[3]))
|
545
|
+
metadata = json.loads(cls._decompress_data(row[6]))
|
546
|
+
|
547
|
+
yield AuditData(
|
548
|
+
name=row[0],
|
549
|
+
release=datetime.fromisoformat(row[1]),
|
550
|
+
type=row[2],
|
551
|
+
context=context,
|
552
|
+
parent_run_id=row[4],
|
553
|
+
run_id=row[5],
|
554
|
+
runs_metadata=metadata,
|
555
|
+
)
|
336
556
|
|
337
557
|
@classmethod
|
338
558
|
def find_audit_with_release(
|
@@ -341,54 +561,205 @@ class SQLiteAudit(BaseAudit): # pragma: no cov
|
|
341
561
|
release: Optional[datetime] = None,
|
342
562
|
*,
|
343
563
|
extras: Optional[DictData] = None,
|
344
|
-
) ->
|
564
|
+
) -> AuditData:
|
565
|
+
"""Find audit data for a specific workflow and release.
|
566
|
+
|
567
|
+
Args:
|
568
|
+
name: The workflow name to search for.
|
569
|
+
release: Optional release datetime. If None, returns latest release.
|
570
|
+
extras: Optional extra parameters to override core config.
|
571
|
+
|
572
|
+
Returns:
|
573
|
+
Self: The audit instance for the specified workflow and release.
|
574
|
+
|
575
|
+
Raises:
|
576
|
+
FileNotFoundError: If the specified workflow/release is not found.
|
577
|
+
"""
|
578
|
+
audit_url = dynamic("audit_url", extras=extras)
|
579
|
+
if audit_url is None or not audit_url.path:
|
580
|
+
raise FileNotFoundError("SQLite database not configured")
|
581
|
+
|
582
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
583
|
+
db_path = Path(audit_url_parse.path)
|
584
|
+
if not db_path.exists():
|
585
|
+
raise FileNotFoundError(f"Database file not found: {db_path}")
|
586
|
+
|
587
|
+
with sqlite3.connect(db_path) as conn:
|
588
|
+
if release is None:
|
589
|
+
# Get latest release
|
590
|
+
cursor = conn.execute(
|
591
|
+
"SELECT * FROM audits WHERE workflow = ? ORDER BY release DESC LIMIT 1",
|
592
|
+
(name,),
|
593
|
+
)
|
594
|
+
else:
|
595
|
+
cursor = conn.execute(
|
596
|
+
"SELECT * FROM audits WHERE workflow = ? AND release = ?",
|
597
|
+
(name, release.isoformat()),
|
598
|
+
)
|
599
|
+
|
600
|
+
row = cursor.fetchone()
|
601
|
+
if not row:
|
602
|
+
raise FileNotFoundError(
|
603
|
+
f"Audit not found for workflow: {name}, release: {release}"
|
604
|
+
)
|
605
|
+
|
606
|
+
# Decompress context and metadata
|
607
|
+
context = json.loads(cls._decompress_data(row[3]))
|
608
|
+
metadata = json.loads(cls._decompress_data(row[6]))
|
609
|
+
|
610
|
+
return AuditData(
|
611
|
+
name=row[0],
|
612
|
+
release=datetime.fromisoformat(row[1]),
|
613
|
+
type=row[2],
|
614
|
+
context=context,
|
615
|
+
parent_run_id=row[4],
|
616
|
+
run_id=row[5],
|
617
|
+
runs_metadata=metadata,
|
618
|
+
)
|
619
|
+
|
620
|
+
@staticmethod
|
621
|
+
def _compress_data(data: str) -> bytes:
|
622
|
+
"""Compress audit data for storage efficiency.
|
623
|
+
|
624
|
+
Args:
|
625
|
+
data: JSON string data to compress.
|
626
|
+
|
627
|
+
Returns:
|
628
|
+
bytes: Compressed data.
|
629
|
+
"""
|
630
|
+
return zlib.compress(data.encode("utf-8"))
|
631
|
+
|
632
|
+
@staticmethod
|
633
|
+
def _decompress_data(data: bytes) -> str:
|
634
|
+
"""Decompress audit data.
|
635
|
+
|
636
|
+
Args:
|
637
|
+
data: Compressed data to decompress.
|
638
|
+
|
639
|
+
Returns:
|
640
|
+
str: Decompressed JSON string.
|
641
|
+
"""
|
642
|
+
return zlib.decompress(data).decode("utf-8")
|
643
|
+
|
644
|
+
def save(self, data: Any, excluded: Optional[list[str]] = None) -> Self:
|
645
|
+
"""Save logging data received from workflow execution result.
|
646
|
+
|
647
|
+
Args:
|
648
|
+
data: Any
|
649
|
+
excluded: Optional list of field names to exclude from saving.
|
650
|
+
|
651
|
+
Returns:
|
652
|
+
Self: The audit instance after saving.
|
345
653
|
|
346
|
-
|
347
|
-
|
348
|
-
execution result.
|
654
|
+
Raises:
|
655
|
+
ValueError: If SQLite database is not properly configured.
|
349
656
|
"""
|
350
|
-
|
351
|
-
|
352
|
-
|
657
|
+
audit = AuditData.model_validate(data)
|
658
|
+
trace: TraceManager = get_trace(
|
659
|
+
audit.run_id,
|
660
|
+
parent_run_id=audit.parent_run_id,
|
353
661
|
extras=self.extras,
|
354
662
|
)
|
355
663
|
|
356
664
|
# NOTE: Check environ variable was set for real writing.
|
357
665
|
if not dynamic("enable_write_audit", extras=self.extras):
|
358
|
-
trace.debug("[AUDIT]: Skip writing log cause config was set")
|
666
|
+
trace.debug("[AUDIT]: Skip writing audit log cause config was set.")
|
359
667
|
return self
|
360
668
|
|
361
|
-
|
669
|
+
audit_url = dynamic("audit_url", extras=self.extras)
|
670
|
+
if audit_url is None or not audit_url.path:
|
671
|
+
raise ValueError(
|
672
|
+
"SQLite audit_url must specify a database file path"
|
673
|
+
)
|
674
|
+
|
675
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
676
|
+
db_path = Path(audit_url_parse.path)
|
677
|
+
db_path.parent.mkdir(parents=True, exist_ok=True)
|
678
|
+
|
679
|
+
# Prepare data for storage
|
680
|
+
exclude_set = set(excluded) if excluded else None
|
681
|
+
model_data = audit.model_dump(exclude=exclude_set)
|
682
|
+
|
683
|
+
# Compress context and metadata
|
684
|
+
context_blob = self._compress_data(
|
685
|
+
json.dumps(model_data.get("context", {}))
|
686
|
+
)
|
687
|
+
metadata_blob = self._compress_data(
|
688
|
+
json.dumps(model_data.get("runs_metadata", {}))
|
689
|
+
)
|
690
|
+
|
691
|
+
with sqlite3.connect(db_path) as conn:
|
692
|
+
conn.execute(
|
693
|
+
"""
|
694
|
+
INSERT OR REPLACE INTO audits
|
695
|
+
(workflow, release, type, context, parent_run_id, run_id, metadata, updated_at)
|
696
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
697
|
+
""",
|
698
|
+
(
|
699
|
+
audit.name,
|
700
|
+
audit.release.isoformat(),
|
701
|
+
audit.type,
|
702
|
+
context_blob,
|
703
|
+
audit.parent_run_id,
|
704
|
+
audit.run_id,
|
705
|
+
metadata_blob,
|
706
|
+
),
|
707
|
+
)
|
708
|
+
conn.commit()
|
709
|
+
|
710
|
+
return self
|
711
|
+
|
712
|
+
def cleanup(self, max_age_days: int = 180) -> int:
|
713
|
+
"""Clean up old audit records based on age.
|
714
|
+
|
715
|
+
Args:
|
716
|
+
max_age_days: Maximum age in days for audit records to keep.
|
362
717
|
|
718
|
+
Returns:
|
719
|
+
int: Number of records cleaned up.
|
720
|
+
"""
|
721
|
+
audit_url = dynamic("audit_url", extras=self.extras)
|
722
|
+
if audit_url is None or not audit_url.path:
|
723
|
+
return 0
|
724
|
+
|
725
|
+
audit_url_parse: ParseResult = urlparse(audit_url)
|
726
|
+
db_path = Path(audit_url_parse.path)
|
727
|
+
if not db_path.exists():
|
728
|
+
return 0
|
729
|
+
|
730
|
+
cutoff_date = (
|
731
|
+
datetime.now() - timedelta(days=max_age_days)
|
732
|
+
).isoformat()
|
733
|
+
|
734
|
+
with sqlite3.connect(db_path) as conn:
|
735
|
+
cursor = conn.execute(
|
736
|
+
"DELETE FROM audits WHERE release < ?", (cutoff_date,)
|
737
|
+
)
|
738
|
+
conn.commit()
|
739
|
+
return cursor.rowcount
|
363
740
|
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
741
|
+
|
742
|
+
Audit = Annotated[
|
743
|
+
Union[
|
744
|
+
FileAudit,
|
745
|
+
SQLiteAudit,
|
746
|
+
],
|
747
|
+
Field(discriminator="type"),
|
368
748
|
]
|
369
749
|
|
370
750
|
|
371
|
-
def
|
751
|
+
def get_audit(
|
752
|
+
*,
|
372
753
|
extras: Optional[DictData] = None,
|
373
|
-
) ->
|
374
|
-
"""Get an audit model
|
754
|
+
) -> Audit: # pragma: no cov
|
755
|
+
"""Get an audit model dynamically based on the config audit path value.
|
375
756
|
|
376
|
-
:
|
757
|
+
Args:
|
758
|
+
extras: Optional extra parameters to override the core config.
|
377
759
|
|
378
|
-
:
|
760
|
+
Returns:
|
761
|
+
Audit: The appropriate audit model class based on configuration.
|
379
762
|
"""
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
)
|
384
|
-
url: ParseResult
|
385
|
-
if (url := dynamic("audit_url", extras=extras)).scheme and (
|
386
|
-
url.scheme == "sqlite"
|
387
|
-
or (url.scheme == "file" and Path(url.path).is_file())
|
388
|
-
):
|
389
|
-
return map_audit_models.get("sqlite", FileAudit)
|
390
|
-
elif url.scheme and url.scheme != "file":
|
391
|
-
raise NotImplementedError(
|
392
|
-
f"Does not implement the audit model support for URL: {url}"
|
393
|
-
)
|
394
|
-
return map_audit_models.get("file", FileAudit)
|
763
|
+
audit_conf = dynamic("audit_conf", extras=extras)
|
764
|
+
model = TypeAdapter(Audit).validate_python(audit_conf | {"extras": extras})
|
765
|
+
return model
|