ddeutil-workflow 0.0.78__py3-none-any.whl → 0.0.80__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. ddeutil/workflow/__about__.py +1 -1
  2. ddeutil/workflow/__init__.py +2 -6
  3. ddeutil/workflow/api/routes/job.py +2 -2
  4. ddeutil/workflow/api/routes/logs.py +5 -5
  5. ddeutil/workflow/api/routes/workflows.py +3 -3
  6. ddeutil/workflow/audits.py +547 -176
  7. ddeutil/workflow/cli.py +19 -1
  8. ddeutil/workflow/conf.py +10 -20
  9. ddeutil/workflow/event.py +15 -6
  10. ddeutil/workflow/job.py +147 -74
  11. ddeutil/workflow/params.py +172 -58
  12. ddeutil/workflow/plugins/__init__.py +0 -0
  13. ddeutil/workflow/plugins/providers/__init__.py +0 -0
  14. ddeutil/workflow/plugins/providers/aws.py +908 -0
  15. ddeutil/workflow/plugins/providers/az.py +1003 -0
  16. ddeutil/workflow/plugins/providers/container.py +703 -0
  17. ddeutil/workflow/plugins/providers/gcs.py +826 -0
  18. ddeutil/workflow/result.py +6 -4
  19. ddeutil/workflow/reusables.py +151 -95
  20. ddeutil/workflow/stages.py +28 -28
  21. ddeutil/workflow/traces.py +1697 -541
  22. ddeutil/workflow/utils.py +109 -67
  23. ddeutil/workflow/workflow.py +42 -30
  24. {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.80.dist-info}/METADATA +39 -19
  25. ddeutil_workflow-0.0.80.dist-info/RECORD +36 -0
  26. ddeutil_workflow-0.0.78.dist-info/RECORD +0 -30
  27. {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.80.dist-info}/WHEEL +0 -0
  28. {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.80.dist-info}/entry_points.txt +0 -0
  29. {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.80.dist-info}/licenses/LICENSE +0 -0
  30. {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.80.dist-info}/top_level.txt +0 -0
@@ -9,26 +9,31 @@ This module provides comprehensive audit capabilities for workflow execution
9
9
  tracking and monitoring. It supports multiple audit backends for capturing
10
10
  execution metadata, status information, and detailed logging.
11
11
 
12
+ Be noted that, you can set only one audit backend setting for the current
13
+ run-time because it will conflinct audit data if it set more than one audit
14
+ backend pointer.
15
+
12
16
  The audit system tracks workflow, job, and stage executions with configurable
13
- storage backends including file-based JSON storage and database persistence.
17
+ storage backends including file-based JSON storage, database persistence, and
18
+ more (Up to this package already implement).
19
+
20
+ That is mean if you release the workflow with the same release date with force mode,
21
+ it will overwrite the previous release log. By the way, if you do not pass any
22
+ release mode, it will not overwrite the previous release log and return the skip
23
+ status to you because it already releases.
14
24
 
15
25
  Classes:
16
- Audit: Pydantic model for audit data validation
26
+ BaseAudit: Abstract base class for audit implementations
17
27
  FileAudit: File-based audit storage implementation
28
+ SQLiteAudit: SQLite database audit storage implementation
18
29
 
19
30
  Functions:
20
31
  get_audit_model: Factory function for creating audit instances
21
32
 
22
33
  Example:
23
34
 
24
- ```python
25
- from ddeutil.workflow.audits import get_audit_model
26
-
27
- # NOTE: Create file-based Audit.
28
- audit = get_audit_model(run_id="run-123")
29
- audit.info("Workflow execution started")
30
- audit.success("Workflow completed successfully")
31
- ```
35
+ >>> from ddeutil.workflow.audits import get_audit
36
+ >>> audit = get_audit(run_id="run-123")
32
37
 
33
38
  Note:
34
39
  Audit instances are automatically configured based on the workflow
@@ -39,35 +44,27 @@ from __future__ import annotations
39
44
  import json
40
45
  import logging
41
46
  import os
47
+ import sqlite3
48
+ import zlib
42
49
  from abc import ABC, abstractmethod
43
50
  from collections.abc import Iterator
44
- from datetime import datetime
51
+ from datetime import datetime, timedelta
45
52
  from pathlib import Path
46
- from typing import ClassVar, Optional, Union
47
- from urllib.parse import ParseResult
53
+ from typing import Annotated, Any, ClassVar, Literal, Optional, Union
54
+ from urllib.parse import ParseResult, urlparse
48
55
 
49
- from pydantic import BaseModel, Field
50
- from pydantic.functional_serializers import field_serializer
51
- from pydantic.functional_validators import model_validator
56
+ from pydantic import BaseModel, Field, TypeAdapter
57
+ from pydantic.functional_validators import field_validator, model_validator
52
58
  from typing_extensions import Self
53
59
 
54
60
  from .__types import DictData
55
61
  from .conf import dynamic
56
- from .traces import Trace, get_trace, set_logging
62
+ from .traces import TraceManager, get_trace, set_logging
57
63
 
58
64
  logger = logging.getLogger("ddeutil.workflow")
59
65
 
60
66
 
61
- class BaseAudit(BaseModel, ABC):
62
- """Base Audit Pydantic Model with abstraction class property that implement
63
- only model fields. This model should to use with inherit to logging
64
- subclass like file, sqlite, etc.
65
- """
66
-
67
- extras: DictData = Field(
68
- default_factory=dict,
69
- description="An extras parameter that want to override core config",
70
- )
67
+ class AuditData(BaseModel):
71
68
  name: str = Field(description="A workflow name.")
72
69
  release: datetime = Field(description="A release datetime.")
73
70
  type: str = Field(description="A running type before logging.")
@@ -84,11 +81,36 @@ class BaseAudit(BaseModel, ABC):
84
81
  description="A runs metadata that will use to tracking this audit log.",
85
82
  )
86
83
 
84
+
85
+ class BaseAudit(BaseModel, ABC):
86
+ """Base Audit Pydantic Model with abstraction class property.
87
+
88
+ This model implements only model fields and should be used as a base class
89
+ for logging subclasses like file, sqlite, etc.
90
+ """
91
+
92
+ type: str
93
+ extras: DictData = Field(
94
+ default_factory=dict,
95
+ description="An extras parameter that want to override core config",
96
+ )
97
+
98
+ @field_validator("extras", mode="before")
99
+ def validate_extras(cls, v: Any) -> DictData:
100
+ """Validate extras field to ensure it's a dictionary."""
101
+ if v is None:
102
+ return {}
103
+ return v
104
+
87
105
  @model_validator(mode="after")
88
106
  def __model_action(self) -> Self:
89
- """Do before the Audit action with WORKFLOW_AUDIT_ENABLE_WRITE env variable.
107
+ """Perform actions before Audit initialization.
108
+
109
+ This method checks the WORKFLOW_AUDIT_ENABLE_WRITE environment variable
110
+ and performs necessary setup actions.
90
111
 
91
- :rtype: Self
112
+ Returns:
113
+ Self: The validated model instance.
92
114
  """
93
115
  if dynamic("enable_write_audit", extras=self.extras):
94
116
  self.do_before()
@@ -97,199 +119,278 @@ class BaseAudit(BaseModel, ABC):
97
119
  set_logging("ddeutil.workflow")
98
120
  return self
99
121
 
100
- @classmethod
101
122
  @abstractmethod
102
123
  def is_pointed(
103
- cls,
104
- name: str,
105
- release: datetime,
124
+ self,
125
+ data: AuditData,
106
126
  *,
107
127
  extras: Optional[DictData] = None,
108
128
  ) -> bool:
129
+ """Check if audit data exists for the given workflow and release.
130
+
131
+ Args:
132
+ data:
133
+ extras: Optional extra parameters to override core config.
134
+
135
+ Returns:
136
+ bool: True if audit data exists, False otherwise.
137
+
138
+ Raises:
139
+ NotImplementedError: If the method is not implemented by subclass.
140
+ """
109
141
  raise NotImplementedError(
110
142
  "Audit should implement `is_pointed` class-method"
111
143
  )
112
144
 
113
- @classmethod
114
145
  @abstractmethod
115
146
  def find_audits(
116
- cls,
147
+ self,
117
148
  name: str,
118
149
  *,
119
150
  extras: Optional[DictData] = None,
120
151
  ) -> Iterator[Self]:
152
+ """Find all audit data for a given workflow name.
153
+
154
+ Args:
155
+ name: The workflow name to search for.
156
+ extras: Optional extra parameters to override core config.
157
+
158
+ Returns:
159
+ Iterator[Self]: Iterator of audit instances.
160
+
161
+ Raises:
162
+ NotImplementedError: If the method is not implemented by subclass.
163
+ """
121
164
  raise NotImplementedError(
122
165
  "Audit should implement `find_audits` class-method"
123
166
  )
124
167
 
125
- @classmethod
126
168
  @abstractmethod
127
169
  def find_audit_with_release(
128
- cls,
170
+ self,
129
171
  name: str,
130
172
  release: Optional[datetime] = None,
131
173
  *,
132
174
  extras: Optional[DictData] = None,
133
175
  ) -> Self:
176
+ """Find audit data for a specific workflow and release.
177
+
178
+ Args:
179
+ name: The workflow name to search for.
180
+ release: Optional release datetime. If None, returns latest release.
181
+ extras: Optional extra parameters to override core config.
182
+
183
+ Returns:
184
+ Self: The audit instance for the specified workflow and release.
185
+
186
+ Raises:
187
+ NotImplementedError: If the method is not implemented by subclass.
188
+ """
134
189
  raise NotImplementedError(
135
190
  "Audit should implement `find_audit_with_release` class-method"
136
191
  )
137
192
 
138
- def do_before(self) -> None: # pragma: no cov
139
- """To something before end up of initial log model."""
193
+ def do_before(self) -> None:
194
+ """Perform actions before the end of initial log model setup.
195
+
196
+ This method is called during model validation and can be overridden
197
+ by subclasses to perform custom initialization actions.
198
+ """
140
199
 
141
200
  @abstractmethod
142
- def save(self, excluded: Optional[list[str]]) -> None: # pragma: no cov
143
- """Save this model logging to target logging store."""
201
+ def save(
202
+ self, data: Any, excluded: Optional[list[str]] = None
203
+ ) -> Self: # pragma: no cov
204
+ """Save this model logging to target logging store.
205
+
206
+ Args:
207
+ data:
208
+ excluded: Optional list of field names to exclude from saving.
209
+
210
+ Returns:
211
+ Self: The audit instance after saving.
212
+
213
+ Raises:
214
+ NotImplementedError: If the method is not implemented by subclass.
215
+ """
144
216
  raise NotImplementedError("Audit should implement `save` method.")
145
217
 
146
218
 
147
219
  class FileAudit(BaseAudit):
148
- """File Audit Pydantic Model that use to saving log data from result of
149
- workflow execution. It inherits from BaseAudit model that implement the
150
- ``self.save`` method for file.
220
+ """File Audit Pydantic Model for saving log data from workflow execution.
221
+
222
+ This class inherits from BaseAudit and implements file-based storage
223
+ for audit logs. It saves workflow execution results to JSON files
224
+ in a structured directory hierarchy.
225
+
226
+ Attributes:
227
+ filename_fmt: Class variable defining the filename format for audit files.
151
228
  """
152
229
 
153
230
  filename_fmt: ClassVar[str] = (
154
231
  "workflow={name}/release={release:%Y%m%d%H%M%S}"
155
232
  )
156
233
 
157
- @field_serializer("extras")
158
- def __serialize_extras(self, value: DictData) -> DictData:
159
- return {
160
- k: (v.geturl() if isinstance(v, ParseResult) else v)
161
- for k, v in value.items()
162
- }
234
+ type: Literal["file"] = "file"
235
+ path: str = Field(
236
+ default="./audits",
237
+ description="A file path that use to manage audit logs.",
238
+ )
163
239
 
164
240
  def do_before(self) -> None:
165
- """Create directory of release before saving log file."""
166
- self.pointer().mkdir(parents=True, exist_ok=True)
241
+ """Create directory of release before saving log file.
242
+
243
+ This method ensures the target directory exists before attempting
244
+ to save audit log files.
245
+ """
246
+ Path(self.path).mkdir(parents=True, exist_ok=True)
167
247
 
168
- @classmethod
169
248
  def find_audits(
170
- cls, name: str, *, extras: Optional[DictData] = None
171
- ) -> Iterator[Self]:
172
- """Generate the audit data that found from logs path with specific a
173
- workflow name.
249
+ self, name: str, *, extras: Optional[DictData] = None
250
+ ) -> Iterator[AuditData]:
251
+ """Generate audit data found from logs path for a specific workflow name.
252
+
253
+ Args:
254
+ name: The workflow name to search for release logging data.
255
+ extras: Optional extra parameters to override core config.
174
256
 
175
- :param name: A workflow name that want to search release logging data.
176
- :param extras: An extra parameter that want to override core config.
257
+ Returns:
258
+ Iterator[Self]: Iterator of audit instances found for the workflow.
177
259
 
178
- :rtype: Iterator[Self]
260
+ Raises:
261
+ FileNotFoundError: If the workflow directory does not exist.
179
262
  """
180
- pointer: Path = (
181
- Path(dynamic("audit_url", extras=extras).path) / f"workflow={name}"
182
- )
263
+ pointer: Path = Path(self.path) / f"workflow={name}"
183
264
  if not pointer.exists():
184
265
  raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
185
266
 
186
267
  for file in pointer.glob("./release=*/*.log"):
187
268
  with file.open(mode="r", encoding="utf-8") as f:
188
- yield cls.model_validate(obj=json.load(f))
269
+ yield AuditData.model_validate(obj=json.load(f))
189
270
 
190
- @classmethod
191
271
  def find_audit_with_release(
192
- cls,
272
+ self,
193
273
  name: str,
194
274
  release: Optional[datetime] = None,
195
275
  *,
196
276
  extras: Optional[DictData] = None,
197
- ) -> Self:
198
- """Return the audit data that found from logs path with specific
199
- workflow name and release values. If a release does not pass to an input
200
- argument, it will return the latest release from the current log path.
277
+ ) -> AuditData:
278
+ """Return audit data found from logs path for specific workflow and release.
279
+
280
+ If a release is not provided, it will return the latest release from
281
+ the current log path.
201
282
 
202
- :param name: (str) A workflow name that want to search log.
203
- :param release: (datetime) A release datetime that want to search log.
204
- :param extras: An extra parameter that want to override core config.
283
+ Args:
284
+ name: The workflow name to search for.
285
+ release: Optional release datetime to search for.
286
+ extras: Optional extra parameters to override core config.
205
287
 
206
- :raise FileNotFoundError:
207
- :raise NotImplementedError: If an input release does not pass to this
208
- method. Because this method does not implement latest log.
288
+ Returns:
289
+ AuditData: The audit instance for the specified workflow and release.
209
290
 
210
- :rtype: Self
291
+ Raises:
292
+ FileNotFoundError: If the specified workflow/release directory does not exist.
293
+ ValueError: If no releases found when release is None.
211
294
  """
212
295
  if release is None:
213
- raise NotImplementedError("Find latest log does not implement yet.")
296
+ pointer: Path = Path(self.path) / f"workflow={name}"
297
+ if not pointer.exists():
298
+ raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
299
+
300
+ if not any(pointer.glob("./release=*")):
301
+ raise FileNotFoundError(
302
+ f"No releases found for workflow: {name}"
303
+ )
304
+
305
+ # NOTE: Get the latest release directory
306
+ release_pointer = max(
307
+ pointer.glob("./release=*"), key=os.path.getctime
308
+ )
309
+ else:
310
+ release_pointer: Path = (
311
+ Path(self.path)
312
+ / f"workflow={name}/release={release:%Y%m%d%H%M%S}"
313
+ )
314
+ if not release_pointer.exists():
315
+ raise FileNotFoundError(
316
+ f"Pointer: {release_pointer} does not found."
317
+ )
214
318
 
215
- pointer: Path = (
216
- Path(dynamic("audit_url", extras=extras).path)
217
- / f"workflow={name}/release={release:%Y%m%d%H%M%S}"
218
- )
219
- if not pointer.exists():
319
+ if not any(release_pointer.glob("./*.log")):
220
320
  raise FileNotFoundError(
221
- f"Pointer: ./logs/workflow={name}/"
222
- f"release={release:%Y%m%d%H%M%S} does not found."
321
+ f"Pointer: {release_pointer} does not contain any log."
223
322
  )
224
323
 
225
- latest_file: Path = max(pointer.glob("./*.log"), key=os.path.getctime)
324
+ latest_file: Path = max(
325
+ release_pointer.glob("./*.log"), key=os.path.getctime
326
+ )
226
327
  with latest_file.open(mode="r", encoding="utf-8") as f:
227
- return cls.model_validate(obj=json.load(f))
328
+ return AuditData.model_validate(obj=json.load(f))
228
329
 
229
- @classmethod
230
330
  def is_pointed(
231
- cls,
232
- name: str,
233
- release: datetime,
234
- *,
235
- extras: Optional[DictData] = None,
331
+ self, data: AuditData, *, extras: Optional[DictData] = None
236
332
  ) -> bool:
237
- """Check the release log already pointed or created at the destination
238
- log path.
333
+ """Check if the release log already exists at the destination log path.
239
334
 
240
- :param name: (str) A workflow name.
241
- :param release: (datetime) A release datetime.
242
- :param extras: An extra parameter that want to override core config.
335
+ Args:
336
+ data: The workflow name.
337
+ extras: Optional extra parameters to override core config.
243
338
 
244
- :rtype: bool
245
- :return: Return False if the release log was not pointed or created.
339
+ Returns:
340
+ bool: True if the release log exists, False otherwise.
246
341
  """
247
342
  # NOTE: Return False if enable writing log flag does not set.
248
343
  if not dynamic("enable_write_audit", extras=extras):
249
344
  return False
345
+ return self.pointer(data).exists()
250
346
 
251
- # NOTE: create pointer path that use the same logic of pointer method.
252
- pointer: Path = Path(
253
- dynamic("audit_url", extras=extras).path
254
- ) / cls.filename_fmt.format(name=name, release=release)
255
-
256
- return pointer.exists()
347
+ def pointer(self, data: AuditData) -> Path:
348
+ """Return release directory path generated from model data.
257
349
 
258
- def pointer(self) -> Path:
259
- """Return release directory path that was generated from model data.
260
-
261
- :rtype: Path
350
+ Returns:
351
+ Path: The directory path for the current workflow and release.
262
352
  """
263
- return Path(
264
- dynamic("audit_url", extras=self.extras).path
265
- ) / self.filename_fmt.format(name=self.name, release=self.release)
353
+ return Path(self.path) / self.filename_fmt.format(
354
+ name=data.name, release=data.release
355
+ )
266
356
 
267
- def save(self, excluded: Optional[list[str]] = None) -> Self:
268
- """Save logging data that receive a context data from a workflow
269
- execution result.
357
+ def save(self, data: Any, excluded: Optional[list[str]] = None) -> Self:
358
+ """Save logging data received from workflow execution result.
270
359
 
271
- :param excluded: An excluded list of key name that want to pass in the
272
- model_dump method.
360
+ Args:
361
+ data:
362
+ excluded: Optional list of field names to exclude from saving.
273
363
 
274
- :rtype: Self
364
+ Returns:
365
+ Self: The audit instance after saving.
275
366
  """
276
- trace: Trace = get_trace(
277
- self.run_id,
278
- parent_run_id=self.parent_run_id,
367
+ audit = AuditData.model_validate(data)
368
+ trace: TraceManager = get_trace(
369
+ audit.run_id,
370
+ parent_run_id=audit.parent_run_id,
279
371
  extras=self.extras,
280
372
  )
281
373
 
282
374
  # NOTE: Check environ variable was set for real writing.
283
375
  if not dynamic("enable_write_audit", extras=self.extras):
284
- trace.debug("[AUDIT]: Skip writing log cause config was set")
376
+ trace.debug("[AUDIT]: Skip writing audit log cause config was set.")
285
377
  return self
286
378
 
287
- log_file: Path = (
288
- self.pointer() / f"{self.parent_run_id or self.run_id}.log"
379
+ pointer: Path = self.pointer(data=audit)
380
+ if not pointer.exists():
381
+ pointer.mkdir(parents=True)
382
+
383
+ log_file: Path = pointer / f"{audit.parent_run_id or audit.run_id}.log"
384
+
385
+ # NOTE: Convert excluded list to set for pydantic compatibility
386
+ exclude_set = set(excluded) if excluded else None
387
+ trace.info(
388
+ f"[AUDIT]: Start writing audit log with "
389
+ f"release: {audit.release:%Y%m%d%H%M%S}"
289
390
  )
290
391
  log_file.write_text(
291
392
  json.dumps(
292
- self.model_dump(exclude=excluded),
393
+ audit.model_dump(exclude=exclude_set),
293
394
  default=str,
294
395
  indent=2,
295
396
  ),
@@ -297,34 +398,116 @@ class FileAudit(BaseAudit):
297
398
  )
298
399
  return self
299
400
 
401
+ def cleanup(self, max_age_days: int = 180) -> int: # pragma: no cov
402
+ """Clean up old audit files based on its age.
403
+
404
+ Args:
405
+ max_age_days: Maximum age in days for audit files to keep.
406
+
407
+ Returns:
408
+ int: Number of files cleaned up.
409
+ """
410
+ audit_url = dynamic("audit_url", extras=self.extras)
411
+ if audit_url is None:
412
+ return 0
413
+
414
+ audit_url_parse: ParseResult = urlparse(audit_url)
415
+ base_path = Path(audit_url_parse.path)
416
+ cutoff_time = datetime.now().timestamp() - (max_age_days * 24 * 3600)
417
+ cleaned_count: int = 0
418
+
419
+ for workflow_dir in base_path.glob("workflow=*"):
420
+ for release_dir in workflow_dir.glob("release=*"):
421
+ if release_dir.stat().st_mtime < cutoff_time:
422
+ import shutil
423
+
424
+ shutil.rmtree(release_dir)
425
+ cleaned_count += 1
426
+
427
+ return cleaned_count
428
+
300
429
 
301
430
  class SQLiteAudit(BaseAudit): # pragma: no cov
302
- """SQLite Audit model."""
431
+ """SQLite Audit model for database-based audit storage.
432
+
433
+ This class inherits from BaseAudit and implements SQLite database storage
434
+ for audit logs with compression support.
435
+
436
+ Attributes:
437
+ table_name: Class variable defining the database table name.
438
+ schemas: Class variable defining the database schema.
439
+ """
303
440
 
304
441
  table_name: ClassVar[str] = "audits"
305
442
  schemas: ClassVar[
306
443
  str
307
444
  ] = """
308
- workflow str
309
- , release int
310
- , type str
311
- , context JSON
312
- , parent_run_id int
313
- , run_id int
314
- , metadata JSON
315
- , created_at datetime
316
- , updated_at datetime
317
- primary key ( workflow, release )
445
+ CREATE TABLE IF NOT EXISTS audits (
446
+ workflow TEXT NOT NULL
447
+ , release TEXT NOT NULL
448
+ , type TEXT NOT NULL
449
+ , context BLOB NOT NULL
450
+ , parent_run_id TEXT
451
+ , run_id TEXT NOT NULL
452
+ , metadata BLOB NOT NULL
453
+ , created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
454
+ , updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
455
+ , PRIMARY KEY ( workflow, release )
456
+ )
318
457
  """
319
458
 
320
- @classmethod
459
+ type: Literal["sqlite"] = "sqlite"
460
+ path: str
461
+
462
+ def _ensure_table_exists(self) -> None:
463
+ """Ensure the audit table exists in the database."""
464
+ audit_url = dynamic("audit_url", extras=self.extras)
465
+ if audit_url is None or not audit_url.path:
466
+ raise ValueError(
467
+ "SQLite audit_url must specify a database file path"
468
+ )
469
+
470
+ audit_url_parse: ParseResult = urlparse(audit_url)
471
+ db_path = Path(audit_url_parse.path)
472
+ db_path.parent.mkdir(parents=True, exist_ok=True)
473
+
474
+ with sqlite3.connect(db_path) as conn:
475
+ conn.execute(self.schemas)
476
+ conn.commit()
477
+
321
478
  def is_pointed(
322
- cls,
323
- name: str,
324
- release: datetime,
479
+ self,
480
+ data: AuditData,
325
481
  *,
326
482
  extras: Optional[DictData] = None,
327
- ) -> bool: ...
483
+ ) -> bool:
484
+ """Check if audit data exists for the given workflow and release.
485
+
486
+ Args:
487
+ data:
488
+ extras: Optional extra parameters to override core config.
489
+
490
+ Returns:
491
+ bool: True if audit data exists, False otherwise.
492
+ """
493
+ if not dynamic("enable_write_audit", extras=extras):
494
+ return False
495
+
496
+ audit_url = dynamic("audit_url", extras=extras)
497
+ if audit_url is None or not audit_url.path:
498
+ return False
499
+
500
+ audit_url_parse: ParseResult = urlparse(audit_url)
501
+ db_path = Path(audit_url_parse.path)
502
+ if not db_path.exists():
503
+ return False
504
+
505
+ with sqlite3.connect(db_path) as conn:
506
+ cursor = conn.execute(
507
+ "SELECT COUNT(*) FROM audits WHERE workflow = ? AND release = ?",
508
+ (data.name, data.release.isoformat()),
509
+ )
510
+ return cursor.fetchone()[0] > 0
328
511
 
329
512
  @classmethod
330
513
  def find_audits(
@@ -332,7 +515,44 @@ class SQLiteAudit(BaseAudit): # pragma: no cov
332
515
  name: str,
333
516
  *,
334
517
  extras: Optional[DictData] = None,
335
- ) -> Iterator[Self]: ...
518
+ ) -> Iterator[Self]:
519
+ """Find all audit data for a given workflow name.
520
+
521
+ Args:
522
+ name: The workflow name to search for.
523
+ extras: Optional extra parameters to override core config.
524
+
525
+ Returns:
526
+ Iterator[Self]: Iterator of audit instances.
527
+ """
528
+ audit_url = dynamic("audit_url", extras=extras)
529
+ if audit_url is None or not audit_url.path:
530
+ return
531
+
532
+ audit_url_parse: ParseResult = urlparse(audit_url)
533
+ db_path = Path(audit_url_parse.path)
534
+ if not db_path.exists():
535
+ return
536
+
537
+ with sqlite3.connect(db_path) as conn:
538
+ cursor = conn.execute(
539
+ "SELECT * FROM audits WHERE workflow = ? ORDER BY release DESC",
540
+ (name,),
541
+ )
542
+ for row in cursor.fetchall():
543
+ # Decompress context and metadata
544
+ context = json.loads(cls._decompress_data(row[3]))
545
+ metadata = json.loads(cls._decompress_data(row[6]))
546
+
547
+ yield AuditData(
548
+ name=row[0],
549
+ release=datetime.fromisoformat(row[1]),
550
+ type=row[2],
551
+ context=context,
552
+ parent_run_id=row[4],
553
+ run_id=row[5],
554
+ runs_metadata=metadata,
555
+ )
336
556
 
337
557
  @classmethod
338
558
  def find_audit_with_release(
@@ -341,54 +561,205 @@ class SQLiteAudit(BaseAudit): # pragma: no cov
341
561
  release: Optional[datetime] = None,
342
562
  *,
343
563
  extras: Optional[DictData] = None,
344
- ) -> Self: ...
564
+ ) -> AuditData:
565
+ """Find audit data for a specific workflow and release.
566
+
567
+ Args:
568
+ name: The workflow name to search for.
569
+ release: Optional release datetime. If None, returns latest release.
570
+ extras: Optional extra parameters to override core config.
571
+
572
+ Returns:
573
+ Self: The audit instance for the specified workflow and release.
574
+
575
+ Raises:
576
+ FileNotFoundError: If the specified workflow/release is not found.
577
+ """
578
+ audit_url = dynamic("audit_url", extras=extras)
579
+ if audit_url is None or not audit_url.path:
580
+ raise FileNotFoundError("SQLite database not configured")
581
+
582
+ audit_url_parse: ParseResult = urlparse(audit_url)
583
+ db_path = Path(audit_url_parse.path)
584
+ if not db_path.exists():
585
+ raise FileNotFoundError(f"Database file not found: {db_path}")
586
+
587
+ with sqlite3.connect(db_path) as conn:
588
+ if release is None:
589
+ # Get latest release
590
+ cursor = conn.execute(
591
+ "SELECT * FROM audits WHERE workflow = ? ORDER BY release DESC LIMIT 1",
592
+ (name,),
593
+ )
594
+ else:
595
+ cursor = conn.execute(
596
+ "SELECT * FROM audits WHERE workflow = ? AND release = ?",
597
+ (name, release.isoformat()),
598
+ )
599
+
600
+ row = cursor.fetchone()
601
+ if not row:
602
+ raise FileNotFoundError(
603
+ f"Audit not found for workflow: {name}, release: {release}"
604
+ )
605
+
606
+ # Decompress context and metadata
607
+ context = json.loads(cls._decompress_data(row[3]))
608
+ metadata = json.loads(cls._decompress_data(row[6]))
609
+
610
+ return AuditData(
611
+ name=row[0],
612
+ release=datetime.fromisoformat(row[1]),
613
+ type=row[2],
614
+ context=context,
615
+ parent_run_id=row[4],
616
+ run_id=row[5],
617
+ runs_metadata=metadata,
618
+ )
619
+
620
+ @staticmethod
621
+ def _compress_data(data: str) -> bytes:
622
+ """Compress audit data for storage efficiency.
623
+
624
+ Args:
625
+ data: JSON string data to compress.
626
+
627
+ Returns:
628
+ bytes: Compressed data.
629
+ """
630
+ return zlib.compress(data.encode("utf-8"))
631
+
632
+ @staticmethod
633
+ def _decompress_data(data: bytes) -> str:
634
+ """Decompress audit data.
635
+
636
+ Args:
637
+ data: Compressed data to decompress.
638
+
639
+ Returns:
640
+ str: Decompressed JSON string.
641
+ """
642
+ return zlib.decompress(data).decode("utf-8")
643
+
644
+ def save(self, data: Any, excluded: Optional[list[str]] = None) -> Self:
645
+ """Save logging data received from workflow execution result.
646
+
647
+ Args:
648
+ data: Any
649
+ excluded: Optional list of field names to exclude from saving.
650
+
651
+ Returns:
652
+ Self: The audit instance after saving.
345
653
 
346
- def save(self, excluded: Optional[list[str]]) -> SQLiteAudit:
347
- """Save logging data that receive a context data from a workflow
348
- execution result.
654
+ Raises:
655
+ ValueError: If SQLite database is not properly configured.
349
656
  """
350
- trace: Trace = get_trace(
351
- self.run_id,
352
- parent_run_id=self.parent_run_id,
657
+ audit = AuditData.model_validate(data)
658
+ trace: TraceManager = get_trace(
659
+ audit.run_id,
660
+ parent_run_id=audit.parent_run_id,
353
661
  extras=self.extras,
354
662
  )
355
663
 
356
664
  # NOTE: Check environ variable was set for real writing.
357
665
  if not dynamic("enable_write_audit", extras=self.extras):
358
- trace.debug("[AUDIT]: Skip writing log cause config was set")
666
+ trace.debug("[AUDIT]: Skip writing audit log cause config was set.")
359
667
  return self
360
668
 
361
- raise NotImplementedError("SQLiteAudit does not implement yet.")
669
+ audit_url = dynamic("audit_url", extras=self.extras)
670
+ if audit_url is None or not audit_url.path:
671
+ raise ValueError(
672
+ "SQLite audit_url must specify a database file path"
673
+ )
674
+
675
+ audit_url_parse: ParseResult = urlparse(audit_url)
676
+ db_path = Path(audit_url_parse.path)
677
+ db_path.parent.mkdir(parents=True, exist_ok=True)
678
+
679
+ # Prepare data for storage
680
+ exclude_set = set(excluded) if excluded else None
681
+ model_data = audit.model_dump(exclude=exclude_set)
682
+
683
+ # Compress context and metadata
684
+ context_blob = self._compress_data(
685
+ json.dumps(model_data.get("context", {}))
686
+ )
687
+ metadata_blob = self._compress_data(
688
+ json.dumps(model_data.get("runs_metadata", {}))
689
+ )
690
+
691
+ with sqlite3.connect(db_path) as conn:
692
+ conn.execute(
693
+ """
694
+ INSERT OR REPLACE INTO audits
695
+ (workflow, release, type, context, parent_run_id, run_id, metadata, updated_at)
696
+ VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
697
+ """,
698
+ (
699
+ audit.name,
700
+ audit.release.isoformat(),
701
+ audit.type,
702
+ context_blob,
703
+ audit.parent_run_id,
704
+ audit.run_id,
705
+ metadata_blob,
706
+ ),
707
+ )
708
+ conn.commit()
709
+
710
+ return self
711
+
712
+ def cleanup(self, max_age_days: int = 180) -> int:
713
+ """Clean up old audit records based on age.
714
+
715
+ Args:
716
+ max_age_days: Maximum age in days for audit records to keep.
362
717
 
718
+ Returns:
719
+ int: Number of records cleaned up.
720
+ """
721
+ audit_url = dynamic("audit_url", extras=self.extras)
722
+ if audit_url is None or not audit_url.path:
723
+ return 0
724
+
725
+ audit_url_parse: ParseResult = urlparse(audit_url)
726
+ db_path = Path(audit_url_parse.path)
727
+ if not db_path.exists():
728
+ return 0
729
+
730
+ cutoff_date = (
731
+ datetime.now() - timedelta(days=max_age_days)
732
+ ).isoformat()
733
+
734
+ with sqlite3.connect(db_path) as conn:
735
+ cursor = conn.execute(
736
+ "DELETE FROM audits WHERE release < ?", (cutoff_date,)
737
+ )
738
+ conn.commit()
739
+ return cursor.rowcount
363
740
 
364
- Audit = Union[
365
- FileAudit,
366
- SQLiteAudit,
367
- BaseAudit,
741
+
742
+ Audit = Annotated[
743
+ Union[
744
+ FileAudit,
745
+ SQLiteAudit,
746
+ ],
747
+ Field(discriminator="type"),
368
748
  ]
369
749
 
370
750
 
371
- def get_audit_model(
751
+ def get_audit(
752
+ *,
372
753
  extras: Optional[DictData] = None,
373
- ) -> type[Audit]: # pragma: no cov
374
- """Get an audit model that dynamic base on the config audit path value.
754
+ ) -> Audit: # pragma: no cov
755
+ """Get an audit model dynamically based on the config audit path value.
375
756
 
376
- :param extras: An extra parameter that want to override the core config.
757
+ Args:
758
+ extras: Optional extra parameters to override the core config.
377
759
 
378
- :rtype: type[Audit]
760
+ Returns:
761
+ Audit: The appropriate audit model class based on configuration.
379
762
  """
380
- # NOTE: Allow you to override trace model by the extra parameter.
381
- map_audit_models: dict[str, type[Trace]] = extras.get(
382
- "audit_model_mapping", {}
383
- )
384
- url: ParseResult
385
- if (url := dynamic("audit_url", extras=extras)).scheme and (
386
- url.scheme == "sqlite"
387
- or (url.scheme == "file" and Path(url.path).is_file())
388
- ):
389
- return map_audit_models.get("sqlite", FileAudit)
390
- elif url.scheme and url.scheme != "file":
391
- raise NotImplementedError(
392
- f"Does not implement the audit model support for URL: {url}"
393
- )
394
- return map_audit_models.get("file", FileAudit)
763
+ audit_conf = dynamic("audit_conf", extras=extras)
764
+ model = TypeAdapter(Audit).validate_python(audit_conf | {"extras": extras})
765
+ return model