ddeutil-workflow 0.0.78__py3-none-any.whl → 0.0.79__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,26 +9,33 @@ This module provides comprehensive audit capabilities for workflow execution
9
9
  tracking and monitoring. It supports multiple audit backends for capturing
10
10
  execution metadata, status information, and detailed logging.
11
11
 
12
+ Be noted that, you can set only one audit backend setting for the current
13
+ run-time because it will conflinct audit data if it set more than one audit
14
+ backend pointer.
15
+
12
16
  The audit system tracks workflow, job, and stage executions with configurable
13
- storage backends including file-based JSON storage and database persistence.
17
+ storage backends including file-based JSON storage, database persistence, and
18
+ more (Up to this package already implement).
19
+
20
+ That is mean if you release the workflow with the same release date with force mode,
21
+ it will overwrite the previous release log. By the way, if you do not pass any
22
+ release mode, it will not overwrite the previous release log and return the skip
23
+ status to you because it already releases.
14
24
 
15
25
  Classes:
16
- Audit: Pydantic model for audit data validation
26
+ BaseAudit: Abstract base class for audit implementations
17
27
  FileAudit: File-based audit storage implementation
28
+ SQLiteAudit: SQLite database audit storage implementation
18
29
 
19
30
  Functions:
20
31
  get_audit_model: Factory function for creating audit instances
21
32
 
22
33
  Example:
23
34
 
24
- ```python
25
- from ddeutil.workflow.audits import get_audit_model
26
-
27
- # NOTE: Create file-based Audit.
28
- audit = get_audit_model(run_id="run-123")
29
- audit.info("Workflow execution started")
30
- audit.success("Workflow completed successfully")
31
- ```
35
+ >>> from ddeutil.workflow.audits import get_audit_model
36
+ >>> audit = get_audit_model(run_id="run-123")
37
+ >>> audit.info("Workflow execution started")
38
+ >>> audit.success("Workflow completed successfully")
32
39
 
33
40
  Note:
34
41
  Audit instances are automatically configured based on the workflow
@@ -39,29 +46,32 @@ from __future__ import annotations
39
46
  import json
40
47
  import logging
41
48
  import os
49
+ import sqlite3
50
+ import zlib
42
51
  from abc import ABC, abstractmethod
43
52
  from collections.abc import Iterator
44
- from datetime import datetime
53
+ from datetime import datetime, timedelta
45
54
  from pathlib import Path
46
- from typing import ClassVar, Optional, Union
47
- from urllib.parse import ParseResult
55
+ from typing import Any, ClassVar, Optional, TypeVar, Union
56
+ from urllib.parse import ParseResult, urlparse
48
57
 
49
58
  from pydantic import BaseModel, Field
50
59
  from pydantic.functional_serializers import field_serializer
51
- from pydantic.functional_validators import model_validator
60
+ from pydantic.functional_validators import field_validator, model_validator
52
61
  from typing_extensions import Self
53
62
 
54
63
  from .__types import DictData
55
64
  from .conf import dynamic
56
- from .traces import Trace, get_trace, set_logging
65
+ from .traces import TraceManager, get_trace, set_logging
57
66
 
58
67
  logger = logging.getLogger("ddeutil.workflow")
59
68
 
60
69
 
61
70
  class BaseAudit(BaseModel, ABC):
62
- """Base Audit Pydantic Model with abstraction class property that implement
63
- only model fields. This model should to use with inherit to logging
64
- subclass like file, sqlite, etc.
71
+ """Base Audit Pydantic Model with abstraction class property.
72
+
73
+ This model implements only model fields and should be used as a base class
74
+ for logging subclasses like file, sqlite, etc.
65
75
  """
66
76
 
67
77
  extras: DictData = Field(
@@ -84,11 +94,22 @@ class BaseAudit(BaseModel, ABC):
84
94
  description="A runs metadata that will use to tracking this audit log.",
85
95
  )
86
96
 
97
+ @field_validator("extras", mode="before")
98
+ def validate_extras(cls, v: Any) -> DictData:
99
+ """Validate extras field to ensure it's a dictionary."""
100
+ if v is None:
101
+ return {}
102
+ return v
103
+
87
104
  @model_validator(mode="after")
88
105
  def __model_action(self) -> Self:
89
- """Do before the Audit action with WORKFLOW_AUDIT_ENABLE_WRITE env variable.
106
+ """Perform actions before Audit initialization.
107
+
108
+ This method checks the WORKFLOW_AUDIT_ENABLE_WRITE environment variable
109
+ and performs necessary setup actions.
90
110
 
91
- :rtype: Self
111
+ Returns:
112
+ Self: The validated model instance.
92
113
  """
93
114
  if dynamic("enable_write_audit", extras=self.extras):
94
115
  self.do_before()
@@ -106,6 +127,19 @@ class BaseAudit(BaseModel, ABC):
106
127
  *,
107
128
  extras: Optional[DictData] = None,
108
129
  ) -> bool:
130
+ """Check if audit data exists for the given workflow and release.
131
+
132
+ Args:
133
+ name: The workflow name to check.
134
+ release: The release datetime to check.
135
+ extras: Optional extra parameters to override core config.
136
+
137
+ Returns:
138
+ bool: True if audit data exists, False otherwise.
139
+
140
+ Raises:
141
+ NotImplementedError: If the method is not implemented by subclass.
142
+ """
109
143
  raise NotImplementedError(
110
144
  "Audit should implement `is_pointed` class-method"
111
145
  )
@@ -118,6 +152,18 @@ class BaseAudit(BaseModel, ABC):
118
152
  *,
119
153
  extras: Optional[DictData] = None,
120
154
  ) -> Iterator[Self]:
155
+ """Find all audit data for a given workflow name.
156
+
157
+ Args:
158
+ name: The workflow name to search for.
159
+ extras: Optional extra parameters to override core config.
160
+
161
+ Returns:
162
+ Iterator[Self]: Iterator of audit instances.
163
+
164
+ Raises:
165
+ NotImplementedError: If the method is not implemented by subclass.
166
+ """
121
167
  raise NotImplementedError(
122
168
  "Audit should implement `find_audits` class-method"
123
169
  )
@@ -131,23 +177,57 @@ class BaseAudit(BaseModel, ABC):
131
177
  *,
132
178
  extras: Optional[DictData] = None,
133
179
  ) -> Self:
180
+ """Find audit data for a specific workflow and release.
181
+
182
+ Args:
183
+ name: The workflow name to search for.
184
+ release: Optional release datetime. If None, returns latest release.
185
+ extras: Optional extra parameters to override core config.
186
+
187
+ Returns:
188
+ Self: The audit instance for the specified workflow and release.
189
+
190
+ Raises:
191
+ NotImplementedError: If the method is not implemented by subclass.
192
+ """
134
193
  raise NotImplementedError(
135
194
  "Audit should implement `find_audit_with_release` class-method"
136
195
  )
137
196
 
138
- def do_before(self) -> None: # pragma: no cov
139
- """To something before end up of initial log model."""
197
+ def do_before(self) -> None:
198
+ """Perform actions before the end of initial log model setup.
199
+
200
+ This method is called during model validation and can be overridden
201
+ by subclasses to perform custom initialization actions.
202
+ """
140
203
 
141
204
  @abstractmethod
142
- def save(self, excluded: Optional[list[str]]) -> None: # pragma: no cov
143
- """Save this model logging to target logging store."""
205
+ def save(
206
+ self, excluded: Optional[list[str]] = None
207
+ ) -> Self: # pragma: no cov
208
+ """Save this model logging to target logging store.
209
+
210
+ Args:
211
+ excluded: Optional list of field names to exclude from saving.
212
+
213
+ Returns:
214
+ Self: The audit instance after saving.
215
+
216
+ Raises:
217
+ NotImplementedError: If the method is not implemented by subclass.
218
+ """
144
219
  raise NotImplementedError("Audit should implement `save` method.")
145
220
 
146
221
 
147
222
  class FileAudit(BaseAudit):
148
- """File Audit Pydantic Model that use to saving log data from result of
149
- workflow execution. It inherits from BaseAudit model that implement the
150
- ``self.save`` method for file.
223
+ """File Audit Pydantic Model for saving log data from workflow execution.
224
+
225
+ This class inherits from BaseAudit and implements file-based storage
226
+ for audit logs. It saves workflow execution results to JSON files
227
+ in a structured directory hierarchy.
228
+
229
+ Attributes:
230
+ filename_fmt: Class variable defining the filename format for audit files.
151
231
  """
152
232
 
153
233
  filename_fmt: ClassVar[str] = (
@@ -156,30 +236,49 @@ class FileAudit(BaseAudit):
156
236
 
157
237
  @field_serializer("extras")
158
238
  def __serialize_extras(self, value: DictData) -> DictData:
239
+ """Serialize extras field, converting ParseResult objects to URLs.
240
+
241
+ Args:
242
+ value: The extras dictionary to serialize.
243
+
244
+ Returns:
245
+ DictData: Serialized extras with ParseResult objects converted to URLs.
246
+ """
159
247
  return {
160
248
  k: (v.geturl() if isinstance(v, ParseResult) else v)
161
249
  for k, v in value.items()
162
250
  }
163
251
 
164
252
  def do_before(self) -> None:
165
- """Create directory of release before saving log file."""
253
+ """Create directory of release before saving log file.
254
+
255
+ This method ensures the target directory exists before attempting
256
+ to save audit log files.
257
+ """
166
258
  self.pointer().mkdir(parents=True, exist_ok=True)
167
259
 
168
260
  @classmethod
169
261
  def find_audits(
170
262
  cls, name: str, *, extras: Optional[DictData] = None
171
263
  ) -> Iterator[Self]:
172
- """Generate the audit data that found from logs path with specific a
173
- workflow name.
264
+ """Generate audit data found from logs path for a specific workflow name.
174
265
 
175
- :param name: A workflow name that want to search release logging data.
176
- :param extras: An extra parameter that want to override core config.
266
+ Args:
267
+ name: The workflow name to search for release logging data.
268
+ extras: Optional extra parameters to override core config.
177
269
 
178
- :rtype: Iterator[Self]
270
+ Returns:
271
+ Iterator[Self]: Iterator of audit instances found for the workflow.
272
+
273
+ Raises:
274
+ FileNotFoundError: If the workflow directory does not exist.
179
275
  """
180
- pointer: Path = (
181
- Path(dynamic("audit_url", extras=extras).path) / f"workflow={name}"
182
- )
276
+ audit_url = dynamic("audit_url", extras=extras)
277
+ if audit_url is None:
278
+ raise ValueError("audit_url configuration is not set")
279
+
280
+ audit_url_parse: ParseResult = urlparse(audit_url)
281
+ pointer: Path = Path(audit_url_parse.path) / f"workflow={name}"
183
282
  if not pointer.exists():
184
283
  raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
185
284
 
@@ -195,34 +294,66 @@ class FileAudit(BaseAudit):
195
294
  *,
196
295
  extras: Optional[DictData] = None,
197
296
  ) -> Self:
198
- """Return the audit data that found from logs path with specific
199
- workflow name and release values. If a release does not pass to an input
200
- argument, it will return the latest release from the current log path.
297
+ """Return audit data found from logs path for specific workflow and release.
298
+
299
+ If a release is not provided, it will return the latest release from
300
+ the current log path.
201
301
 
202
- :param name: (str) A workflow name that want to search log.
203
- :param release: (datetime) A release datetime that want to search log.
204
- :param extras: An extra parameter that want to override core config.
302
+ Args:
303
+ name: The workflow name to search for.
304
+ release: Optional release datetime to search for.
305
+ extras: Optional extra parameters to override core config.
205
306
 
206
- :raise FileNotFoundError:
207
- :raise NotImplementedError: If an input release does not pass to this
208
- method. Because this method does not implement latest log.
307
+ Returns:
308
+ Self: The audit instance for the specified workflow and release.
209
309
 
210
- :rtype: Self
310
+ Raises:
311
+ FileNotFoundError: If the specified workflow/release directory does not exist.
312
+ ValueError: If no releases found when release is None.
211
313
  """
212
314
  if release is None:
213
- raise NotImplementedError("Find latest log does not implement yet.")
315
+ # Find the latest release
316
+ audit_url = dynamic("audit_url", extras=extras)
317
+ if audit_url is None:
318
+ raise ValueError("audit_url configuration is not set")
319
+
320
+ audit_url_parse: ParseResult = urlparse(audit_url)
321
+ pointer: Path = Path(audit_url_parse.path) / f"workflow={name}"
322
+ if not pointer.exists():
323
+ raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
324
+
325
+ if not any(pointer.glob("./release=*")):
326
+ raise FileNotFoundError(
327
+ f"No releases found for workflow: {name}"
328
+ )
329
+
330
+ # NOTE: Get the latest release directory
331
+ release_pointer = max(
332
+ pointer.glob("./release=*"), key=os.path.getctime
333
+ )
334
+ else:
335
+ audit_url = dynamic("audit_url", extras=extras)
336
+ if audit_url is None:
337
+ raise ValueError("audit_url configuration is not set")
338
+
339
+ audit_url_parse: ParseResult = urlparse(audit_url)
340
+ release_pointer: Path = (
341
+ Path(audit_url_parse.path)
342
+ / f"workflow={name}/release={release:%Y%m%d%H%M%S}"
343
+ )
344
+ if not release_pointer.exists():
345
+ raise FileNotFoundError(
346
+ f"Pointer: {release_pointer} does not found."
347
+ )
214
348
 
215
- pointer: Path = (
216
- Path(dynamic("audit_url", extras=extras).path)
217
- / f"workflow={name}/release={release:%Y%m%d%H%M%S}"
218
- )
219
- if not pointer.exists():
349
+ if not any(release_pointer.glob("./*.log")):
220
350
  raise FileNotFoundError(
221
- f"Pointer: ./logs/workflow={name}/"
222
- f"release={release:%Y%m%d%H%M%S} does not found."
351
+ f"Pointer: {release_pointer} does not contain any log."
223
352
  )
224
353
 
225
- latest_file: Path = max(pointer.glob("./*.log"), key=os.path.getctime)
354
+ latest_file: Path = max(
355
+ release_pointer.glob("./*.log"), key=os.path.getctime
356
+ )
226
357
  with latest_file.open(mode="r", encoding="utf-8") as f:
227
358
  return cls.model_validate(obj=json.load(f))
228
359
 
@@ -234,46 +365,57 @@ class FileAudit(BaseAudit):
234
365
  *,
235
366
  extras: Optional[DictData] = None,
236
367
  ) -> bool:
237
- """Check the release log already pointed or created at the destination
238
- log path.
368
+ """Check if the release log already exists at the destination log path.
239
369
 
240
- :param name: (str) A workflow name.
241
- :param release: (datetime) A release datetime.
242
- :param extras: An extra parameter that want to override core config.
370
+ Args:
371
+ name: The workflow name.
372
+ release: The release datetime.
373
+ extras: Optional extra parameters to override core config.
243
374
 
244
- :rtype: bool
245
- :return: Return False if the release log was not pointed or created.
375
+ Returns:
376
+ bool: True if the release log exists, False otherwise.
246
377
  """
247
378
  # NOTE: Return False if enable writing log flag does not set.
248
379
  if not dynamic("enable_write_audit", extras=extras):
249
380
  return False
250
381
 
251
382
  # NOTE: create pointer path that use the same logic of pointer method.
252
- pointer: Path = Path(
253
- dynamic("audit_url", extras=extras).path
254
- ) / cls.filename_fmt.format(name=name, release=release)
383
+ audit_url: Optional[str] = dynamic("audit_url", extras=extras)
384
+ if audit_url is None:
385
+ return False
386
+
387
+ audit_url_parse: ParseResult = urlparse(audit_url)
388
+ pointer: Path = Path(audit_url_parse.path) / cls.filename_fmt.format(
389
+ name=name, release=release
390
+ )
255
391
 
256
392
  return pointer.exists()
257
393
 
258
394
  def pointer(self) -> Path:
259
- """Return release directory path that was generated from model data.
395
+ """Return release directory path generated from model data.
260
396
 
261
- :rtype: Path
397
+ Returns:
398
+ Path: The directory path for the current workflow and release.
262
399
  """
263
- return Path(
264
- dynamic("audit_url", extras=self.extras).path
265
- ) / self.filename_fmt.format(name=self.name, release=self.release)
400
+ audit_url = dynamic("audit_url", extras=self.extras)
401
+ if audit_url is None:
402
+ raise ValueError("audit_url configuration is not set")
403
+
404
+ audit_url_parse: ParseResult = urlparse(audit_url)
405
+ return Path(audit_url_parse.path) / self.filename_fmt.format(
406
+ name=self.name, release=self.release
407
+ )
266
408
 
267
409
  def save(self, excluded: Optional[list[str]] = None) -> Self:
268
- """Save logging data that receive a context data from a workflow
269
- execution result.
410
+ """Save logging data received from workflow execution result.
270
411
 
271
- :param excluded: An excluded list of key name that want to pass in the
272
- model_dump method.
412
+ Args:
413
+ excluded: Optional list of field names to exclude from saving.
273
414
 
274
- :rtype: Self
415
+ Returns:
416
+ Self: The audit instance after saving.
275
417
  """
276
- trace: Trace = get_trace(
418
+ trace: TraceManager = get_trace(
277
419
  self.run_id,
278
420
  parent_run_id=self.parent_run_id,
279
421
  extras=self.extras,
@@ -281,15 +423,22 @@ class FileAudit(BaseAudit):
281
423
 
282
424
  # NOTE: Check environ variable was set for real writing.
283
425
  if not dynamic("enable_write_audit", extras=self.extras):
284
- trace.debug("[AUDIT]: Skip writing log cause config was set")
426
+ trace.debug("[AUDIT]: Skip writing audit log cause config was set.")
285
427
  return self
286
428
 
287
429
  log_file: Path = (
288
430
  self.pointer() / f"{self.parent_run_id or self.run_id}.log"
289
431
  )
432
+
433
+ # NOTE: Convert excluded list to set for pydantic compatibility
434
+ exclude_set = set(excluded) if excluded else None
435
+ trace.info(
436
+ f"[AUDIT]: Start writing audit log with "
437
+ f"release: {self.release:%Y%m%d%H%M%S}"
438
+ )
290
439
  log_file.write_text(
291
440
  json.dumps(
292
- self.model_dump(exclude=excluded),
441
+ self.model_dump(exclude=exclude_set),
293
442
  default=str,
294
443
  indent=2,
295
444
  ),
@@ -297,26 +446,80 @@ class FileAudit(BaseAudit):
297
446
  )
298
447
  return self
299
448
 
449
+ def cleanup(self, max_age_days: int = 180) -> int: # pragma: no cov
450
+ """Clean up old audit files based on its age.
451
+
452
+ Args:
453
+ max_age_days: Maximum age in days for audit files to keep.
454
+
455
+ Returns:
456
+ int: Number of files cleaned up.
457
+ """
458
+ audit_url = dynamic("audit_url", extras=self.extras)
459
+ if audit_url is None:
460
+ return 0
461
+
462
+ audit_url_parse: ParseResult = urlparse(audit_url)
463
+ base_path = Path(audit_url_parse.path)
464
+ cutoff_time = datetime.now().timestamp() - (max_age_days * 24 * 3600)
465
+ cleaned_count: int = 0
466
+
467
+ for workflow_dir in base_path.glob("workflow=*"):
468
+ for release_dir in workflow_dir.glob("release=*"):
469
+ if release_dir.stat().st_mtime < cutoff_time:
470
+ import shutil
471
+
472
+ shutil.rmtree(release_dir)
473
+ cleaned_count += 1
474
+
475
+ return cleaned_count
476
+
300
477
 
301
478
  class SQLiteAudit(BaseAudit): # pragma: no cov
302
- """SQLite Audit model."""
479
+ """SQLite Audit model for database-based audit storage.
480
+
481
+ This class inherits from BaseAudit and implements SQLite database storage
482
+ for audit logs with compression support.
483
+
484
+ Attributes:
485
+ table_name: Class variable defining the database table name.
486
+ schemas: Class variable defining the database schema.
487
+ """
303
488
 
304
489
  table_name: ClassVar[str] = "audits"
305
490
  schemas: ClassVar[
306
491
  str
307
492
  ] = """
308
- workflow str
309
- , release int
310
- , type str
311
- , context JSON
312
- , parent_run_id int
313
- , run_id int
314
- , metadata JSON
315
- , created_at datetime
316
- , updated_at datetime
317
- primary key ( workflow, release )
493
+ CREATE TABLE IF NOT EXISTS audits (
494
+ workflow TEXT NOT NULL
495
+ , release TEXT NOT NULL
496
+ , type TEXT NOT NULL
497
+ , context BLOB NOT NULL
498
+ , parent_run_id TEXT
499
+ , run_id TEXT NOT NULL
500
+ , metadata BLOB NOT NULL
501
+ , created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
502
+ , updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
503
+ , PRIMARY KEY ( workflow, release )
504
+ )
318
505
  """
319
506
 
507
+ def _ensure_table_exists(self) -> None:
508
+ """Ensure the audit table exists in the database."""
509
+ audit_url = dynamic("audit_url", extras=self.extras)
510
+ if audit_url is None or not audit_url.path:
511
+ raise ValueError(
512
+ "SQLite audit_url must specify a database file path"
513
+ )
514
+
515
+ audit_url_parse: ParseResult = urlparse(audit_url)
516
+ db_path = Path(audit_url_parse.path)
517
+ db_path.parent.mkdir(parents=True, exist_ok=True)
518
+
519
+ with sqlite3.connect(db_path) as conn:
520
+ conn.execute(self.schemas)
521
+ conn.commit()
522
+
320
523
  @classmethod
321
524
  def is_pointed(
322
525
  cls,
@@ -324,7 +527,35 @@ class SQLiteAudit(BaseAudit): # pragma: no cov
324
527
  release: datetime,
325
528
  *,
326
529
  extras: Optional[DictData] = None,
327
- ) -> bool: ...
530
+ ) -> bool:
531
+ """Check if audit data exists for the given workflow and release.
532
+
533
+ Args:
534
+ name: The workflow name to check.
535
+ release: The release datetime to check.
536
+ extras: Optional extra parameters to override core config.
537
+
538
+ Returns:
539
+ bool: True if audit data exists, False otherwise.
540
+ """
541
+ if not dynamic("enable_write_audit", extras=extras):
542
+ return False
543
+
544
+ audit_url = dynamic("audit_url", extras=extras)
545
+ if audit_url is None or not audit_url.path:
546
+ return False
547
+
548
+ audit_url_parse: ParseResult = urlparse(audit_url)
549
+ db_path = Path(audit_url_parse.path)
550
+ if not db_path.exists():
551
+ return False
552
+
553
+ with sqlite3.connect(db_path) as conn:
554
+ cursor = conn.execute(
555
+ "SELECT COUNT(*) FROM audits WHERE workflow = ? AND release = ?",
556
+ (name, release.isoformat()),
557
+ )
558
+ return cursor.fetchone()[0] > 0
328
559
 
329
560
  @classmethod
330
561
  def find_audits(
@@ -332,7 +563,45 @@ class SQLiteAudit(BaseAudit): # pragma: no cov
332
563
  name: str,
333
564
  *,
334
565
  extras: Optional[DictData] = None,
335
- ) -> Iterator[Self]: ...
566
+ ) -> Iterator[Self]:
567
+ """Find all audit data for a given workflow name.
568
+
569
+ Args:
570
+ name: The workflow name to search for.
571
+ extras: Optional extra parameters to override core config.
572
+
573
+ Returns:
574
+ Iterator[Self]: Iterator of audit instances.
575
+ """
576
+ audit_url = dynamic("audit_url", extras=extras)
577
+ if audit_url is None or not audit_url.path:
578
+ return
579
+
580
+ audit_url_parse: ParseResult = urlparse(audit_url)
581
+ db_path = Path(audit_url_parse.path)
582
+ if not db_path.exists():
583
+ return
584
+
585
+ with sqlite3.connect(db_path) as conn:
586
+ cursor = conn.execute(
587
+ "SELECT * FROM audits WHERE workflow = ? ORDER BY release DESC",
588
+ (name,),
589
+ )
590
+ for row in cursor.fetchall():
591
+ # Decompress context and metadata
592
+ context = json.loads(cls._decompress_data(row[3]))
593
+ metadata = json.loads(cls._decompress_data(row[6]))
594
+
595
+ yield cls(
596
+ name=row[0],
597
+ release=datetime.fromisoformat(row[1]),
598
+ type=row[2],
599
+ context=context,
600
+ parent_run_id=row[4],
601
+ run_id=row[5],
602
+ runs_metadata=metadata,
603
+ extras=extras or {},
604
+ )
336
605
 
337
606
  @classmethod
338
607
  def find_audit_with_release(
@@ -341,13 +610,100 @@ class SQLiteAudit(BaseAudit): # pragma: no cov
341
610
  release: Optional[datetime] = None,
342
611
  *,
343
612
  extras: Optional[DictData] = None,
344
- ) -> Self: ...
613
+ ) -> Self:
614
+ """Find audit data for a specific workflow and release.
615
+
616
+ Args:
617
+ name: The workflow name to search for.
618
+ release: Optional release datetime. If None, returns latest release.
619
+ extras: Optional extra parameters to override core config.
620
+
621
+ Returns:
622
+ Self: The audit instance for the specified workflow and release.
623
+
624
+ Raises:
625
+ FileNotFoundError: If the specified workflow/release is not found.
626
+ """
627
+ audit_url = dynamic("audit_url", extras=extras)
628
+ if audit_url is None or not audit_url.path:
629
+ raise FileNotFoundError("SQLite database not configured")
630
+
631
+ audit_url_parse: ParseResult = urlparse(audit_url)
632
+ db_path = Path(audit_url_parse.path)
633
+ if not db_path.exists():
634
+ raise FileNotFoundError(f"Database file not found: {db_path}")
635
+
636
+ with sqlite3.connect(db_path) as conn:
637
+ if release is None:
638
+ # Get latest release
639
+ cursor = conn.execute(
640
+ "SELECT * FROM audits WHERE workflow = ? ORDER BY release DESC LIMIT 1",
641
+ (name,),
642
+ )
643
+ else:
644
+ cursor = conn.execute(
645
+ "SELECT * FROM audits WHERE workflow = ? AND release = ?",
646
+ (name, release.isoformat()),
647
+ )
648
+
649
+ row = cursor.fetchone()
650
+ if not row:
651
+ raise FileNotFoundError(
652
+ f"Audit not found for workflow: {name}, release: {release}"
653
+ )
654
+
655
+ # Decompress context and metadata
656
+ context = json.loads(cls._decompress_data(row[3]))
657
+ metadata = json.loads(cls._decompress_data(row[6]))
658
+
659
+ return cls(
660
+ name=row[0],
661
+ release=datetime.fromisoformat(row[1]),
662
+ type=row[2],
663
+ context=context,
664
+ parent_run_id=row[4],
665
+ run_id=row[5],
666
+ runs_metadata=metadata,
667
+ extras=extras or {},
668
+ )
669
+
670
+ @staticmethod
671
+ def _compress_data(data: str) -> bytes:
672
+ """Compress audit data for storage efficiency.
673
+
674
+ Args:
675
+ data: JSON string data to compress.
345
676
 
346
- def save(self, excluded: Optional[list[str]]) -> SQLiteAudit:
347
- """Save logging data that receive a context data from a workflow
348
- execution result.
677
+ Returns:
678
+ bytes: Compressed data.
349
679
  """
350
- trace: Trace = get_trace(
680
+ return zlib.compress(data.encode("utf-8"))
681
+
682
+ @staticmethod
683
+ def _decompress_data(data: bytes) -> str:
684
+ """Decompress audit data.
685
+
686
+ Args:
687
+ data: Compressed data to decompress.
688
+
689
+ Returns:
690
+ str: Decompressed JSON string.
691
+ """
692
+ return zlib.decompress(data).decode("utf-8")
693
+
694
+ def save(self, excluded: Optional[list[str]] = None) -> Self:
695
+ """Save logging data received from workflow execution result.
696
+
697
+ Args:
698
+ excluded: Optional list of field names to exclude from saving.
699
+
700
+ Returns:
701
+ Self: The audit instance after saving.
702
+
703
+ Raises:
704
+ ValueError: If SQLite database is not properly configured.
705
+ """
706
+ trace: TraceManager = get_trace(
351
707
  self.run_id,
352
708
  parent_run_id=self.parent_run_id,
353
709
  extras=self.extras,
@@ -355,40 +711,126 @@ class SQLiteAudit(BaseAudit): # pragma: no cov
355
711
 
356
712
  # NOTE: Check environ variable was set for real writing.
357
713
  if not dynamic("enable_write_audit", extras=self.extras):
358
- trace.debug("[AUDIT]: Skip writing log cause config was set")
714
+ trace.debug("[AUDIT]: Skip writing audit log cause config was set.")
359
715
  return self
360
716
 
361
- raise NotImplementedError("SQLiteAudit does not implement yet.")
717
+ audit_url = dynamic("audit_url", extras=self.extras)
718
+ if audit_url is None or not audit_url.path:
719
+ raise ValueError(
720
+ "SQLite audit_url must specify a database file path"
721
+ )
722
+
723
+ audit_url_parse: ParseResult = urlparse(audit_url)
724
+ db_path = Path(audit_url_parse.path)
725
+ db_path.parent.mkdir(parents=True, exist_ok=True)
726
+
727
+ # Prepare data for storage
728
+ exclude_set = set(excluded) if excluded else None
729
+ model_data = self.model_dump(exclude=exclude_set)
730
+
731
+ # Compress context and metadata
732
+ context_blob = self._compress_data(
733
+ json.dumps(model_data.get("context", {}))
734
+ )
735
+ metadata_blob = self._compress_data(
736
+ json.dumps(model_data.get("runs_metadata", {}))
737
+ )
738
+
739
+ with sqlite3.connect(db_path) as conn:
740
+ conn.execute(
741
+ """
742
+ INSERT OR REPLACE INTO audits
743
+ (workflow, release, type, context, parent_run_id, run_id, metadata, updated_at)
744
+ VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
745
+ """,
746
+ (
747
+ self.name,
748
+ self.release.isoformat(),
749
+ self.type,
750
+ context_blob,
751
+ self.parent_run_id,
752
+ self.run_id,
753
+ metadata_blob,
754
+ ),
755
+ )
756
+ conn.commit()
757
+
758
+ return self
759
+
760
+ def cleanup(self, max_age_days: int = 180) -> int:
761
+ """Clean up old audit records based on age.
762
+
763
+ Args:
764
+ max_age_days: Maximum age in days for audit records to keep.
765
+
766
+ Returns:
767
+ int: Number of records cleaned up.
768
+ """
769
+ audit_url = dynamic("audit_url", extras=self.extras)
770
+ if audit_url is None or not audit_url.path:
771
+ return 0
772
+
773
+ audit_url_parse: ParseResult = urlparse(audit_url)
774
+ db_path = Path(audit_url_parse.path)
775
+ if not db_path.exists():
776
+ return 0
777
+
778
+ cutoff_date = (
779
+ datetime.now() - timedelta(days=max_age_days)
780
+ ).isoformat()
781
+
782
+ with sqlite3.connect(db_path) as conn:
783
+ cursor = conn.execute(
784
+ "DELETE FROM audits WHERE release < ?", (cutoff_date,)
785
+ )
786
+ conn.commit()
787
+ return cursor.rowcount
362
788
 
363
789
 
364
790
  Audit = Union[
365
791
  FileAudit,
366
792
  SQLiteAudit,
367
- BaseAudit,
368
793
  ]
794
+ AuditType = TypeVar("AuditType", bound=BaseAudit)
369
795
 
370
796
 
371
797
  def get_audit_model(
798
+ *,
372
799
  extras: Optional[DictData] = None,
373
- ) -> type[Audit]: # pragma: no cov
374
- """Get an audit model that dynamic base on the config audit path value.
800
+ ) -> type[AuditType]: # pragma: no cov
801
+ """Get an audit model dynamically based on the config audit path value.
802
+
803
+ Args:
804
+ extras: Optional extra parameters to override the core config.
805
+ This function allow you to pass `audit_model_mapping` for override
806
+ the audit model object with your custom model.
375
807
 
376
- :param extras: An extra parameter that want to override the core config.
808
+ Returns:
809
+ type[Audit]: The appropriate audit model class based on configuration.
377
810
 
378
- :rtype: type[Audit]
811
+ Raises:
812
+ NotImplementedError: If the audit URL scheme is not supported.
379
813
  """
380
- # NOTE: Allow you to override trace model by the extra parameter.
381
- map_audit_models: dict[str, type[Trace]] = extras.get(
814
+ # NOTE: Allow you to override audit model by the extra parameter.
815
+ map_audit_models: dict[str, type[AuditType]] = (extras or {}).get(
382
816
  "audit_model_mapping", {}
383
817
  )
384
- url: ParseResult
385
- if (url := dynamic("audit_url", extras=extras)).scheme and (
386
- url.scheme == "sqlite"
387
- or (url.scheme == "file" and Path(url.path).is_file())
818
+
819
+ audit_url = dynamic("audit_url", extras=extras)
820
+ if audit_url is None:
821
+ return map_audit_models.get("file", FileAudit)
822
+
823
+ audit_url_parse: ParseResult = urlparse(audit_url)
824
+ if not audit_url_parse.scheme:
825
+ return map_audit_models.get("file", FileAudit)
826
+
827
+ if audit_url_parse.scheme == "sqlite" or (
828
+ audit_url_parse.scheme == "file"
829
+ and Path(audit_url_parse.path).is_file()
388
830
  ):
389
- return map_audit_models.get("sqlite", FileAudit)
390
- elif url.scheme and url.scheme != "file":
831
+ return map_audit_models.get("sqlite", SQLiteAudit)
832
+ elif audit_url_parse.scheme != "file":
391
833
  raise NotImplementedError(
392
- f"Does not implement the audit model support for URL: {url}"
834
+ f"Does not implement the audit model support for URL: {audit_url_parse}"
393
835
  )
394
836
  return map_audit_models.get("file", FileAudit)