ddeutil-workflow 0.0.68__py3-none-any.whl → 0.0.70__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +14 -12
- ddeutil/workflow/api/log_conf.py +16 -29
- ddeutil/workflow/api/routes/logs.py +1 -1
- ddeutil/workflow/api/routes/workflows.py +3 -3
- ddeutil/workflow/audits.py +374 -0
- ddeutil/workflow/cli.py +70 -6
- ddeutil/workflow/conf.py +4 -51
- ddeutil/workflow/errors.py +7 -1
- ddeutil/workflow/event.py +2 -2
- ddeutil/workflow/job.py +9 -3
- ddeutil/workflow/result.py +10 -1
- ddeutil/workflow/reusables.py +1 -1
- ddeutil/workflow/{logs.py → traces.py} +224 -409
- ddeutil/workflow/utils.py +19 -11
- ddeutil/workflow/workflow.py +226 -18
- {ddeutil_workflow-0.0.68.dist-info → ddeutil_workflow-0.0.70.dist-info}/METADATA +29 -27
- ddeutil_workflow-0.0.70.dist-info/RECORD +30 -0
- ddeutil_workflow-0.0.68.dist-info/RECORD +0 -29
- {ddeutil_workflow-0.0.68.dist-info → ddeutil_workflow-0.0.70.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.68.dist-info → ddeutil_workflow-0.0.70.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.68.dist-info → ddeutil_workflow-0.0.70.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.68.dist-info → ddeutil_workflow-0.0.70.dist-info}/top_level.txt +0 -0
@@ -3,12 +3,7 @@
|
|
3
3
|
# Licensed under the MIT License. See LICENSE in the project root for
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
|
-
# [x] Use fix config for `
|
7
|
-
"""A Logs module contain Trace and Audit Pydantic models for process log from
|
8
|
-
the core workflow engine. I separate part of log to 2 types:
|
9
|
-
- Trace: A stdout and stderr log
|
10
|
-
- Audit: An audit release log for tracking incremental running workflow.
|
11
|
-
"""
|
6
|
+
# [x] Use fix config for `set_logging`, and Model initialize step.
|
12
7
|
from __future__ import annotations
|
13
8
|
|
14
9
|
import json
|
@@ -17,7 +12,6 @@ import os
|
|
17
12
|
import re
|
18
13
|
from abc import ABC, abstractmethod
|
19
14
|
from collections.abc import Iterator
|
20
|
-
from datetime import datetime
|
21
15
|
from functools import lru_cache
|
22
16
|
from inspect import Traceback, currentframe, getframeinfo
|
23
17
|
from pathlib import Path
|
@@ -26,7 +20,6 @@ from types import FrameType
|
|
26
20
|
from typing import ClassVar, Final, Literal, Optional, TypeVar, Union
|
27
21
|
|
28
22
|
from pydantic import BaseModel, ConfigDict, Field
|
29
|
-
from pydantic.functional_validators import model_validator
|
30
23
|
from typing_extensions import Self
|
31
24
|
|
32
25
|
from .__types import DictData
|
@@ -34,6 +27,7 @@ from .conf import config, dynamic
|
|
34
27
|
from .utils import cut_id, get_dt_now, prepare_newline
|
35
28
|
|
36
29
|
METADATA: str = "metadata.json"
|
30
|
+
logger = logging.getLogger("ddeutil.workflow")
|
37
31
|
|
38
32
|
|
39
33
|
@lru_cache
|
@@ -64,17 +58,6 @@ def set_logging(name: str) -> logging.Logger:
|
|
64
58
|
return _logger
|
65
59
|
|
66
60
|
|
67
|
-
logger = logging.getLogger("ddeutil.workflow")
|
68
|
-
|
69
|
-
|
70
|
-
def get_dt_tznow() -> datetime: # pragma: no cov
|
71
|
-
"""Return the current datetime object that passing the config timezone.
|
72
|
-
|
73
|
-
:rtype: datetime
|
74
|
-
"""
|
75
|
-
return get_dt_now(tz=config.tz)
|
76
|
-
|
77
|
-
|
78
61
|
PREFIX_LOGS: Final[dict[str, dict]] = {
|
79
62
|
"CALLER": {
|
80
63
|
"emoji": "📍",
|
@@ -98,8 +81,22 @@ class PrefixMsg(BaseModel):
|
|
98
81
|
from logging message.
|
99
82
|
"""
|
100
83
|
|
101
|
-
name: Optional[str] = Field(default=None)
|
102
|
-
message: Optional[str] = Field(default=None)
|
84
|
+
name: Optional[str] = Field(default=None, description="A prefix name.")
|
85
|
+
message: Optional[str] = Field(default=None, description="A message.")
|
86
|
+
|
87
|
+
@classmethod
|
88
|
+
def from_str(cls, msg: str) -> Self:
|
89
|
+
"""Extract message prefix from an input message.
|
90
|
+
|
91
|
+
Args:
|
92
|
+
msg (str): A message that want to extract.
|
93
|
+
|
94
|
+
Returns:
|
95
|
+
PrefixMsg: the validated model from a string message.
|
96
|
+
"""
|
97
|
+
return PrefixMsg.model_validate(
|
98
|
+
obj=PREFIX_LOGS_REGEX.search(msg).groupdict()
|
99
|
+
)
|
103
100
|
|
104
101
|
def prepare(self, extras: Optional[DictData] = None) -> str:
|
105
102
|
"""Prepare message with force add prefix before writing trace log.
|
@@ -118,18 +115,6 @@ class PrefixMsg(BaseModel):
|
|
118
115
|
return f"{emoji}[{name}]: {self.message}"
|
119
116
|
|
120
117
|
|
121
|
-
def extract_msg_prefix(msg: str) -> PrefixMsg:
|
122
|
-
"""Extract message prefix from an input message.
|
123
|
-
|
124
|
-
:param msg: A message that want to extract.
|
125
|
-
|
126
|
-
:rtype: PrefixMsg
|
127
|
-
"""
|
128
|
-
return PrefixMsg.model_validate(
|
129
|
-
obj=PREFIX_LOGS_REGEX.search(msg).groupdict()
|
130
|
-
)
|
131
|
-
|
132
|
-
|
133
118
|
class TraceMeta(BaseModel): # pragma: no cov
|
134
119
|
"""Trace Metadata model for making the current metadata of this CPU, Memory
|
135
120
|
process, and thread data.
|
@@ -248,10 +233,6 @@ class BaseTrace(BaseModel, ABC): # pragma: no cov
|
|
248
233
|
|
249
234
|
model_config = ConfigDict(frozen=True)
|
250
235
|
|
251
|
-
run_id: str = Field(default="A running ID")
|
252
|
-
parent_run_id: Optional[str] = Field(
|
253
|
-
default=None, description="A parent running ID"
|
254
|
-
)
|
255
236
|
extras: DictData = Field(
|
256
237
|
default_factory=dict,
|
257
238
|
description=(
|
@@ -259,6 +240,11 @@ class BaseTrace(BaseModel, ABC): # pragma: no cov
|
|
259
240
|
"values."
|
260
241
|
),
|
261
242
|
)
|
243
|
+
run_id: str = Field(description="A running ID")
|
244
|
+
parent_run_id: Optional[str] = Field(
|
245
|
+
default=None,
|
246
|
+
description="A parent running ID",
|
247
|
+
)
|
262
248
|
|
263
249
|
@classmethod
|
264
250
|
@abstractmethod
|
@@ -267,6 +253,17 @@ class BaseTrace(BaseModel, ABC): # pragma: no cov
|
|
267
253
|
path: Optional[Path] = None,
|
268
254
|
extras: Optional[DictData] = None,
|
269
255
|
) -> Iterator[TraceData]: # pragma: no cov
|
256
|
+
"""Return iterator of TraceData models from the target pointer.
|
257
|
+
|
258
|
+
Args:
|
259
|
+
path (:obj:`Path`, optional): A pointer path that want to override.
|
260
|
+
extras (:obj:`DictData`, optional): An extras parameter that want to
|
261
|
+
override default engine config.
|
262
|
+
|
263
|
+
Returns:
|
264
|
+
Iterator[TracData]: An iterator object that generate a TracData
|
265
|
+
model.
|
266
|
+
"""
|
270
267
|
raise NotImplementedError(
|
271
268
|
"Trace dataclass should implement `find_traces` class-method."
|
272
269
|
)
|
@@ -287,7 +284,12 @@ class BaseTrace(BaseModel, ABC): # pragma: no cov
|
|
287
284
|
)
|
288
285
|
|
289
286
|
@abstractmethod
|
290
|
-
def writer(
|
287
|
+
def writer(
|
288
|
+
self,
|
289
|
+
message: str,
|
290
|
+
level: str,
|
291
|
+
is_err: bool = False,
|
292
|
+
) -> None:
|
291
293
|
"""Write a trace message after making to target pointer object. The
|
292
294
|
target can be anything be inherited this class and overwrite this method
|
293
295
|
such as file, console, or database.
|
@@ -303,7 +305,10 @@ class BaseTrace(BaseModel, ABC): # pragma: no cov
|
|
303
305
|
|
304
306
|
@abstractmethod
|
305
307
|
async def awriter(
|
306
|
-
self,
|
308
|
+
self,
|
309
|
+
message: str,
|
310
|
+
level: str,
|
311
|
+
is_err: bool = False,
|
307
312
|
) -> None:
|
308
313
|
"""Async Write a trace message after making to target pointer object.
|
309
314
|
|
@@ -328,22 +333,24 @@ class BaseTrace(BaseModel, ABC): # pragma: no cov
|
|
328
333
|
"Adjust make message method for this trace object before using."
|
329
334
|
)
|
330
335
|
|
331
|
-
|
332
|
-
|
333
|
-
|
336
|
+
@abstractmethod
|
337
|
+
def _logging(
|
338
|
+
self,
|
339
|
+
message: str,
|
340
|
+
mode: str,
|
341
|
+
*,
|
342
|
+
is_err: bool = False,
|
343
|
+
):
|
334
344
|
"""Write trace log with append mode and logging this message with any
|
335
345
|
logging level.
|
336
346
|
|
337
347
|
:param message: (str) A message that want to log.
|
348
|
+
:param mode: (str)
|
349
|
+
:param is_err: (bool)
|
338
350
|
"""
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
mode == "debug" and dynamic("debug", extras=self.extras)
|
343
|
-
):
|
344
|
-
self.writer(msg, level=mode, is_err=is_err)
|
345
|
-
|
346
|
-
getattr(logger, mode)(msg, stacklevel=3)
|
351
|
+
raise NotImplementedError(
|
352
|
+
"Logging action should be implement for making trace log."
|
353
|
+
)
|
347
354
|
|
348
355
|
def debug(self, message: str):
|
349
356
|
"""Write trace log with append mode and logging this message with the
|
@@ -351,7 +358,7 @@ class BaseTrace(BaseModel, ABC): # pragma: no cov
|
|
351
358
|
|
352
359
|
:param message: (str) A message that want to log.
|
353
360
|
"""
|
354
|
-
self.
|
361
|
+
self._logging(message, mode="debug")
|
355
362
|
|
356
363
|
def info(self, message: str) -> None:
|
357
364
|
"""Write trace log with append mode and logging this message with the
|
@@ -359,7 +366,7 @@ class BaseTrace(BaseModel, ABC): # pragma: no cov
|
|
359
366
|
|
360
367
|
:param message: (str) A message that want to log.
|
361
368
|
"""
|
362
|
-
self.
|
369
|
+
self._logging(message, mode="info")
|
363
370
|
|
364
371
|
def warning(self, message: str) -> None:
|
365
372
|
"""Write trace log with append mode and logging this message with the
|
@@ -367,7 +374,7 @@ class BaseTrace(BaseModel, ABC): # pragma: no cov
|
|
367
374
|
|
368
375
|
:param message: (str) A message that want to log.
|
369
376
|
"""
|
370
|
-
self.
|
377
|
+
self._logging(message, mode="warning")
|
371
378
|
|
372
379
|
def error(self, message: str) -> None:
|
373
380
|
"""Write trace log with append mode and logging this message with the
|
@@ -375,7 +382,7 @@ class BaseTrace(BaseModel, ABC): # pragma: no cov
|
|
375
382
|
|
376
383
|
:param message: (str) A message that want to log.
|
377
384
|
"""
|
378
|
-
self.
|
385
|
+
self._logging(message, mode="error", is_err=True)
|
379
386
|
|
380
387
|
def exception(self, message: str) -> None:
|
381
388
|
"""Write trace log with append mode and logging this message with the
|
@@ -383,24 +390,26 @@ class BaseTrace(BaseModel, ABC): # pragma: no cov
|
|
383
390
|
|
384
391
|
:param message: (str) A message that want to log.
|
385
392
|
"""
|
386
|
-
self.
|
393
|
+
self._logging(message, mode="exception", is_err=True)
|
387
394
|
|
388
|
-
|
389
|
-
|
395
|
+
@abstractmethod
|
396
|
+
async def _alogging(
|
397
|
+
self,
|
398
|
+
message: str,
|
399
|
+
mode: str,
|
400
|
+
*,
|
401
|
+
is_err: bool = False,
|
390
402
|
) -> None:
|
391
|
-
"""
|
392
|
-
logging level.
|
403
|
+
"""Async write trace log with append mode and logging this message with
|
404
|
+
any logging level.
|
393
405
|
|
394
406
|
:param message: (str) A message that want to log.
|
407
|
+
:param mode: (str)
|
408
|
+
:param is_err: (bool)
|
395
409
|
"""
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
mode == "debug" and dynamic("debug", extras=self.extras)
|
400
|
-
):
|
401
|
-
await self.awriter(msg, level=mode, is_err=is_err)
|
402
|
-
|
403
|
-
getattr(logger, mode)(msg, stacklevel=3)
|
410
|
+
raise NotImplementedError(
|
411
|
+
"Async Logging action should be implement for making trace log."
|
412
|
+
)
|
404
413
|
|
405
414
|
async def adebug(self, message: str) -> None: # pragma: no cov
|
406
415
|
"""Async write trace log with append mode and logging this message with
|
@@ -408,7 +417,7 @@ class BaseTrace(BaseModel, ABC): # pragma: no cov
|
|
408
417
|
|
409
418
|
:param message: (str) A message that want to log.
|
410
419
|
"""
|
411
|
-
await self.
|
420
|
+
await self._alogging(message, mode="debug")
|
412
421
|
|
413
422
|
async def ainfo(self, message: str) -> None: # pragma: no cov
|
414
423
|
"""Async write trace log with append mode and logging this message with
|
@@ -416,7 +425,7 @@ class BaseTrace(BaseModel, ABC): # pragma: no cov
|
|
416
425
|
|
417
426
|
:param message: (str) A message that want to log.
|
418
427
|
"""
|
419
|
-
await self.
|
428
|
+
await self._alogging(message, mode="info")
|
420
429
|
|
421
430
|
async def awarning(self, message: str) -> None: # pragma: no cov
|
422
431
|
"""Async write trace log with append mode and logging this message with
|
@@ -424,7 +433,7 @@ class BaseTrace(BaseModel, ABC): # pragma: no cov
|
|
424
433
|
|
425
434
|
:param message: (str) A message that want to log.
|
426
435
|
"""
|
427
|
-
await self.
|
436
|
+
await self._alogging(message, mode="warning")
|
428
437
|
|
429
438
|
async def aerror(self, message: str) -> None: # pragma: no cov
|
430
439
|
"""Async write trace log with append mode and logging this message with
|
@@ -432,7 +441,7 @@ class BaseTrace(BaseModel, ABC): # pragma: no cov
|
|
432
441
|
|
433
442
|
:param message: (str) A message that want to log.
|
434
443
|
"""
|
435
|
-
await self.
|
444
|
+
await self._alogging(message, mode="error", is_err=True)
|
436
445
|
|
437
446
|
async def aexception(self, message: str) -> None: # pragma: no cov
|
438
447
|
"""Async write trace log with append mode and logging this message with
|
@@ -440,10 +449,127 @@ class BaseTrace(BaseModel, ABC): # pragma: no cov
|
|
440
449
|
|
441
450
|
:param message: (str) A message that want to log.
|
442
451
|
"""
|
443
|
-
await self.
|
452
|
+
await self._alogging(message, mode="exception", is_err=True)
|
453
|
+
|
454
|
+
|
455
|
+
class ConsoleTrace(BaseTrace): # pragma: no cov
|
456
|
+
"""Console Trace log model."""
|
457
|
+
|
458
|
+
@classmethod
|
459
|
+
def find_traces(
|
460
|
+
cls,
|
461
|
+
path: Optional[Path] = None,
|
462
|
+
extras: Optional[DictData] = None,
|
463
|
+
) -> Iterator[TraceData]: # pragma: no cov
|
464
|
+
raise NotImplementedError(
|
465
|
+
"Console Trace does not support to find history traces data."
|
466
|
+
)
|
467
|
+
|
468
|
+
@classmethod
|
469
|
+
def find_trace_with_id(
|
470
|
+
cls,
|
471
|
+
run_id: str,
|
472
|
+
force_raise: bool = True,
|
473
|
+
*,
|
474
|
+
path: Optional[Path] = None,
|
475
|
+
extras: Optional[DictData] = None,
|
476
|
+
) -> TraceData:
|
477
|
+
raise NotImplementedError(
|
478
|
+
"Console Trace does not support to find history traces data with "
|
479
|
+
"the specific running ID."
|
480
|
+
)
|
481
|
+
|
482
|
+
def writer(
|
483
|
+
self,
|
484
|
+
message: str,
|
485
|
+
level: str,
|
486
|
+
is_err: bool = False,
|
487
|
+
) -> None:
|
488
|
+
"""Write a trace message after making to target pointer object. The
|
489
|
+
target can be anything be inherited this class and overwrite this method
|
490
|
+
such as file, console, or database.
|
491
|
+
|
492
|
+
:param message: (str) A message after making.
|
493
|
+
:param level: (str) A log level.
|
494
|
+
:param is_err: (bool) A flag for writing with an error trace or not.
|
495
|
+
(Default be False)
|
496
|
+
"""
|
497
|
+
|
498
|
+
async def awriter(
|
499
|
+
self,
|
500
|
+
message: str,
|
501
|
+
level: str,
|
502
|
+
is_err: bool = False,
|
503
|
+
) -> None:
|
504
|
+
"""Async Write a trace message after making to target pointer object.
|
505
|
+
|
506
|
+
:param message: (str) A message after making.
|
507
|
+
:param level: (str) A log level.
|
508
|
+
:param is_err: (bool) A flag for writing with an error trace or not.
|
509
|
+
(Default be False)
|
510
|
+
"""
|
511
|
+
|
512
|
+
@property
|
513
|
+
def cut_id(self) -> str:
|
514
|
+
"""Combine cutting ID of parent running ID if it set.
|
515
|
+
|
516
|
+
:rtype: str
|
517
|
+
"""
|
518
|
+
cut_run_id: str = cut_id(self.run_id)
|
519
|
+
if not self.parent_run_id:
|
520
|
+
return f"{cut_run_id}"
|
521
|
+
|
522
|
+
cut_parent_run_id: str = cut_id(self.parent_run_id)
|
523
|
+
return f"{cut_parent_run_id} -> {cut_run_id}"
|
524
|
+
|
525
|
+
def make_message(self, message: str) -> str:
|
526
|
+
"""Prepare and Make a message before write and log steps.
|
527
|
+
|
528
|
+
:param message: (str) A message that want to prepare and make before.
|
529
|
+
|
530
|
+
:rtype: str
|
531
|
+
"""
|
532
|
+
return prepare_newline(
|
533
|
+
f"({self.cut_id}) "
|
534
|
+
f"{PrefixMsg.from_str(message).prepare(self.extras)}"
|
535
|
+
)
|
536
|
+
|
537
|
+
def _logging(
|
538
|
+
self, message: str, mode: str, *, is_err: bool = False
|
539
|
+
) -> None:
|
540
|
+
"""Write trace log with append mode and logging this message with any
|
541
|
+
logging level.
|
542
|
+
|
543
|
+
:param message: (str) A message that want to log.
|
544
|
+
"""
|
545
|
+
msg: str = self.make_message(message)
|
546
|
+
|
547
|
+
if mode != "debug" or (
|
548
|
+
mode == "debug" and dynamic("debug", extras=self.extras)
|
549
|
+
):
|
550
|
+
self.writer(msg, level=mode, is_err=is_err)
|
551
|
+
|
552
|
+
getattr(logger, mode)(msg, stacklevel=3, extra={"cut_id": self.cut_id})
|
553
|
+
|
554
|
+
async def _alogging(
|
555
|
+
self, message: str, mode: str, *, is_err: bool = False
|
556
|
+
) -> None:
|
557
|
+
"""Write trace log with append mode and logging this message with any
|
558
|
+
logging level.
|
559
|
+
|
560
|
+
:param message: (str) A message that want to log.
|
561
|
+
"""
|
562
|
+
msg: str = self.make_message(message)
|
444
563
|
|
564
|
+
if mode != "debug" or (
|
565
|
+
mode == "debug" and dynamic("debug", extras=self.extras)
|
566
|
+
):
|
567
|
+
await self.awriter(msg, level=mode, is_err=is_err)
|
568
|
+
|
569
|
+
getattr(logger, mode)(msg, stacklevel=3, extra={"cut_id": self.cut_id})
|
445
570
|
|
446
|
-
|
571
|
+
|
572
|
+
class FileTrace(ConsoleTrace): # pragma: no cov
|
447
573
|
"""File Trace dataclass that write file to the local storage."""
|
448
574
|
|
449
575
|
@classmethod
|
@@ -492,6 +618,11 @@ class FileTrace(BaseTrace): # pragma: no cov
|
|
492
618
|
|
493
619
|
@property
|
494
620
|
def pointer(self) -> Path:
|
621
|
+
"""Pointer of the target path that use to writing trace log or searching
|
622
|
+
trace log.
|
623
|
+
|
624
|
+
:rtype: Path
|
625
|
+
"""
|
495
626
|
log_file: Path = (
|
496
627
|
dynamic("trace_path", extras=self.extras)
|
497
628
|
/ f"run_id={self.parent_run_id or self.run_id}"
|
@@ -500,31 +631,12 @@ class FileTrace(BaseTrace): # pragma: no cov
|
|
500
631
|
log_file.mkdir(parents=True)
|
501
632
|
return log_file
|
502
633
|
|
503
|
-
|
504
|
-
|
505
|
-
|
506
|
-
|
507
|
-
:
|
508
|
-
|
509
|
-
cut_run_id: str = cut_id(self.run_id)
|
510
|
-
if not self.parent_run_id:
|
511
|
-
return f"{cut_run_id}"
|
512
|
-
|
513
|
-
cut_parent_run_id: str = cut_id(self.parent_run_id)
|
514
|
-
return f"{cut_parent_run_id} -> {cut_run_id}"
|
515
|
-
|
516
|
-
def make_message(self, message: str) -> str:
|
517
|
-
"""Prepare and Make a message before write and log steps.
|
518
|
-
|
519
|
-
:param message: (str) A message that want to prepare and make before.
|
520
|
-
|
521
|
-
:rtype: str
|
522
|
-
"""
|
523
|
-
return prepare_newline(
|
524
|
-
f"({self.cut_id}) {extract_msg_prefix(message).prepare(self.extras)}"
|
525
|
-
)
|
526
|
-
|
527
|
-
def writer(self, message: str, level: str, is_err: bool = False) -> None:
|
634
|
+
def writer(
|
635
|
+
self,
|
636
|
+
message: str,
|
637
|
+
level: str,
|
638
|
+
is_err: bool = False,
|
639
|
+
) -> None:
|
528
640
|
"""Write a trace message after making to target file and write metadata
|
529
641
|
in the same path of standard files.
|
530
642
|
|
@@ -556,7 +668,10 @@ class FileTrace(BaseTrace): # pragma: no cov
|
|
556
668
|
f.write(trace_meta.model_dump_json() + "\n")
|
557
669
|
|
558
670
|
async def awriter(
|
559
|
-
self,
|
671
|
+
self,
|
672
|
+
message: str,
|
673
|
+
level: str,
|
674
|
+
is_err: bool = False,
|
560
675
|
) -> None: # pragma: no cov
|
561
676
|
"""Write with async mode."""
|
562
677
|
if not dynamic("enable_write_log", extras=self.extras):
|
@@ -584,7 +699,7 @@ class FileTrace(BaseTrace): # pragma: no cov
|
|
584
699
|
await f.write(trace_meta.model_dump_json() + "\n")
|
585
700
|
|
586
701
|
|
587
|
-
class SQLiteTrace(
|
702
|
+
class SQLiteTrace(ConsoleTrace): # pragma: no cov
|
588
703
|
"""SQLite Trace dataclass that write trace log to the SQLite database file."""
|
589
704
|
|
590
705
|
table_name: ClassVar[str] = "audits"
|
@@ -618,16 +733,23 @@ class SQLiteTrace(BaseTrace): # pragma: no cov
|
|
618
733
|
def make_message(self, message: str) -> str: ...
|
619
734
|
|
620
735
|
def writer(
|
621
|
-
self,
|
736
|
+
self,
|
737
|
+
message: str,
|
738
|
+
level: str,
|
739
|
+
is_err: bool = False,
|
622
740
|
) -> None: ...
|
623
741
|
|
624
742
|
def awriter(
|
625
|
-
self,
|
743
|
+
self,
|
744
|
+
message: str,
|
745
|
+
level: str,
|
746
|
+
is_err: bool = False,
|
626
747
|
) -> None: ...
|
627
748
|
|
628
749
|
|
629
750
|
Trace = TypeVar("Trace", bound=BaseTrace)
|
630
751
|
TraceModel = Union[
|
752
|
+
ConsoleTrace,
|
631
753
|
FileTrace,
|
632
754
|
SQLiteTrace,
|
633
755
|
]
|
@@ -656,310 +778,3 @@ def get_trace(
|
|
656
778
|
return FileTrace(
|
657
779
|
run_id=run_id, parent_run_id=parent_run_id, extras=(extras or {})
|
658
780
|
)
|
659
|
-
|
660
|
-
|
661
|
-
class BaseAudit(BaseModel, ABC):
|
662
|
-
"""Base Audit Pydantic Model with abstraction class property that implement
|
663
|
-
only model fields. This model should to use with inherit to logging
|
664
|
-
subclass like file, sqlite, etc.
|
665
|
-
"""
|
666
|
-
|
667
|
-
extras: DictData = Field(
|
668
|
-
default_factory=dict,
|
669
|
-
description="An extras parameter that want to override core config",
|
670
|
-
)
|
671
|
-
name: str = Field(description="A workflow name.")
|
672
|
-
release: datetime = Field(description="A release datetime.")
|
673
|
-
type: str = Field(description="A running type before logging.")
|
674
|
-
context: DictData = Field(
|
675
|
-
default_factory=dict,
|
676
|
-
description="A context that receive from a workflow execution result.",
|
677
|
-
)
|
678
|
-
parent_run_id: Optional[str] = Field(
|
679
|
-
default=None, description="A parent running ID."
|
680
|
-
)
|
681
|
-
run_id: str = Field(description="A running ID")
|
682
|
-
update: datetime = Field(default_factory=get_dt_tznow)
|
683
|
-
execution_time: float = Field(default=0, description="An execution time.")
|
684
|
-
|
685
|
-
@model_validator(mode="after")
|
686
|
-
def __model_action(self) -> Self:
|
687
|
-
"""Do before the Audit action with WORKFLOW_AUDIT_ENABLE_WRITE env variable.
|
688
|
-
|
689
|
-
:rtype: Self
|
690
|
-
"""
|
691
|
-
if dynamic("enable_write_audit", extras=self.extras):
|
692
|
-
self.do_before()
|
693
|
-
|
694
|
-
# NOTE: Start setting log config in this line with cache.
|
695
|
-
set_logging("ddeutil.workflow")
|
696
|
-
return self
|
697
|
-
|
698
|
-
@classmethod
|
699
|
-
@abstractmethod
|
700
|
-
def is_pointed(
|
701
|
-
cls,
|
702
|
-
name: str,
|
703
|
-
release: datetime,
|
704
|
-
*,
|
705
|
-
extras: Optional[DictData] = None,
|
706
|
-
) -> bool:
|
707
|
-
raise NotImplementedError(
|
708
|
-
"Audit should implement `is_pointed` class-method"
|
709
|
-
)
|
710
|
-
|
711
|
-
@classmethod
|
712
|
-
@abstractmethod
|
713
|
-
def find_audits(
|
714
|
-
cls, name: str, *, extras: Optional[DictData] = None
|
715
|
-
) -> Iterator[Self]:
|
716
|
-
raise NotImplementedError(
|
717
|
-
"Audit should implement `find_audits` class-method"
|
718
|
-
)
|
719
|
-
|
720
|
-
@classmethod
|
721
|
-
@abstractmethod
|
722
|
-
def find_audit_with_release(
|
723
|
-
cls,
|
724
|
-
name: str,
|
725
|
-
release: Optional[datetime] = None,
|
726
|
-
*,
|
727
|
-
extras: Optional[DictData] = None,
|
728
|
-
) -> Self:
|
729
|
-
raise NotImplementedError(
|
730
|
-
"Audit should implement `find_audit_with_release` class-method"
|
731
|
-
)
|
732
|
-
|
733
|
-
def do_before(self) -> None: # pragma: no cov
|
734
|
-
"""To something before end up of initial log model."""
|
735
|
-
|
736
|
-
@abstractmethod
|
737
|
-
def save(self, excluded: Optional[list[str]]) -> None: # pragma: no cov
|
738
|
-
"""Save this model logging to target logging store."""
|
739
|
-
raise NotImplementedError("Audit should implement `save` method.")
|
740
|
-
|
741
|
-
|
742
|
-
class FileAudit(BaseAudit):
|
743
|
-
"""File Audit Pydantic Model that use to saving log data from result of
|
744
|
-
workflow execution. It inherits from BaseAudit model that implement the
|
745
|
-
``self.save`` method for file.
|
746
|
-
"""
|
747
|
-
|
748
|
-
filename_fmt: ClassVar[str] = (
|
749
|
-
"workflow={name}/release={release:%Y%m%d%H%M%S}"
|
750
|
-
)
|
751
|
-
|
752
|
-
def do_before(self) -> None:
|
753
|
-
"""Create directory of release before saving log file."""
|
754
|
-
self.pointer().mkdir(parents=True, exist_ok=True)
|
755
|
-
|
756
|
-
@classmethod
|
757
|
-
def find_audits(
|
758
|
-
cls, name: str, *, extras: Optional[DictData] = None
|
759
|
-
) -> Iterator[Self]:
|
760
|
-
"""Generate the audit data that found from logs path with specific a
|
761
|
-
workflow name.
|
762
|
-
|
763
|
-
:param name: A workflow name that want to search release logging data.
|
764
|
-
:param extras: An extra parameter that want to override core config.
|
765
|
-
|
766
|
-
:rtype: Iterator[Self]
|
767
|
-
"""
|
768
|
-
pointer: Path = (
|
769
|
-
dynamic("audit_path", extras=extras) / f"workflow={name}"
|
770
|
-
)
|
771
|
-
if not pointer.exists():
|
772
|
-
raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
|
773
|
-
|
774
|
-
for file in pointer.glob("./release=*/*.log"):
|
775
|
-
with file.open(mode="r", encoding="utf-8") as f:
|
776
|
-
yield cls.model_validate(obj=json.load(f))
|
777
|
-
|
778
|
-
@classmethod
|
779
|
-
def find_audit_with_release(
|
780
|
-
cls,
|
781
|
-
name: str,
|
782
|
-
release: Optional[datetime] = None,
|
783
|
-
*,
|
784
|
-
extras: Optional[DictData] = None,
|
785
|
-
) -> Self:
|
786
|
-
"""Return the audit data that found from logs path with specific
|
787
|
-
workflow name and release values. If a release does not pass to an input
|
788
|
-
argument, it will return the latest release from the current log path.
|
789
|
-
|
790
|
-
:param name: (str) A workflow name that want to search log.
|
791
|
-
:param release: (datetime) A release datetime that want to search log.
|
792
|
-
:param extras: An extra parameter that want to override core config.
|
793
|
-
|
794
|
-
:raise FileNotFoundError:
|
795
|
-
:raise NotImplementedError: If an input release does not pass to this
|
796
|
-
method. Because this method does not implement latest log.
|
797
|
-
|
798
|
-
:rtype: Self
|
799
|
-
"""
|
800
|
-
if release is None:
|
801
|
-
raise NotImplementedError("Find latest log does not implement yet.")
|
802
|
-
|
803
|
-
pointer: Path = (
|
804
|
-
dynamic("audit_path", extras=extras)
|
805
|
-
/ f"workflow={name}/release={release:%Y%m%d%H%M%S}"
|
806
|
-
)
|
807
|
-
if not pointer.exists():
|
808
|
-
raise FileNotFoundError(
|
809
|
-
f"Pointer: ./logs/workflow={name}/"
|
810
|
-
f"release={release:%Y%m%d%H%M%S} does not found."
|
811
|
-
)
|
812
|
-
|
813
|
-
latest_file: Path = max(pointer.glob("./*.log"), key=os.path.getctime)
|
814
|
-
with latest_file.open(mode="r", encoding="utf-8") as f:
|
815
|
-
return cls.model_validate(obj=json.load(f))
|
816
|
-
|
817
|
-
@classmethod
|
818
|
-
def is_pointed(
|
819
|
-
cls,
|
820
|
-
name: str,
|
821
|
-
release: datetime,
|
822
|
-
*,
|
823
|
-
extras: Optional[DictData] = None,
|
824
|
-
) -> bool:
|
825
|
-
"""Check the release log already pointed or created at the destination
|
826
|
-
log path.
|
827
|
-
|
828
|
-
:param name: (str) A workflow name.
|
829
|
-
:param release: (datetime) A release datetime.
|
830
|
-
:param extras: An extra parameter that want to override core config.
|
831
|
-
|
832
|
-
:rtype: bool
|
833
|
-
:return: Return False if the release log was not pointed or created.
|
834
|
-
"""
|
835
|
-
# NOTE: Return False if enable writing log flag does not set.
|
836
|
-
if not dynamic("enable_write_audit", extras=extras):
|
837
|
-
return False
|
838
|
-
|
839
|
-
# NOTE: create pointer path that use the same logic of pointer method.
|
840
|
-
pointer: Path = dynamic(
|
841
|
-
"audit_path", extras=extras
|
842
|
-
) / cls.filename_fmt.format(name=name, release=release)
|
843
|
-
|
844
|
-
return pointer.exists()
|
845
|
-
|
846
|
-
def pointer(self) -> Path:
|
847
|
-
"""Return release directory path that was generated from model data.
|
848
|
-
|
849
|
-
:rtype: Path
|
850
|
-
"""
|
851
|
-
return dynamic(
|
852
|
-
"audit_path", extras=self.extras
|
853
|
-
) / self.filename_fmt.format(name=self.name, release=self.release)
|
854
|
-
|
855
|
-
def save(self, excluded: Optional[list[str]] = None) -> Self:
|
856
|
-
"""Save logging data that receive a context data from a workflow
|
857
|
-
execution result.
|
858
|
-
|
859
|
-
:param excluded: An excluded list of key name that want to pass in the
|
860
|
-
model_dump method.
|
861
|
-
|
862
|
-
:rtype: Self
|
863
|
-
"""
|
864
|
-
trace: TraceModel = get_trace(
|
865
|
-
self.run_id,
|
866
|
-
parent_run_id=self.parent_run_id,
|
867
|
-
extras=self.extras,
|
868
|
-
)
|
869
|
-
|
870
|
-
# NOTE: Check environ variable was set for real writing.
|
871
|
-
if not dynamic("enable_write_audit", extras=self.extras):
|
872
|
-
trace.debug("[AUDIT]: Skip writing log cause config was set")
|
873
|
-
return self
|
874
|
-
|
875
|
-
log_file: Path = (
|
876
|
-
self.pointer() / f"{self.parent_run_id or self.run_id}.log"
|
877
|
-
)
|
878
|
-
log_file.write_text(
|
879
|
-
json.dumps(
|
880
|
-
self.model_dump(exclude=excluded),
|
881
|
-
default=str,
|
882
|
-
indent=2,
|
883
|
-
),
|
884
|
-
encoding="utf-8",
|
885
|
-
)
|
886
|
-
return self
|
887
|
-
|
888
|
-
|
889
|
-
class SQLiteAudit(BaseAudit): # pragma: no cov
|
890
|
-
"""SQLite Audit Pydantic Model."""
|
891
|
-
|
892
|
-
table_name: ClassVar[str] = "audits"
|
893
|
-
schemas: ClassVar[
|
894
|
-
str
|
895
|
-
] = """
|
896
|
-
workflow str,
|
897
|
-
release int,
|
898
|
-
type str,
|
899
|
-
context json,
|
900
|
-
parent_run_id int,
|
901
|
-
run_id int,
|
902
|
-
update datetime
|
903
|
-
primary key ( run_id )
|
904
|
-
"""
|
905
|
-
|
906
|
-
@classmethod
|
907
|
-
def is_pointed(
|
908
|
-
cls,
|
909
|
-
name: str,
|
910
|
-
release: datetime,
|
911
|
-
*,
|
912
|
-
extras: Optional[DictData] = None,
|
913
|
-
) -> bool: ...
|
914
|
-
|
915
|
-
@classmethod
|
916
|
-
def find_audits(
|
917
|
-
cls, name: str, *, extras: Optional[DictData] = None
|
918
|
-
) -> Iterator[Self]: ...
|
919
|
-
|
920
|
-
@classmethod
|
921
|
-
def find_audit_with_release(
|
922
|
-
cls,
|
923
|
-
name: str,
|
924
|
-
release: Optional[datetime] = None,
|
925
|
-
*,
|
926
|
-
extras: Optional[DictData] = None,
|
927
|
-
) -> Self: ...
|
928
|
-
|
929
|
-
def save(self, excluded: Optional[list[str]]) -> SQLiteAudit:
|
930
|
-
"""Save logging data that receive a context data from a workflow
|
931
|
-
execution result.
|
932
|
-
"""
|
933
|
-
trace: TraceModel = get_trace(
|
934
|
-
self.run_id,
|
935
|
-
parent_run_id=self.parent_run_id,
|
936
|
-
extras=self.extras,
|
937
|
-
)
|
938
|
-
|
939
|
-
# NOTE: Check environ variable was set for real writing.
|
940
|
-
if not dynamic("enable_write_audit", extras=self.extras):
|
941
|
-
trace.debug("[AUDIT]: Skip writing log cause config was set")
|
942
|
-
return self
|
943
|
-
|
944
|
-
raise NotImplementedError("SQLiteAudit does not implement yet.")
|
945
|
-
|
946
|
-
|
947
|
-
Audit = TypeVar("Audit", bound=BaseAudit)
|
948
|
-
AuditModel = Union[
|
949
|
-
FileAudit,
|
950
|
-
SQLiteAudit,
|
951
|
-
]
|
952
|
-
|
953
|
-
|
954
|
-
def get_audit(
|
955
|
-
extras: Optional[DictData] = None,
|
956
|
-
) -> type[AuditModel]: # pragma: no cov
|
957
|
-
"""Get an audit class that dynamic base on the config audit path value.
|
958
|
-
|
959
|
-
:param extras: An extra parameter that want to override the core config.
|
960
|
-
|
961
|
-
:rtype: type[Audit]
|
962
|
-
"""
|
963
|
-
if dynamic("audit_path", extras=extras).is_file():
|
964
|
-
return SQLiteAudit
|
965
|
-
return FileAudit
|