ddeutil-workflow 0.0.47__py3-none-any.whl → 0.0.49__py3-none-any.whl

ddeutil/workflow/logs.py

@@ -3,46 +3,63 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [x] Use fix config
-"""A Logs module contain TraceLog dataclass and AuditLog model.
+# [x] Use dynamic config
+# [x] Use fix config for `get_logger`, and Model initialize step.
+"""A Logs module contain Trace dataclass and Audit Pydantic model.
 """
 from __future__ import annotations
 
 import json
+import logging
 import os
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
+from dataclasses import field
 from datetime import datetime
+from functools import lru_cache
 from inspect import Traceback, currentframe, getframeinfo
 from pathlib import Path
 from threading import get_ident
-from typing import ClassVar, Literal, Optional, Union
+from typing import ClassVar, Literal, Optional, TypeVar, Union
 
 from pydantic import BaseModel, Field
 from pydantic.dataclasses import dataclass
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
-from .__types import DictData, DictStr, TupleStr
-from .conf import config, get_logger
+from .__types import DictData, DictStr
+from .conf import config, dynamic
 from .utils import cut_id, get_dt_now
 
-logger = get_logger("ddeutil.workflow")
 
-__all__: TupleStr = (
-    "FileTraceLog",
-    "SQLiteTraceLog",
-    "TraceData",
-    "TraceMeda",
-    "TraceLog",
-    "get_dt_tznow",
-    "get_trace",
-    "get_trace_obj",
-    "get_audit",
-    "FileAudit",
-    "SQLiteAudit",
-    "Audit",
-)
+@lru_cache
+def get_logger(name: str):
+    """Return logger object with an input module name.
+
+    :param name: A module name that want to log.
+    """
+    lg = logging.getLogger(name)
+
+    # NOTE: Developers using this package can then disable all logging just for
+    #   this package by;
+    #
+    #   `logging.getLogger('ddeutil.workflow').propagate = False`
+    #
+    lg.addHandler(logging.NullHandler())
+
+    formatter = logging.Formatter(
+        fmt=config.log_format,
+        datefmt=config.log_datetime_format,
+    )
+    stream = logging.StreamHandler()
+    stream.setFormatter(formatter)
+    lg.addHandler(stream)
+
+    lg.setLevel(logging.DEBUG if config.debug else logging.INFO)
+    return lg
+
+
+logger = get_logger("ddeutil.workflow")
 
 
 def get_dt_tznow() -> datetime:  # pragma: no cov
@@ -53,7 +70,9 @@ def get_dt_tznow() -> datetime:  # pragma: no cov
     return get_dt_now(tz=config.tz)
 
 
-class TraceMeda(BaseModel):  # pragma: no cov
+class TraceMeta(BaseModel):  # pragma: no cov
+    """Trace Meta model."""
+
     mode: Literal["stdout", "stderr"]
     datetime: str
     process: int
@@ -63,14 +82,28 @@ class TraceMeda(BaseModel):  # pragma: no cov
     lineno: int
 
     @classmethod
-    def make(cls, mode: Literal["stdout", "stderr"], message: str) -> Self:
-        """Make a TraceMeda instance."""
+    def make(
+        cls,
+        mode: Literal["stdout", "stderr"],
+        message: str,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Self:
+        """Make the current TraceMeta instance that catching local state.
+
+        :rtype: Self
+        """
         frame_info: Traceback = getframeinfo(
             currentframe().f_back.f_back.f_back
         )
+        extras: DictData = extras or {}
         return cls(
             mode=mode,
-            datetime=get_dt_tznow().strftime(config.log_datetime_format),
+            datetime=(
+                get_dt_now(tz=dynamic("tz", extras=extras)).strftime(
+                    dynamic("log_datetime_format", extras=extras)
+                )
+            ),
             process=os.getpid(),
             thread=get_ident(),
             message=message,
@@ -84,7 +117,7 @@ class TraceData(BaseModel):  # pragma: no cov
 
     stdout: str = Field(description="A standard output trace data.")
     stderr: str = Field(description="A standard error trace data.")
-    meta: list[TraceMeda] = Field(
+    meta: list[TraceMeta] = Field(
         default_factory=list,
         description=(
             "A metadata mapping of this output and error before making it to "
@@ -122,11 +155,38 @@ class TraceData(BaseModel):  # pragma: no cov
 
 
 @dataclass(frozen=True)
-class BaseTraceLog(ABC):  # pragma: no cov
-    """Base Trace Log dataclass object."""
+class BaseTrace(ABC):  # pragma: no cov
+    """Base Trace dataclass with abstraction class property."""
 
     run_id: str
-    parent_run_id: Optional[str] = None
+    parent_run_id: Optional[str] = field(default=None)
+    extras: DictData = field(default_factory=dict, compare=False, repr=False)
+
+    @classmethod
+    @abstractmethod
+    def find_traces(
+        cls,
+        path: Path | None = None,
+        extras: Optional[DictData] = None,
+    ) -> Iterator[TraceData]:  # pragma: no cov
+        raise NotImplementedError(
+            "Trace dataclass should implement `find_traces` class-method."
+        )
+
+    @classmethod
+    @abstractmethod
+    def find_trace_with_id(
+        cls,
+        run_id: str,
+        force_raise: bool = True,
+        *,
+        path: Path | None = None,
+        extras: Optional[DictData] = None,
+    ) -> TraceData:
+        raise NotImplementedError(
+            "Trace dataclass should implement `find_trace_with_id` "
+            "class-method."
+        )
 
     @abstractmethod
     def writer(self, message: str, is_err: bool = False) -> None:
@@ -148,6 +208,9 @@ class BaseTraceLog(ABC):  # pragma: no cov
         :param message:
         :param is_err:
         """
+        raise NotImplementedError(
+            "Create async writer logic for this trace object before using."
+        )
 
     @abstractmethod
     def make_message(self, message: str) -> str:
@@ -169,7 +232,7 @@ class BaseTraceLog(ABC):  # pragma: no cov
         """
         msg: str = self.make_message(message)
 
-        if config.debug:
+        if dynamic("debug", extras=self.extras):
             self.writer(msg)
 
         logger.debug(msg, stacklevel=2)
@@ -221,7 +284,7 @@ class BaseTraceLog(ABC):  # pragma: no cov
         :param message: (str) A message that want to log.
         """
         msg: str = self.make_message(message)
-        if config.debug:
+        if dynamic("debug", extras=self.extras):
             await self.awriter(msg)
         logger.info(msg, stacklevel=2)
 
@@ -266,26 +329,43 @@ class BaseTraceLog(ABC):  # pragma: no cov
         logger.exception(msg, stacklevel=2)
 
 
-class FileTraceLog(BaseTraceLog):  # pragma: no cov
-    """Trace Log object that write file to the local storage."""
+class FileTrace(BaseTrace):  # pragma: no cov
+    """File Trace dataclass that write file to the local storage."""
 
     @classmethod
-    def find_logs(
-        cls, path: Path | None = None
+    def find_traces(
+        cls,
+        path: Path | None = None,
+        extras: Optional[DictData] = None,
     ) -> Iterator[TraceData]:  # pragma: no cov
-        """Find trace logs."""
+        """Find trace logs.
+
+        :param path: (Path)
+        :param extras: An extra parameter that want to override core config.
+        """
         for file in sorted(
-            (path or config.log_path).glob("./run_id=*"),
+            (path or dynamic("trace_path", extras=extras)).glob("./run_id=*"),
             key=lambda f: f.lstat().st_mtime,
         ):
             yield TraceData.from_path(file)
 
     @classmethod
-    def find_log_with_id(
-        cls, run_id: str, force_raise: bool = True, *, path: Path | None = None
+    def find_trace_with_id(
+        cls,
+        run_id: str,
+        force_raise: bool = True,
+        *,
+        path: Path | None = None,
+        extras: Optional[DictData] = None,
     ) -> TraceData:
-        """Find trace log with an input specific run ID."""
-        base_path: Path = path or config.log_path
+        """Find trace log with an input specific run ID.
+
+        :param run_id: A running ID of trace log.
+        :param force_raise:
+        :param path:
+        :param extras: An extra parameter that want to override core config.
+        """
+        base_path: Path = path or dynamic("trace_path", extras=extras)
         file: Path = base_path / f"run_id={run_id}"
         if file.exists():
             return TraceData.from_path(file)
@@ -299,7 +379,8 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
     @property
     def pointer(self) -> Path:
         log_file: Path = (
-            config.log_path / f"run_id={self.parent_run_id or self.run_id}"
+            dynamic("trace_path", extras=self.extras)
+            / f"run_id={self.parent_run_id or self.run_id}"
         )
         if not log_file.exists():
             log_file.mkdir(parents=True)
@@ -340,18 +421,17 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
         :param message: A message after making.
         :param is_err: A flag for writing with an error trace or not.
         """
-        if not config.enable_write_log:
+        if not dynamic("enable_write_log", extras=self.extras):
             return
 
         write_file: str = "stderr" if is_err else "stdout"
-        trace_meta: TraceMeda = TraceMeda.make(mode=write_file, message=message)
+        trace_meta: TraceMeta = TraceMeta.make(mode=write_file, message=message)
 
         with (self.pointer / f"{write_file}.txt").open(
             mode="at", encoding="utf-8"
         ) as f:
-            f.write(
-                f"{config.log_format_file}\n".format(**trace_meta.model_dump())
-            )
+            fmt: str = dynamic("log_format_file", extras=self.extras)
+            f.write(f"{fmt}\n".format(**trace_meta.model_dump()))
 
         with (self.pointer / "metadata.json").open(
             mode="at", encoding="utf-8"
@@ -361,7 +441,7 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
     async def awriter(
         self, message: str, is_err: bool = False
     ) -> None:  # pragma: no cov
-        if not config.enable_write_log:
+        if not dynamic("enable_write_log", extras=self.extras):
             return
 
         try:
@@ -370,14 +450,13 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
             raise ImportError("Async mode need aiofiles package") from e
 
         write_file: str = "stderr" if is_err else "stdout"
-        trace_meta: TraceMeda = TraceMeda.make(mode=write_file, message=message)
+        trace_meta: TraceMeta = TraceMeta.make(mode=write_file, message=message)
 
         async with aiofiles.open(
             self.pointer / f"{write_file}.txt", mode="at", encoding="utf-8"
         ) as f:
-            await f.write(
-                f"{config.log_format_file}\n".format(**trace_meta.model_dump())
-            )
+            fmt: str = dynamic("log_format_file", extras=self.extras)
+            await f.write(f"{fmt}\n".format(**trace_meta.model_dump()))
 
         async with aiofiles.open(
             self.pointer / "metadata.json", mode="at", encoding="utf-8"
@@ -385,8 +464,8 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
             await f.write(trace_meta.model_dump_json() + "\n")
 
 
-class SQLiteTraceLog(BaseTraceLog):  # pragma: no cov
-    """Trace Log object that write trace log to the SQLite database file."""
+class SQLiteTrace(BaseTrace):  # pragma: no cov
+    """SQLite Trace dataclass that write trace log to the SQLite database file."""
 
     table_name: ClassVar[str] = "audits"
     schemas: ClassVar[
@@ -400,10 +479,21 @@ class SQLiteTraceLog(BaseTraceLog):  # pragma: no cov
         """
 
     @classmethod
-    def find_logs(cls) -> Iterator[DictStr]: ...
+    def find_traces(
+        cls,
+        path: Path | None = None,
+        extras: Optional[DictData] = None,
+    ) -> Iterator[TraceData]: ...
 
     @classmethod
-    def find_log_with_id(cls, run_id: str) -> DictStr: ...
+    def find_trace_with_id(
+        cls,
+        run_id: str,
+        force_raise: bool = True,
+        *,
+        path: Path | None = None,
+        extras: Optional[DictData] = None,
+    ) -> TraceData: ...
 
     def make_message(self, message: str) -> str: ...
 
@@ -412,25 +502,34 @@ class SQLiteTraceLog(BaseTraceLog):  # pragma: no cov
     def awriter(self, message: str, is_err: bool = False) -> None: ...
 
 
-TraceLog = Union[
-    FileTraceLog,
-    SQLiteTraceLog,
+Trace = TypeVar("Trace", bound=BaseTrace)
+TraceModel = Union[
+    FileTrace,
+    SQLiteTrace,
 ]
 
 
 def get_trace(
-    run_id: str, parent_run_id: str | None = None
-) -> TraceLog:  # pragma: no cov
-    """Get dynamic TraceLog object from the setting config."""
-    if config.log_path.is_file():
-        return SQLiteTraceLog(run_id, parent_run_id=parent_run_id)
-    return FileTraceLog(run_id, parent_run_id=parent_run_id)
-
-
-def get_trace_obj() -> type[TraceLog]:  # pragma: no cov
-    if config.log_path.is_file():
-        return SQLiteTraceLog
-    return FileTraceLog
+    run_id: str,
+    *,
+    parent_run_id: str | None = None,
+    extras: Optional[DictData] = None,
+) -> TraceModel:  # pragma: no cov
+    """Get dynamic Trace instance from the core config (it can override by an
+    extras argument) that passing running ID and parent running ID.
+
+    :param run_id: (str) A running ID.
+    :param parent_run_id: (str) A parent running ID.
+    :param extras: (DictData) An extra parameter that want to override the core
+        config values.
+
+    :rtype: TraceLog
+    """
+    if dynamic("trace_path", extras=extras).is_file():
+        return SQLiteTrace(
+            run_id, parent_run_id=parent_run_id, extras=(extras or {})
+        )
+    return FileTrace(run_id, parent_run_id=parent_run_id, extras=(extras or {}))
 
 
 class BaseAudit(BaseModel, ABC):
@@ -439,6 +538,10 @@ class BaseAudit(BaseModel, ABC):
     subclass like file, sqlite, etc.
     """
 
+    extras: DictData = Field(
+        default_factory=dict,
+        description="An extras parameter that want to override core config",
+    )
     name: str = Field(description="A workflow name.")
     release: datetime = Field(description="A release datetime.")
     type: str = Field(description="A running type before logging.")
@@ -459,10 +562,45 @@ class BaseAudit(BaseModel, ABC):
 
         :rtype: Self
         """
-        if config.enable_write_audit:
+        if dynamic("enable_write_audit", extras=self.extras):
             self.do_before()
         return self
 
+    @classmethod
+    @abstractmethod
+    def is_pointed(
+        cls,
+        name: str,
+        release: datetime,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> bool:
+        raise NotImplementedError(
+            "Audit should implement `is_pointed` class-method"
+        )
+
+    @classmethod
+    @abstractmethod
+    def find_audits(
+        cls, name: str, *, extras: Optional[DictData] = None
+    ) -> Iterator[Self]:
+        raise NotImplementedError(
+            "Audit should implement `find_audits` class-method"
+        )
+
+    @classmethod
+    @abstractmethod
+    def find_audit_with_release(
+        cls,
+        name: str,
+        release: datetime | None = None,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Self:
+        raise NotImplementedError(
+            "Audit should implement `find_audit_with_release` class-method"
+        )
+
     def do_before(self) -> None:  # pragma: no cov
         """To something before end up of initial log model."""
 
@@ -487,15 +625,20 @@ class FileAudit(BaseAudit):
         self.pointer().mkdir(parents=True, exist_ok=True)
 
     @classmethod
-    def find_audits(cls, name: str) -> Iterator[Self]:
+    def find_audits(
+        cls, name: str, *, extras: Optional[DictData] = None
+    ) -> Iterator[Self]:
         """Generate the audit data that found from logs path with specific a
         workflow name.
 
         :param name: A workflow name that want to search release logging data.
+        :param extras: An extra parameter that want to override core config.
 
         :rtype: Iterator[Self]
         """
-        pointer: Path = config.audit_path / f"workflow={name}"
+        pointer: Path = (
+            dynamic("audit_path", extras=extras) / f"workflow={name}"
+        )
         if not pointer.exists():
             raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
 
@@ -508,13 +651,16 @@ class FileAudit(BaseAudit):
         cls,
         name: str,
         release: datetime | None = None,
+        *,
+        extras: Optional[DictData] = None,
     ) -> Self:
         """Return the audit data that found from logs path with specific
         workflow name and release values. If a release does not pass to an input
         argument, it will return the latest release from the current log path.
 
-        :param name: A workflow name that want to search log.
-        :param release: A release datetime that want to search log.
+        :param name: (str) A workflow name that want to search log.
+        :param release: (datetime) A release datetime that want to search log.
+        :param extras: An extra parameter that want to override core config.
 
         :raise FileNotFoundError:
         :raise NotImplementedError: If an input release does not pass to this
@@ -526,7 +672,7 @@ class FileAudit(BaseAudit):
             raise NotImplementedError("Find latest log does not implement yet.")
 
         pointer: Path = (
-            config.audit_path
+            dynamic("audit_path", extras=extras)
             / f"workflow={name}/release={release:%Y%m%d%H%M%S}"
         )
         if not pointer.exists():
@@ -540,24 +686,31 @@ class FileAudit(BaseAudit):
             return cls.model_validate(obj=json.load(f))
 
     @classmethod
-    def is_pointed(cls, name: str, release: datetime) -> bool:
+    def is_pointed(
+        cls,
+        name: str,
+        release: datetime,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> bool:
         """Check the release log already pointed or created at the destination
         log path.
 
-        :param name: A workflow name.
-        :param release: A release datetime.
+        :param name: (str) A workflow name.
+        :param release: (datetime) A release datetime.
+        :param extras: An extra parameter that want to override core config.
 
         :rtype: bool
         :return: Return False if the release log was not pointed or created.
         """
         # NOTE: Return False if enable writing log flag does not set.
-        if not config.enable_write_audit:
+        if not dynamic("enable_write_audit", extras=extras):
             return False
 
         # NOTE: create pointer path that use the same logic of pointer method.
-        pointer: Path = config.audit_path / cls.filename_fmt.format(
-            name=name, release=release
-        )
+        pointer: Path = dynamic(
+            "audit_path", extras=extras
+        ) / cls.filename_fmt.format(name=name, release=release)
 
         return pointer.exists()
 
@@ -566,9 +719,9 @@ class FileAudit(BaseAudit):
 
         :rtype: Path
         """
-        return config.audit_path / self.filename_fmt.format(
-            name=self.name, release=self.release
-        )
+        return dynamic(
+            "audit_path", extras=self.extras
+        ) / self.filename_fmt.format(name=self.name, release=self.release)
 
     def save(self, excluded: list[str] | None) -> Self:
         """Save logging data that receive a context data from a workflow
@@ -579,10 +732,14 @@ class FileAudit(BaseAudit):
 
         :rtype: Self
         """
-        trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
+        trace: Trace = get_trace(
+            self.run_id,
+            parent_run_id=self.parent_run_id,
+            extras=self.extras,
+        )
 
         # NOTE: Check environ variable was set for real writing.
-        if not config.enable_write_audit:
+        if not dynamic("enable_write_audit", extras=self.extras):
             trace.debug("[LOG]: Skip writing log cause config was set")
             return self
 
@@ -617,43 +774,63 @@ class SQLiteAudit(BaseAudit):  # pragma: no cov
         primary key ( run_id )
         """
 
+    @classmethod
+    def is_pointed(
+        cls,
+        name: str,
+        release: datetime,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> bool: ...
+
+    @classmethod
+    def find_audits(
+        cls, name: str, *, extras: Optional[DictData] = None
+    ) -> Iterator[Self]: ...
+
+    @classmethod
+    def find_audit_with_release(
+        cls,
+        name: str,
+        release: datetime | None = None,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Self: ...
+
     def save(self, excluded: list[str] | None) -> SQLiteAudit:
         """Save logging data that receive a context data from a workflow
         execution result.
         """
-        trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
+        trace: Trace = get_trace(
+            self.run_id,
+            parent_run_id=self.parent_run_id,
+            extras=self.extras,
+        )
 
         # NOTE: Check environ variable was set for real writing.
-        if not config.enable_write_audit:
+        if not dynamic("enable_write_audit", extras=self.extras):
             trace.debug("[LOG]: Skip writing log cause config was set")
             return self
 
         raise NotImplementedError("SQLiteAudit does not implement yet.")
 
 
-class RemoteFileAudit(FileAudit):  # pragma: no cov
-    """Remote File Audit Pydantic Model."""
-
-    def save(self, excluded: list[str] | None) -> RemoteFileAudit: ...
-
-
-class RedisAudit(BaseAudit):  # pragma: no cov
-    """Redis Audit Pydantic Model."""
-
-    def save(self, excluded: list[str] | None) -> RedisAudit: ...
-
-
-Audit = Union[
+Audit = TypeVar("Audit", bound=BaseAudit)
+AuditModel = Union[
     FileAudit,
     SQLiteAudit,
 ]
 
 
-def get_audit() -> type[Audit]:  # pragma: no cov
+def get_audit(
+    extras: Optional[DictData] = None,
+) -> type[AuditModel]:  # pragma: no cov
     """Get an audit class that dynamic base on the config audit path value.
 
+    :param extras: An extra parameter that want to override the core config.
+
     :rtype: type[Audit]
     """
-    if config.audit_path.is_file():
+    if dynamic("audit_path", extras=extras).is_file():
         return SQLiteAudit
     return FileAudit