ddeutil-workflow 0.0.46__py3-none-any.whl → 0.0.48__py3-none-any.whl

ddeutil/workflow/logs.py

@@ -9,33 +9,65 @@
 from __future__ import annotations
 
 import json
+import logging
 import os
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
+from dataclasses import field
 from datetime import datetime
+from functools import lru_cache
 from inspect import Traceback, currentframe, getframeinfo
 from pathlib import Path
 from threading import get_ident
 from typing import ClassVar, Literal, Optional, Union
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, ValidationInfo
 from pydantic.dataclasses import dataclass
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, TupleStr
-from .conf import config, get_logger
+from .conf import config, dynamic
 from .utils import cut_id, get_dt_now
 
+
+@lru_cache
+def get_logger(name: str):
+    """Return logger object with an input module name.
+
+    :param name: A module name that want to log.
+    """
+    lg = logging.getLogger(name)
+
+    # NOTE: Developers using this package can then disable all logging just for
+    #   this package by;
+    #
+    #   `logging.getLogger('ddeutil.workflow').propagate = False`
+    #
+    lg.addHandler(logging.NullHandler())
+
+    formatter = logging.Formatter(
+        fmt=config.log_format,
+        datefmt=config.log_datetime_format,
+    )
+    stream = logging.StreamHandler()
+    stream.setFormatter(formatter)
+    lg.addHandler(stream)
+
+    lg.setLevel(logging.DEBUG if config.debug else logging.INFO)
+    return lg
+
+
 logger = get_logger("ddeutil.workflow")
 
 __all__: TupleStr = (
     "FileTraceLog",
     "SQLiteTraceLog",
     "TraceData",
-    "TraceMeda",
+    "TraceMeta",
     "TraceLog",
     "get_dt_tznow",
+    "get_logger",
     "get_trace",
     "get_trace_obj",
     "get_audit",
@@ -53,7 +85,9 @@ def get_dt_tznow() -> datetime:  # pragma: no cov
     return get_dt_now(tz=config.tz)
 
 
-class TraceMeda(BaseModel):  # pragma: no cov
+class TraceMeta(BaseModel):  # pragma: no cov
+    """Trace Meta model."""
+
     mode: Literal["stdout", "stderr"]
     datetime: str
     process: int
@@ -63,14 +97,28 @@ class TraceMeda(BaseModel):  # pragma: no cov
     lineno: int
 
     @classmethod
-    def make(cls, mode: Literal["stdout", "stderr"], message: str) -> Self:
-        """Make a TraceMeda instance."""
+    def make(
+        cls,
+        mode: Literal["stdout", "stderr"],
+        message: str,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Self:
+        """Make the current TraceMeta instance that catching local state.
+
+        :rtype: Self
+        """
         frame_info: Traceback = getframeinfo(
             currentframe().f_back.f_back.f_back
         )
+        extras: DictData = extras or {}
         return cls(
             mode=mode,
-            datetime=get_dt_tznow().strftime(config.log_datetime_format),
+            datetime=(
+                get_dt_now(tz=dynamic("tz", extras=extras)).strftime(
+                    dynamic("log_datetime_format", extras=extras)
+                )
+            ),
             process=os.getpid(),
             thread=get_ident(),
             message=message,
@@ -84,7 +132,7 @@ class TraceData(BaseModel):  # pragma: no cov
 
     stdout: str = Field(description="A standard output trace data.")
     stderr: str = Field(description="A standard error trace data.")
-    meta: list[TraceMeda] = Field(
+    meta: list[TraceMeta] = Field(
         default_factory=list,
         description=(
             "A metadata mapping of this output and error before making it to "
@@ -126,7 +174,8 @@ class BaseTraceLog(ABC):  # pragma: no cov
     """Base Trace Log dataclass object."""
 
     run_id: str
-    parent_run_id: Optional[str] = None
+    parent_run_id: Optional[str] = field(default=None)
+    extras: DictData = field(default_factory=dict, compare=False, repr=False)
 
     @abstractmethod
     def writer(self, message: str, is_err: bool = False) -> None:
@@ -148,6 +197,9 @@ class BaseTraceLog(ABC):  # pragma: no cov
         :param message:
         :param is_err:
         """
+        raise NotImplementedError(
+            "Create async writer logic for this trace object before using."
+        )
 
     @abstractmethod
     def make_message(self, message: str) -> str:
@@ -169,7 +221,7 @@ class BaseTraceLog(ABC):  # pragma: no cov
         """
         msg: str = self.make_message(message)
 
-        if config.debug:
+        if dynamic("debug", extras=self.extras):
             self.writer(msg)
 
         logger.debug(msg, stacklevel=2)
@@ -221,7 +273,7 @@ class BaseTraceLog(ABC):  # pragma: no cov
         :param message: (str) A message that want to log.
         """
         msg: str = self.make_message(message)
-        if config.debug:
+        if dynamic("debug", extras=self.extras):
             await self.awriter(msg)
         logger.info(msg, stacklevel=2)
 
@@ -271,21 +323,38 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
 
     @classmethod
     def find_logs(
-        cls, path: Path | None = None
+        cls,
+        path: Path | None = None,
+        extras: Optional[DictData] = None,
     ) -> Iterator[TraceData]:  # pragma: no cov
-        """Find trace logs."""
+        """Find trace logs.
+
+        :param path: (Path)
+        :param extras: An extra parameter that want to override core config.
+        """
         for file in sorted(
-            (path or config.log_path).glob("./run_id=*"),
+            (path or dynamic("log_path", extras=extras)).glob("./run_id=*"),
             key=lambda f: f.lstat().st_mtime,
         ):
             yield TraceData.from_path(file)
 
     @classmethod
     def find_log_with_id(
-        cls, run_id: str, force_raise: bool = True, *, path: Path | None = None
+        cls,
+        run_id: str,
+        force_raise: bool = True,
+        *,
+        path: Path | None = None,
+        extras: Optional[DictData] = None,
     ) -> TraceData:
-        """Find trace log with an input specific run ID."""
-        base_path: Path = path or config.log_path
+        """Find trace log with an input specific run ID.
+
+        :param run_id: A running ID of trace log.
+        :param force_raise:
+        :param path:
+        :param extras: An extra parameter that want to override core config.
+        """
+        base_path: Path = path or dynamic("log_path", extras=extras)
         file: Path = base_path / f"run_id={run_id}"
         if file.exists():
             return TraceData.from_path(file)
@@ -299,7 +368,8 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
     @property
     def pointer(self) -> Path:
         log_file: Path = (
-            config.log_path / f"run_id={self.parent_run_id or self.run_id}"
+            dynamic("log_path", extras=self.extras)
+            / f"run_id={self.parent_run_id or self.run_id}"
         )
         if not log_file.exists():
             log_file.mkdir(parents=True)
@@ -340,18 +410,17 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
         :param message: A message after making.
         :param is_err: A flag for writing with an error trace or not.
         """
-        if not config.enable_write_log:
+        if not dynamic("enable_write_log", extras=self.extras):
             return
 
         write_file: str = "stderr" if is_err else "stdout"
-        trace_meta: TraceMeda = TraceMeda.make(mode=write_file, message=message)
+        trace_meta: TraceMeta = TraceMeta.make(mode=write_file, message=message)
 
         with (self.pointer / f"{write_file}.txt").open(
             mode="at", encoding="utf-8"
         ) as f:
-            f.write(
-                f"{config.log_format_file}\n".format(**trace_meta.model_dump())
-            )
+            fmt: str = dynamic("log_format_file", extras=self.extras)
+            f.write(f"{fmt}\n".format(**trace_meta.model_dump()))
 
         with (self.pointer / "metadata.json").open(
             mode="at", encoding="utf-8"
@@ -361,7 +430,7 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
     async def awriter(
         self, message: str, is_err: bool = False
     ) -> None:  # pragma: no cov
-        if not config.enable_write_log:
+        if not dynamic("enable_write_log", extras=self.extras):
             return
 
         try:
@@ -370,14 +439,13 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
             raise ImportError("Async mode need aiofiles package") from e
 
         write_file: str = "stderr" if is_err else "stdout"
-        trace_meta: TraceMeda = TraceMeda.make(mode=write_file, message=message)
+        trace_meta: TraceMeta = TraceMeta.make(mode=write_file, message=message)
 
         async with aiofiles.open(
             self.pointer / f"{write_file}.txt", mode="at", encoding="utf-8"
         ) as f:
-            await f.write(
-                f"{config.log_format_file}\n".format(**trace_meta.model_dump())
-            )
+            fmt: str = dynamic("log_format_file", extras=self.extras)
+            await f.write(f"{fmt}\n".format(**trace_meta.model_dump()))
 
         async with aiofiles.open(
             self.pointer / "metadata.json", mode="at", encoding="utf-8"
@@ -419,16 +487,38 @@ TraceLog = Union[
 
 
 def get_trace(
-    run_id: str, parent_run_id: str | None = None
+    run_id: str,
+    parent_run_id: str | None = None,
+    *,
+    extras: Optional[DictData] = None,
 ) -> TraceLog:  # pragma: no cov
-    """Get dynamic TraceLog object from the setting config."""
-    if config.log_path.is_file():
-        return SQLiteTraceLog(run_id, parent_run_id=parent_run_id)
-    return FileTraceLog(run_id, parent_run_id=parent_run_id)
+    """Get dynamic TraceLog object from the setting config.
+
+    :param run_id: A running ID.
+    :param parent_run_id: A parent running ID.
+    :param extras: An extra parameter that want to override the core config.
+
+    :rtype: TraceLog
+    """
+    if dynamic("log_path", extras=extras).is_file():
+        return SQLiteTraceLog(
+            run_id, parent_run_id=parent_run_id, extras=(extras or {})
+        )
+    return FileTraceLog(
+        run_id, parent_run_id=parent_run_id, extras=(extras or {})
+    )
+
+
+def get_trace_obj(
+    extras: Optional[DictData] = None,
+) -> type[TraceLog]:  # pragma: no cov
+    """Get trace object.
 
+    :param extras: An extra parameter that want to override the core config.
 
-def get_trace_obj() -> type[TraceLog]:  # pragma: no cov
-    if config.log_path.is_file():
+    :rtype: type[TraceLog]
+    """
+    if dynamic("log_path", extras=extras).is_file():
         return SQLiteTraceLog
     return FileTraceLog
 
@@ -439,6 +529,10 @@ class BaseAudit(BaseModel, ABC):
     subclass like file, sqlite, etc.
     """
 
+    extras: DictData = Field(
+        default_factory=dict,
+        description="An extras parameter that want to override core config",
+    )
     name: str = Field(description="A workflow name.")
     release: datetime = Field(description="A release datetime.")
     type: str = Field(description="A running type before logging.")
@@ -454,12 +548,12 @@ class BaseAudit(BaseModel, ABC):
     execution_time: float = Field(default=0, description="An execution time.")
 
     @model_validator(mode="after")
-    def __model_action(self) -> Self:
+    def __model_action(self, info: ValidationInfo) -> Self:
         """Do before the Audit action with WORKFLOW_AUDIT_ENABLE_WRITE env variable.
 
         :rtype: Self
         """
-        if config.enable_write_audit:
+        if dynamic("enable_write_audit", extras=info.data.get("extras")):
             self.do_before()
         return self
 
@@ -487,15 +581,20 @@ class FileAudit(BaseAudit):
         self.pointer().mkdir(parents=True, exist_ok=True)
 
     @classmethod
-    def find_audits(cls, name: str) -> Iterator[Self]:
+    def find_audits(
+        cls, name: str, *, extras: Optional[DictData] = None
+    ) -> Iterator[Self]:
         """Generate the audit data that found from logs path with specific a
         workflow name.
 
         :param name: A workflow name that want to search release logging data.
+        :param extras: An extra parameter that want to override core config.
 
         :rtype: Iterator[Self]
         """
-        pointer: Path = config.audit_path / f"workflow={name}"
+        pointer: Path = (
+            dynamic("audit_path", extras=extras) / f"workflow={name}"
+        )
         if not pointer.exists():
             raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
 
@@ -508,13 +607,16 @@ class FileAudit(BaseAudit):
         cls,
         name: str,
         release: datetime | None = None,
+        *,
+        extras: Optional[DictData] = None,
     ) -> Self:
         """Return the audit data that found from logs path with specific
         workflow name and release values. If a release does not pass to an input
         argument, it will return the latest release from the current log path.
 
-        :param name: A workflow name that want to search log.
-        :param release: A release datetime that want to search log.
+        :param name: (str) A workflow name that want to search log.
+        :param release: (datetime) A release datetime that want to search log.
+        :param extras: An extra parameter that want to override core config.
 
         :raise FileNotFoundError:
         :raise NotImplementedError: If an input release does not pass to this
@@ -526,7 +628,7 @@ class FileAudit(BaseAudit):
             raise NotImplementedError("Find latest log does not implement yet.")
 
         pointer: Path = (
-            config.audit_path
+            dynamic("audit_path", extras=extras)
             / f"workflow={name}/release={release:%Y%m%d%H%M%S}"
         )
         if not pointer.exists():
@@ -540,24 +642,31 @@ class FileAudit(BaseAudit):
             return cls.model_validate(obj=json.load(f))
 
     @classmethod
-    def is_pointed(cls, name: str, release: datetime) -> bool:
+    def is_pointed(
+        cls,
+        name: str,
+        release: datetime,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> bool:
         """Check the release log already pointed or created at the destination
         log path.
 
-        :param name: A workflow name.
-        :param release: A release datetime.
+        :param name: (str) A workflow name.
+        :param release: (datetime) A release datetime.
+        :param extras: An extra parameter that want to override core config.
 
         :rtype: bool
         :return: Return False if the release log was not pointed or created.
         """
         # NOTE: Return False if enable writing log flag does not set.
-        if not config.enable_write_audit:
+        if not dynamic("enable_write_audit", extras=extras):
             return False
 
         # NOTE: create pointer path that use the same logic of pointer method.
-        pointer: Path = config.audit_path / cls.filename_fmt.format(
-            name=name, release=release
-        )
+        pointer: Path = dynamic(
+            "audit_path", extras=extras
+        ) / cls.filename_fmt.format(name=name, release=release)
 
         return pointer.exists()
 
@@ -566,9 +675,9 @@ class FileAudit(BaseAudit):
 
         :rtype: Path
         """
-        return config.audit_path / self.filename_fmt.format(
-            name=self.name, release=self.release
-        )
+        return dynamic(
+            "audit_path", extras=self.extras
+        ) / self.filename_fmt.format(name=self.name, release=self.release)
 
     def save(self, excluded: list[str] | None) -> Self:
         """Save logging data that receive a context data from a workflow
@@ -582,7 +691,7 @@ class FileAudit(BaseAudit):
         trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
 
         # NOTE: Check environ variable was set for real writing.
-        if not config.enable_write_audit:
+        if not dynamic("enable_write_audit", extras=self.extras):
             trace.debug("[LOG]: Skip writing log cause config was set")
             return self
 
@@ -624,7 +733,7 @@ class SQLiteAudit(BaseAudit):  # pragma: no cov
         trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
 
         # NOTE: Check environ variable was set for real writing.
-        if not config.enable_write_audit:
+        if not dynamic("enable_write_audit", extras=self.extras):
             trace.debug("[LOG]: Skip writing log cause config was set")
             return self
 
@@ -649,11 +758,15 @@ Audit = Union[
 ]
 
 
-def get_audit() -> type[Audit]:  # pragma: no cov
+def get_audit(
+    extras: Optional[DictData] = None,
+) -> type[Audit]:  # pragma: no cov
     """Get an audit class that dynamic base on the config audit path value.
 
+    :param extras: An extra parameter that want to override the core config.
+
     :rtype: type[Audit]
     """
-    if config.audit_path.is_file():
+    if dynamic("audit_path", extras=extras).is_file():
         return SQLiteAudit
     return FileAudit

ddeutil/workflow/result.py

@@ -3,6 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+# [x] Use dynamic config
 """Result module. It is the data context transfer objects that use by all object
 in this package. This module provide Status enum object and Result dataclass.
 """
@@ -18,18 +19,10 @@ from pydantic.dataclasses import dataclass
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
-from .__types import DictData, TupleStr
+from .__types import DictData
+from .conf import dynamic
 from .logs import TraceLog, get_dt_tznow, get_trace
-from .utils import default_gen_id, gen_id
-
-__all__: TupleStr = (
-    "SUCCESS",
-    "FAILED",
-    "WAIT",
-    "SKIP",
-    "Result",
-    "Status",
-)
+from .utils import default_gen_id, gen_id, get_dt_now
 
 
 class Status(IntEnum):
@@ -62,6 +55,10 @@ class Result:
 
         For comparison property, this result will use ``status``, ``context``,
     and ``_run_id`` fields to comparing with other result instance.
+
+    Warning:
+        I use dataclass object instead of Pydantic model object because context
+    field that keep dict value change its ID when update new value to it.
     """
 
     status: Status = field(default=WAIT)
@@ -72,7 +69,7 @@ class Result:
     ts: datetime = field(default_factory=get_dt_tznow, compare=False)
 
     trace: Optional[TraceLog] = field(default=None, compare=False, repr=False)
-    extras: DictData = field(default_factory=dict)
+    extras: DictData = field(default_factory=dict, compare=False, repr=False)
 
     @classmethod
     def construct_with_rs_or_id(
@@ -87,11 +84,11 @@ class Result:
         """Create the Result object or set parent running id if passing Result
         object.
 
-        :param result:
-        :param run_id:
-        :param parent_run_id:
-        :param id_logic:
-        :param extras:
+        :param result: A Result instance.
+        :param run_id: A running ID.
+        :param parent_run_id: A parent running ID.
+        :param id_logic: A logic function that use to generate a running ID.
+        :param extras: An extra parameter that want to override the core config.
 
         :rtype: Self
         """
@@ -116,17 +113,22 @@ class Result:
         :rtype: Self
         """
         if self.trace is None:  # pragma: no cov
-            self.trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
+            self.trace: TraceLog = get_trace(
+                self.run_id, self.parent_run_id, extras=self.extras
+            )
         return self
 
     def set_parent_run_id(self, running_id: str) -> Self:
         """Set a parent running ID.
 
-        :param running_id: A running ID that want to update on this model.
+        :param running_id: (str) A running ID that want to update on this model.
+
         :rtype: Self
         """
         self.parent_run_id: str = running_id
-        self.trace: TraceLog = get_trace(self.run_id, running_id)
+        self.trace: TraceLog = get_trace(
+            self.run_id, running_id, extras=self.extras
+        )
         return self
 
     def catch(
@@ -157,4 +159,6 @@ class Result:
 
         :rtype: float
         """
-        return (get_dt_tznow() - self.ts).total_seconds()
+        return (
+            get_dt_now(tz=dynamic("tz", extras=self.extras)) - self.ts
+        ).total_seconds()

ddeutil/workflow/reusables.py

@@ -10,7 +10,6 @@ from __future__ import annotations
 import inspect
 import logging
 from ast import Call, Constant, Expr, Module, Name, parse
-from dataclasses import dataclass
 from datetime import datetime
 from functools import wraps
 from importlib import import_module
@@ -23,6 +22,7 @@ except ImportError:
 
 from ddeutil.core import getdot, import_string, lazy
 from ddeutil.io import search_env_replace
+from pydantic.dataclasses import dataclass
 
 from .__types import DictData, Re
 from .conf import dynamic
@@ -437,6 +437,7 @@ Registry = dict[str, Callable[[], TagFunc]]
 
 def make_registry(
     submodule: str,
+    *,
     registries: Optional[list[str]] = None,
 ) -> dict[str, Registry]:
     """Return registries of all functions that able to called with task.
@@ -533,19 +534,18 @@ def extract_call(
 
     call: CallSearchData = CallSearchData(**found.groupdict())
     rgt: dict[str, Registry] = make_registry(
-        submodule=f"{call.path}",
-        registries=registries,
+        submodule=f"{call.path}", registries=registries
     )
 
     if call.func not in rgt:
         raise NotImplementedError(
-            f"`REGISTER-MODULES.{call.path}.registries` not implement "
+            f"`REGISTERS.{call.path}.registries` not implement "
             f"registry: {call.func!r}."
         )
 
     if call.tag not in rgt[call.func]:
         raise NotImplementedError(
             f"tag: {call.tag!r} not found on registry func: "
-            f"`REGISTER-MODULES.{call.path}.registries.{call.func}`"
+            f"`REGISTER.{call.path}.registries.{call.func}`"
         )
     return rgt[call.func][call.tag]