ddeutil-workflow 0.0.33__py3-none-any.whl → 0.0.34__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.33"
+__version__: str = "0.0.34"

ddeutil/workflow/__init__.py

@@ -9,6 +9,13 @@ from .audit import (
     Audit,
     get_audit,
 )
+from .call import (
+    ReturnTagFunc,
+    TagFunc,
+    extract_call,
+    make_registry,
+    tag,
+)
 from .conf import (
     Config,
     Loader,
@@ -28,13 +35,6 @@ from .exceptions import (
     UtilException,
     WorkflowException,
 )
-from .hook import (
-    ReturnTagFunc,
-    TagFunc,
-    extract_hook,
-    make_registry,
-    tag,
-)
 from .job import (
     Job,
     Strategy,
@@ -46,7 +46,13 @@ from .params import (
     Param,
     StrParam,
 )
-from .result import Result
+from .result import (
+    Result,
+    Status,
+    TraceLog,
+    default_gen_id,
+    get_dt_tznow,
+)
 from .scheduler import (
     Schedule,
     ScheduleWorkflow,
@@ -54,10 +60,10 @@ from .scheduler import (
     schedule_runner,
     schedule_task,
 )
-from .stage import (
+from .stages import (
     BashStage,
+    CallStage,
     EmptyStage,
-    HookStage,
     PyStage,
     Stage,
     TriggerStage,

ddeutil/workflow/api/route.py

@@ -110,7 +110,7 @@ async def get_workflow_logs(name: str):
                     exclude_unset=True,
                     exclude_defaults=True,
                 )
-                for log in get_audit().find_logs(name=name)
+                for log in get_audit().find_audits(name=name)
             ],
         }
     except FileNotFoundError:
@@ -123,7 +123,7 @@ async def get_workflow_logs(name: str):
 @workflow_route.get(path="/{name}/logs/{release}")
 async def get_workflow_release_log(name: str, release: str):
     try:
-        log: Audit = get_audit().find_log_with_release(
+        log: Audit = get_audit().find_audit_with_release(
             name=name, release=datetime.strptime(release, "%Y%m%d%H%M%S")
         )
     except FileNotFoundError:

ddeutil/workflow/audit.py

@@ -12,16 +12,15 @@ from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from datetime import datetime
 from pathlib import Path
-from typing import Any, ClassVar, Optional, Union
+from typing import ClassVar, Optional, Union
 
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, TupleStr
-from .conf import config, get_logger
-
-logger = get_logger("ddeutil.workflow")
+from .conf import config
+from .result import TraceLog
 
 __all__: TupleStr = (
     "get_audit",
@@ -55,7 +54,7 @@ class BaseAudit(BaseModel, ABC):
 
         :rtype: Self
         """
-        if config.enable_write_log:
+        if config.enable_write_audit:
             self.do_before()
         return self
 
@@ -83,8 +82,8 @@ class FileAudit(BaseAudit):
         self.pointer().mkdir(parents=True, exist_ok=True)
 
     @classmethod
-    def find_logs(cls, name: str) -> Iterator[Self]:
-        """Generate the logging data that found from logs path with specific a
+    def find_audits(cls, name: str) -> Iterator[Self]:
+        """Generate the audit data that found from logs path with specific a
         workflow name.
 
         :param name: A workflow name that want to search release logging data.
@@ -100,12 +99,12 @@ class FileAudit(BaseAudit):
                 yield cls.model_validate(obj=json.load(f))
 
     @classmethod
-    def find_log_with_release(
+    def find_audit_with_release(
         cls,
         name: str,
         release: datetime | None = None,
     ) -> Self:
-        """Return the logging data that found from logs path with specific
+        """Return the audit data that found from logs path with specific
         workflow name and release values. If a release does not pass to an input
         argument, it will return the latest release from the current log path.
 
@@ -147,7 +146,7 @@ class FileAudit(BaseAudit):
         :return: Return False if the release log was not pointed or created.
         """
         # NOTE: Return False if enable writing log flag does not set.
-        if not config.enable_write_log:
+        if not config.enable_write_audit:
             return False
 
         # NOTE: create pointer path that use the same logic of pointer method.
@@ -175,14 +174,11 @@ class FileAudit(BaseAudit):
 
         :rtype: Self
         """
-        from .utils import cut_id
+        trace: TraceLog = TraceLog(self.run_id, self.parent_run_id)
 
         # NOTE: Check environ variable was set for real writing.
-        if not config.enable_write_log:
-            logger.debug(
-                f"({cut_id(self.run_id)}) [LOG]: Skip writing log cause "
-                f"config was set"
-            )
+        if not config.enable_write_audit:
+            trace.debug("[LOG]: Skip writing log cause config was set")
             return self
 
         log_file: Path = self.pointer() / f"{self.run_id}.log"
@@ -200,34 +196,29 @@ class FileAudit(BaseAudit):
 class SQLiteAudit(BaseAudit):  # pragma: no cov
     """SQLite Audit Pydantic Model."""
 
-    @staticmethod
-    def meta() -> dict[str, Any]:
-        return {
-            "table": "workflow_log",
-            "ddl": """
-                workflow        str,
-                release         int,
-                type            str,
-                context         json,
-                parent_run_id   int,
-                run_id          int,
-                update          datetime
-                primary key ( run_id )
-                """,
-        }
+    table_name: ClassVar[str] = "workflow_log"
+    schemas: ClassVar[
+        str
+    ] = """
+        workflow        str,
+        release         int,
+        type            str,
+        context         json,
+        parent_run_id   int,
+        run_id          int,
+        update          datetime
+        primary key ( run_id )
+        """
 
     def save(self, excluded: list[str] | None) -> SQLiteAudit:
         """Save logging data that receive a context data from a workflow
         execution result.
         """
-        from .utils import cut_id
+        trace: TraceLog = TraceLog(self.run_id, self.parent_run_id)
 
         # NOTE: Check environ variable was set for real writing.
-        if not config.enable_write_log:
-            logger.debug(
-                f"({cut_id(self.run_id)}) [LOG]: Skip writing log cause "
-                f"config was set"
-            )
+        if not config.enable_write_audit:
+            trace.debug("[LOG]: Skip writing log cause config was set")
             return self
 
         raise NotImplementedError("SQLiteAudit does not implement yet.")

ddeutil/workflow/{hook.py → call.py}

@@ -60,7 +60,7 @@ def tag(
 
         @wraps(func)
         def wrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
-            # NOTE: Able to do anything before calling hook function.
+            # NOTE: Able to do anything before calling call function.
             return func(*args, **kwargs)
 
         return wrapped
@@ -79,9 +79,9 @@ def make_registry(submodule: str) -> dict[str, Registry]:
     :rtype: dict[str, Registry]
     """
     rs: dict[str, Registry] = {}
-    regis_hooks: list[str] = config.regis_hook
-    regis_hooks.extend(["ddeutil.vendors"])
-    for module in regis_hooks:
+    regis_calls: list[str] = config.regis_call
+    regis_calls.extend(["ddeutil.vendors"])
+    for module in regis_calls:
         # NOTE: try to sequential import task functions
         try:
             importer = import_module(f"{module}.{submodule}")
@@ -114,9 +114,9 @@ def make_registry(submodule: str) -> dict[str, Registry]:
 
 
 @dataclass(frozen=True)
-class HookSearchData:
-    """Hook Search dataclass that use for receive regular expression grouping
-    dict from searching hook string value.
+class CallSearchData:
+    """Call Search dataclass that use for receive regular expression grouping
+    dict from searching call string value.
     """
 
     path: str
@@ -124,49 +124,49 @@ class HookSearchData:
     tag: str
 
 
-def extract_hook(hook: str) -> Callable[[], TagFunc]:
-    """Extract Hook function from string value to hook partial function that
+def extract_call(call: str) -> Callable[[], TagFunc]:
+    """Extract Call function from string value to call partial function that
     does run it at runtime.
 
-    :raise NotImplementedError: When the searching hook's function result does
+    :raise NotImplementedError: When the searching call's function result does
         not exist in the registry.
-    :raise NotImplementedError: When the searching hook's tag result does not
+    :raise NotImplementedError: When the searching call's tag result does not
         exist in the registry with its function key.
 
-    :param hook: A hook value that able to match with Task regex.
+    :param call: A call value that able to match with Task regex.
 
-        The format of hook value should contain 3 regular expression groups
+        The format of call value should contain 3 regular expression groups
     which match with the below config format:
 
         >>> "^(?P<path>[^/@]+)/(?P<func>[^@]+)@(?P<tag>.+)$"
 
     Examples:
-        >>> extract_hook("tasks/el-postgres-to-delta@polars")
+        >>> extract_call("tasks/el-postgres-to-delta@polars")
         ...
-        >>> extract_hook("tasks/return-type-not-valid@raise")
+        >>> extract_call("tasks/return-type-not-valid@raise")
         ...
 
     :rtype: Callable[[], TagFunc]
     """
-    if not (found := Re.RE_TASK_FMT.search(hook)):
+    if not (found := Re.RE_TASK_FMT.search(call)):
         raise ValueError(
-            f"Hook {hook!r} does not match with hook format regex."
+            f"Call {call!r} does not match with call format regex."
         )
 
-    # NOTE: Pass the searching hook string to `path`, `func`, and `tag`.
-    hook: HookSearchData = HookSearchData(**found.groupdict())
+    # NOTE: Pass the searching call string to `path`, `func`, and `tag`.
+    call: CallSearchData = CallSearchData(**found.groupdict())
 
     # NOTE: Registry object should implement on this package only.
-    rgt: dict[str, Registry] = make_registry(f"{hook.path}")
-    if hook.func not in rgt:
+    rgt: dict[str, Registry] = make_registry(f"{call.path}")
+    if call.func not in rgt:
         raise NotImplementedError(
-            f"``REGISTER-MODULES.{hook.path}.registries`` does not "
-            f"implement registry: {hook.func!r}."
+            f"``REGISTER-MODULES.{call.path}.registries`` does not "
+            f"implement registry: {call.func!r}."
         )
 
-    if hook.tag not in rgt[hook.func]:
+    if call.tag not in rgt[call.func]:
         raise NotImplementedError(
-            f"tag: {hook.tag!r} does not found on registry func: "
-            f"``REGISTER-MODULES.{hook.path}.registries.{hook.func}``"
+            f"tag: {call.tag!r} does not found on registry func: "
+            f"``REGISTER-MODULES.{call.path}.registries.{call.func}``"
         )
-    return rgt[hook.func][hook.tag]
+    return rgt[call.func][call.tag]

ddeutil/workflow/conf.py

@@ -38,6 +38,7 @@ __all__: TupleStr = (
     "SimLoad",
     "Loader",
     "config",
+    "glob_files",
 )
 
 
@@ -97,9 +98,9 @@ class Config(BaseConfig):  # pragma: no cov
 
     # NOTE: Register
     @property
-    def regis_hook(self) -> list[str]:
-        regis_hook_str: str = env("CORE_REGISTRY", ".")
-        return [r.strip() for r in regis_hook_str.split(",")]
+    def regis_call(self) -> list[str]:
+        regis_call_str: str = env("CORE_REGISTRY", ".")
+        return [r.strip() for r in regis_call_str.split(",")]
 
     @property
     def regis_filter(self) -> list[str]:
@@ -129,16 +130,26 @@ class Config(BaseConfig):  # pragma: no cov
         )
 
     @property
-    def enable_rotate_file(self) -> bool:
-        return str2bool(env("LOG_ENABLE_ROTATED_FILE", "false"))
+    def log_format_file(self) -> str:
+        return env(
+            "LOG_FORMAT_FILE",
+            (
+                "{datetime} ({process:5d}, {thread:5d}) {message:120s} "
+                "({filename}:{lineno})"
+            ),
+        )
+
+    @property
+    def enable_write_log(self) -> bool:
+        return str2bool(env("LOG_ENABLE_WRITE", "false"))
 
     # NOTE: Audit Log
     @property
     def audit_path(self) -> Path:
-        return Path(env("AUDIT_PATH", "./logs"))
+        return Path(env("AUDIT_PATH", "./audits"))
 
     @property
-    def enable_write_log(self) -> bool:
+    def enable_write_audit(self) -> bool:
         return str2bool(env("AUDIT_ENABLE_WRITE", "false"))
 
     @property
@@ -254,18 +265,22 @@ class SimLoad:
         conf_path: Path,
         externals: DictData | None = None,
     ) -> None:
+        self.conf_path: Path = conf_path
+        self.externals: DictData = externals or {}
+
         self.data: DictData = {}
         for file in glob_files(conf_path):
 
-            if data := self.filter_suffix(file, name):
+            if self.is_ignore(file, conf_path):
+                continue
+
+            if data := self.filter_suffix(file, name=name):
                 self.data = data
 
         # VALIDATE: check the data that reading should not empty.
         if not self.data:
             raise ValueError(f"Config {name!r} does not found on conf path")
 
-        self.conf_path: Path = conf_path
-        self.externals: DictData = externals or {}
         self.data.update(self.externals)
 
     @classmethod
@@ -283,8 +298,10 @@ class SimLoad:
 
         :param obj: An object that want to validate matching before return.
         :param conf_path: A config object.
-        :param included:
-        :param excluded:
+        :param included: An excluded list of data key that want to reject this
+            data if any key exist.
+        :param excluded: An included list of data key that want to filter from
+            data.
 
         :rtype: Iterator[tuple[str, DictData]]
         """
@@ -293,6 +310,9 @@ class SimLoad:
 
             for key, data in cls.filter_suffix(file).items():
 
+                if cls.is_ignore(file, conf_path):
+                    continue
+
                 if key in exclude:
                     continue
 
@@ -303,11 +323,26 @@ class SimLoad:
                         else data
                     )
 
+    @classmethod
+    def is_ignore(cls, file: Path, conf_path: Path) -> bool:
+        ignore_file: Path = conf_path / ".confignore"
+        ignore: list[str] = []
+        if ignore_file.exists():
+            ignore = ignore_file.read_text(encoding="utf-8").splitlines()
+
+        if any(
+            (file.match(f"**/{pattern}/*") or file.match(f"**/{pattern}*"))
+            for pattern in ignore
+        ):
+            return True
+        return False
+
     @classmethod
     def filter_suffix(cls, file: Path, name: str | None = None) -> DictData:
         if any(file.suffix.endswith(s) for s in (".yml", ".yaml")):
             values: DictData = YamlFlResolve(file).read()
             return values.get(name, {}) if name else values
+
         return {}
 
     @cached_property