ddeutil-workflow 0.0.33__py3-none-any.whl → 0.0.35__py3-none-any.whl

ddeutil/workflow/{hook.py → caller.py}

@@ -60,7 +60,7 @@ def tag(
 
         @wraps(func)
         def wrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
-            # NOTE: Able to do anything before calling hook function.
+            # NOTE: Able to do anything before calling the call function.
             return func(*args, **kwargs)
 
         return wrapped
@@ -79,9 +79,9 @@ def make_registry(submodule: str) -> dict[str, Registry]:
     :rtype: dict[str, Registry]
     """
     rs: dict[str, Registry] = {}
-    regis_hooks: list[str] = config.regis_hook
-    regis_hooks.extend(["ddeutil.vendors"])
-    for module in regis_hooks:
+    regis_calls: list[str] = config.regis_call
+    regis_calls.extend(["ddeutil.vendors"])
+    for module in regis_calls:
         # NOTE: try to sequential import task functions
         try:
             importer = import_module(f"{module}.{submodule}")
@@ -114,9 +114,9 @@ def make_registry(submodule: str) -> dict[str, Registry]:
 
 
 @dataclass(frozen=True)
-class HookSearchData:
-    """Hook Search dataclass that use for receive regular expression grouping
-    dict from searching hook string value.
+class CallSearchData:
+    """Call Search dataclass that use for receive regular expression grouping
+    dict from searching call string value.
     """
 
     path: str
@@ -124,49 +124,49 @@ class HookSearchData:
     tag: str
 
 
-def extract_hook(hook: str) -> Callable[[], TagFunc]:
-    """Extract Hook function from string value to hook partial function that
+def extract_call(call: str) -> Callable[[], TagFunc]:
+    """Extract Call function from string value to call partial function that
     does run it at runtime.
 
-    :raise NotImplementedError: When the searching hook's function result does
+    :raise NotImplementedError: When the searching call's function result does
         not exist in the registry.
-    :raise NotImplementedError: When the searching hook's tag result does not
+    :raise NotImplementedError: When the searching call's tag result does not
         exist in the registry with its function key.
 
-    :param hook: A hook value that able to match with Task regex.
+    :param call: A call value that able to match with Task regex.
 
-        The format of hook value should contain 3 regular expression groups
+        The format of call value should contain 3 regular expression groups
     which match with the below config format:
 
         >>> "^(?P<path>[^/@]+)/(?P<func>[^@]+)@(?P<tag>.+)$"
 
     Examples:
-        >>> extract_hook("tasks/el-postgres-to-delta@polars")
+        >>> extract_call("tasks/el-postgres-to-delta@polars")
         ...
-        >>> extract_hook("tasks/return-type-not-valid@raise")
+        >>> extract_call("tasks/return-type-not-valid@raise")
         ...
 
     :rtype: Callable[[], TagFunc]
     """
-    if not (found := Re.RE_TASK_FMT.search(hook)):
+    if not (found := Re.RE_TASK_FMT.search(call)):
         raise ValueError(
-            f"Hook {hook!r} does not match with hook format regex."
+            f"Call {call!r} does not match with the call regex format."
         )
 
-    # NOTE: Pass the searching hook string to `path`, `func`, and `tag`.
-    hook: HookSearchData = HookSearchData(**found.groupdict())
+    # NOTE: Pass the searching call string to `path`, `func`, and `tag`.
+    call: CallSearchData = CallSearchData(**found.groupdict())
 
     # NOTE: Registry object should implement on this package only.
-    rgt: dict[str, Registry] = make_registry(f"{hook.path}")
-    if hook.func not in rgt:
+    rgt: dict[str, Registry] = make_registry(f"{call.path}")
+    if call.func not in rgt:
         raise NotImplementedError(
-            f"``REGISTER-MODULES.{hook.path}.registries`` does not "
-            f"implement registry: {hook.func!r}."
+            f"`REGISTER-MODULES.{call.path}.registries` does not "
+            f"implement registry: {call.func!r}."
         )
 
-    if hook.tag not in rgt[hook.func]:
+    if call.tag not in rgt[call.func]:
         raise NotImplementedError(
-            f"tag: {hook.tag!r} does not found on registry func: "
-            f"``REGISTER-MODULES.{hook.path}.registries.{hook.func}``"
+            f"tag: {call.tag!r} does not found on registry func: "
+            f"`REGISTER-MODULES.{call.path}.registries.{call.func}`"
         )
-    return rgt[hook.func][hook.tag]
+    return rgt[call.func][call.tag]

ddeutil/workflow/conf.py

@@ -38,6 +38,7 @@ __all__: TupleStr = (
     "SimLoad",
     "Loader",
     "config",
+    "glob_files",
 )
 
 
@@ -97,9 +98,9 @@ class Config(BaseConfig):  # pragma: no cov
 
     # NOTE: Register
     @property
-    def regis_hook(self) -> list[str]:
-        regis_hook_str: str = env("CORE_REGISTRY", ".")
-        return [r.strip() for r in regis_hook_str.split(",")]
+    def regis_call(self) -> list[str]:
+        regis_call_str: str = env("CORE_REGISTRY", ".")
+        return [r.strip() for r in regis_call_str.split(",")]
 
     @property
     def regis_filter(self) -> list[str]:
@@ -129,16 +130,26 @@ class Config(BaseConfig):  # pragma: no cov
         )
 
     @property
-    def enable_rotate_file(self) -> bool:
-        return str2bool(env("LOG_ENABLE_ROTATED_FILE", "false"))
+    def log_format_file(self) -> str:
+        return env(
+            "LOG_FORMAT_FILE",
+            (
+                "{datetime} ({process:5d}, {thread:5d}) {message:120s} "
+                "({filename}:{lineno})"
+            ),
+        )
+
+    @property
+    def enable_write_log(self) -> bool:
+        return str2bool(env("LOG_ENABLE_WRITE", "false"))
 
     # NOTE: Audit Log
     @property
     def audit_path(self) -> Path:
-        return Path(env("AUDIT_PATH", "./logs"))
+        return Path(env("AUDIT_PATH", "./audits"))
 
     @property
-    def enable_write_log(self) -> bool:
+    def enable_write_audit(self) -> bool:
         return str2bool(env("AUDIT_ENABLE_WRITE", "false"))
 
     @property
@@ -254,18 +265,22 @@ class SimLoad:
         conf_path: Path,
         externals: DictData | None = None,
     ) -> None:
+        self.conf_path: Path = conf_path
+        self.externals: DictData = externals or {}
+
         self.data: DictData = {}
         for file in glob_files(conf_path):
 
-            if data := self.filter_suffix(file, name):
+            if self.is_ignore(file, conf_path):
+                continue
+
+            if data := self.filter_suffix(file, name=name):
                 self.data = data
 
         # VALIDATE: check the data that reading should not empty.
         if not self.data:
             raise ValueError(f"Config {name!r} does not found on conf path")
 
-        self.conf_path: Path = conf_path
-        self.externals: DictData = externals or {}
         self.data.update(self.externals)
 
     @classmethod
@@ -283,8 +298,10 @@ class SimLoad:
 
         :param obj: An object that want to validate matching before return.
         :param conf_path: A config object.
-        :param included:
-        :param excluded:
+        :param included: An excluded list of data key that want to reject this
+            data if any key exist.
+        :param excluded: An included list of data key that want to filter from
+            data.
 
         :rtype: Iterator[tuple[str, DictData]]
         """
@@ -293,6 +310,9 @@ class SimLoad:
 
             for key, data in cls.filter_suffix(file).items():
 
+                if cls.is_ignore(file, conf_path):
+                    continue
+
                 if key in exclude:
                     continue
 
@@ -303,11 +323,26 @@ class SimLoad:
                         else data
                     )
 
+    @classmethod
+    def is_ignore(cls, file: Path, conf_path: Path) -> bool:
+        ignore_file: Path = conf_path / ".confignore"
+        ignore: list[str] = []
+        if ignore_file.exists():
+            ignore = ignore_file.read_text(encoding="utf-8").splitlines()
+
+        if any(
+            (file.match(f"**/{pattern}/*") or file.match(f"**/{pattern}*"))
+            for pattern in ignore
+        ):
+            return True
+        return False
+
     @classmethod
     def filter_suffix(cls, file: Path, name: str | None = None) -> DictData:
         if any(file.suffix.endswith(s) for s in (".yml", ".yaml")):
             values: DictData = YamlFlResolve(file).read()
             return values.get(name, {}) if name else values
+
         return {}
 
     @cached_property

ddeutil/workflow/job.py

@@ -5,7 +5,7 @@
 # ------------------------------------------------------------------------------
 """Job Model that use for keeping stages and node that running its stages.
 The job handle the lineage of stages and location of execution of stages that
-mean the job model able to define ``runs-on`` key that allow you to run this
+mean the job model able to define `runs-on` key that allow you to run this
 job.
 
     This module include Strategy Model that use on the job strategy field.
@@ -24,10 +24,10 @@ from enum import Enum
 from functools import lru_cache
 from textwrap import dedent
 from threading import Event
-from typing import Any, Optional, Union
+from typing import Annotated, Any, Literal, Optional, Union
 
 from ddeutil.core import freeze_args
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
@@ -39,7 +39,7 @@ from .exceptions import (
     UtilException,
 )
 from .result import Result, Status
-from .stage import Stage
+from .stages import Stage
 from .templates import has_template
 from .utils import (
     cross_product,
@@ -56,6 +56,11 @@ __all__: TupleStr = (
     "Strategy",
     "Job",
     "TriggerRules",
+    "RunsOn",
+    "RunsOnLocal",
+    "RunsOnSelfHosted",
+    "RunsOnDocker",
+    "RunsOnK8s",
     "make",
 )
 
@@ -216,13 +221,60 @@ class TriggerRules(str, Enum):
     none_skipped: str = "none_skipped"
 
 
-class RunsOn(str, Enum):
+class RunsOnType(str, Enum):
     """Runs-On enum object."""
 
-    local: str = "local"
-    docker: str = "docker"
-    self_hosted: str = "self_hosted"
-    k8s: str = "k8s"
+    LOCAL: str = "local"
+    DOCKER: str = "docker"
+    SELF_HOSTED: str = "self_hosted"
+    K8S: str = "k8s"
+
+
+class BaseRunsOn(BaseModel):
+    model_config = ConfigDict(use_enum_values=True)
+
+    type: Literal[RunsOnType.LOCAL]
+    args: DictData = Field(
+        default_factory=dict,
+        alias="with",
+    )
+
+
+class RunsOnLocal(BaseRunsOn):
+    """Runs-on local."""
+
+    type: Literal[RunsOnType.LOCAL] = Field(default=RunsOnType.LOCAL)
+
+
+class RunsOnSelfHosted(BaseRunsOn):
+    """Runs-on self-hosted."""
+
+    type: Literal[RunsOnType.SELF_HOSTED] = Field(
+        default=RunsOnType.SELF_HOSTED
+    )
+
+
+class RunsOnDocker(BaseRunsOn):
+    """Runs-on local Docker."""
+
+    type: Literal[RunsOnType.DOCKER] = Field(default=RunsOnType.DOCKER)
+
+
+class RunsOnK8s(BaseRunsOn):
+    """Runs-on Kubernetes."""
+
+    type: Literal[RunsOnType.K8S] = Field(default=RunsOnType.K8S)
+
+
+RunsOn = Annotated[
+    Union[
+        RunsOnLocal,
+        RunsOnSelfHosted,
+        RunsOnDocker,
+        RunsOnK8s,
+    ],
+    Field(discriminator="type"),
+]
 
 
 class Job(BaseModel):
@@ -263,9 +315,9 @@ class Job(BaseModel):
         default=None,
         description="A job description that can be string of markdown content.",
     )
-    runs_on: Optional[str] = Field(
-        default=None,
-        description="A target executor node for this job use to execution.",
+    runs_on: RunsOn = Field(
+        default_factory=RunsOnLocal,
+        description="A target node for this job to use for execution.",
         serialization_alias="runs-on",
     )
     stages: list[Stage] = Field(
@@ -359,7 +411,7 @@ class Job(BaseModel):
 
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
         """Set an outputs from execution process to the received context. The
-        result from execution will pass to value of ``strategies`` key.
+        result from execution will pass to value of `strategies` key.
 
             For example of setting output method, If you receive execute output
         and want to set on the `to` like;
@@ -400,10 +452,15 @@ class Job(BaseModel):
         # NOTE: If the job ID did not set, it will use index of jobs key
         #   instead.
         _id: str = self.id or str(len(to["jobs"]) + 1)
+
+        errors: DictData = (
+            {"errors": output.pop("errors", {})} if "errors" in output else {}
+        )
+
         to["jobs"][_id] = (
-            {"strategies": output}
+            {"strategies": output, **errors}
             if self.strategy.is_set()
-            else output.get(next(iter(output), "DUMMY"), {})
+            else {**output.get(next(iter(output), "DUMMY"), {}), **errors}
         )
         return to
 
@@ -412,7 +469,6 @@ class Job(BaseModel):
         strategy: DictData,
         params: DictData,
         *,
-        run_id: str | None = None,
         result: Result | None = None,
         event: Event | None = None,
     ) -> Result:
@@ -420,19 +476,18 @@ class Job(BaseModel):
         workflow execution to strategy matrix.
 
             This execution is the minimum level of execution of this job model.
-        It different with ``self.execute`` because this method run only one
+        It different with `self.execute` because this method run only one
         strategy and return with context of this strategy data.
 
             The result of this execution will return result with strategy ID
         that generated from the `gen_id` function with an input strategy value.
 
-        :raise JobException: If it has any error from ``StageException`` or
-            ``UtilException``.
+        :raise JobException: If it has any error from `StageException` or
+            `UtilException`.
 
         :param strategy: A strategy metrix value that use on this execution.
             This value will pass to the `matrix` key for templating.
         :param params: A dynamic parameters that will deepcopy to the context.
-        :param run_id: A job running ID for this strategy execution.
         :param result: (Result) A result object for keeping context and status
             data.
         :param event: An event manager that pass to the PoolThreadExecutor.
@@ -440,9 +495,7 @@ class Job(BaseModel):
         :rtype: Result
         """
         if result is None:  # pragma: no cov
-            result: Result = Result(
-                run_id=(run_id or gen_id(self.id or "", unique=True))
-            )
+            result: Result = Result(run_id=gen_id(self.id or "", unique=True))
 
         strategy_id: str = gen_id(strategy)
 
@@ -492,8 +545,11 @@ class Job(BaseModel):
                             # "stages": filter_func(context.pop("stages", {})),
                             #
                             "stages": context.pop("stages", {}),
-                            "error": JobException(error_msg),
-                            "error_message": error_msg,
+                            "errors": {
+                                "class": JobException(error_msg),
+                                "name": "JobException",
+                                "message": error_msg,
+                            },
                         },
                     },
                 )
@@ -506,7 +562,7 @@ class Job(BaseModel):
             #
             #       ... params |= stage.execute(params=params)
             #
-            #   This step will add the stage result to ``stages`` key in
+            #   This step will add the stage result to `stages` key in
             #   that stage id. It will have structure like;
             #
             #   {
@@ -516,10 +572,18 @@ class Job(BaseModel):
             #       "stages": { { "stage-id-1": ... }, ... }
             #   }
             #
+            # IMPORTANT:
+            #   This execution change all stage running IDs to the current job
+            #   running ID, but it still trac log to the same parent running ID
+            #   (with passing `run_id` and `parent_run_id` to the stage
+            #   execution arguments).
+            #
             try:
                 stage.set_outputs(
                     stage.handler_execute(
-                        params=context, run_id=result.run_id
+                        params=context,
+                        run_id=result.run_id,
+                        parent_run_id=result.parent_run_id,
                     ).context,
                     to=context,
                 )
@@ -527,17 +591,21 @@ class Job(BaseModel):
                 result.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
                 if config.job_raise_error:
                     raise JobException(
-                        f"Get stage execution error: {err.__class__.__name__}: "
+                        f"Stage execution error: {err.__class__.__name__}: "
                         f"{err}"
                     ) from None
+
                 return result.catch(
                     status=1,
                     context={
                         strategy_id: {
                             "matrix": strategy,
                             "stages": context.pop("stages", {}),
-                            "error": err,
-                            "error_message": f"{err.__class__.__name__}: {err}",
+                            "errors": {
+                                "class": err,
+                                "name": err.__class__.__name__,
+                                "message": f"{err.__class__.__name__}: {err}",
+                            },
                         },
                     },
                 )
@@ -560,25 +628,28 @@ class Job(BaseModel):
         params: DictData,
         *,
         run_id: str | None = None,
+        parent_run_id: str | None = None,
         result: Result | None = None,
     ) -> Result:
         """Job execution with passing dynamic parameters from the workflow
         execution. It will generate matrix values at the first step and run
-        multithread on this metrics to the ``stages`` field of this job.
+        multithread on this metrics to the `stages` field of this job.
 
         :param params: An input parameters that use on job execution.
         :param run_id: A job running ID for this execution.
+        :param parent_run_id: A parent workflow running ID for this release.
         :param result: (Result) A result object for keeping context and status
             data.
 
         :rtype: Result
         """
-
-        # NOTE: I use this condition because this method allow passing empty
-        #   params and I do not want to create new dict object.
         if result is None:  # pragma: no cov
-            run_id: str = run_id or gen_id(self.id or "", unique=True)
-            result: Result = Result(run_id=run_id)
+            result: Result = Result(
+                run_id=(run_id or gen_id(self.id or "", unique=True)),
+                parent_run_id=parent_run_id,
+            )
+        elif parent_run_id:  # pragma: no cov
+            result.set_parent_run_id(parent_run_id)
 
         # NOTE: Normal Job execution without parallel strategy matrix. It uses
         #   for-loop to control strategy execution sequentially.
@@ -614,110 +685,50 @@ class Job(BaseModel):
                 for strategy in self.strategy.make()
             ]
 
-            return (
-                self.__catch_fail_fast(event, futures=futures, result=result)
-                if self.strategy.fail_fast
-                else self.__catch_all_completed(futures=futures, result=result)
-            )
-
-    @staticmethod
-    def __catch_fail_fast(
-        event: Event,
-        futures: list[Future],
-        result: Result,
-        *,
-        timeout: int = 1800,
-    ) -> Result:
-        """Job parallel pool futures catching with fail-fast mode. That will
-        stop and set event on all not done futures if it receives the first
-        exception from all running futures.
-
-        :param event: An event manager instance that able to set stopper on the
-            observing multithreading.
-        :param futures: A list of futures.
-        :param result: (Result) A result object for keeping context and status
-            data.
-        :param timeout: A timeout to waiting all futures complete.
-
-        :rtype: Result
-        """
-        context: DictData = {}
-        status: Status = Status.SUCCESS
+            context: DictData = {}
+            status: Status = Status.SUCCESS
+            fail_fast_flag: bool = self.strategy.fail_fast
 
-        # NOTE: Get results from a collection of tasks with a timeout that has
-        #   the first exception.
-        done, not_done = wait(
-            futures, timeout=timeout, return_when=FIRST_EXCEPTION
-        )
-        nd: str = (
-            f", the strategies do not run is {not_done}" if not_done else ""
-        )
-        result.trace.debug(f"[JOB]: Strategy is set Fail Fast{nd}")
-
-        # NOTE:
-        #       Stop all running tasks with setting the event manager and cancel
-        #   any scheduled tasks.
-        #
-        if len(done) != len(futures):
-            event.set()
-            for future in not_done:
-                future.cancel()
-
-        future: Future
-        for future in done:
-
-            # NOTE: Handle the first exception from feature
-            if err := future.exception():
-                status: Status = Status.FAILED
-                result.trace.error(
-                    f"[JOB]: Fail-fast catching:\n\t{future.exception()}"
+            if fail_fast_flag:
+                # NOTE: Get results from a collection of tasks with a timeout
+                #   that has the first exception.
+                done, not_done = wait(
+                    futures, timeout=1800, return_when=FIRST_EXCEPTION
                 )
-                context.update(
-                    {
-                        "error": err,
-                        "error_message": f"{err.__class__.__name__}: {err}",
-                    },
-                )
-                continue
-
-            # NOTE: Update the result context to main job context.
-            future.result()
-
-        return result.catch(status=status, context=context)
-
-    @staticmethod
-    def __catch_all_completed(
-        futures: list[Future],
-        result: Result,
-        *,
-        timeout: int = 1800,
-    ) -> Result:
-        """Job parallel pool futures catching with all-completed mode.
-
-        :param futures: A list of futures.
-        :param result: (Result) A result object for keeping context and status
-            data.
-        :param timeout: A timeout to waiting all futures complete.
-
-        :rtype: Result
-        """
-        context: DictData = {}
-        status: Status = Status.SUCCESS
-
-        for future in as_completed(futures, timeout=timeout):
-            try:
-                future.result()
-            except JobException as err:
-                status = Status.FAILED
-                result.trace.error(
-                    f"[JOB]: All-completed catching:\n\t"
-                    f"{err.__class__.__name__}:\n\t{err}"
-                )
-                context.update(
-                    {
-                        "error": err,
-                        "error_message": f"{err.__class__.__name__}: {err}",
-                    },
+                nd: str = (
+                    f", the strategies do not run is {not_done}"
+                    if not_done
+                    else ""
                 )
+                result.trace.debug(f"[JOB]: Strategy is set Fail Fast{nd}")
+
+                # NOTE: Stop all running tasks with setting the event manager
+                #   and cancel any scheduled tasks.
+                if len(done) != len(futures):
+                    event.set()
+                    for future in not_done:
+                        future.cancel()
+            else:
+                done = as_completed(futures, timeout=1800)
+
+            for future in done:
+                try:
+                    future.result()
+                except JobException as err:
+                    status = Status.FAILED
+                    ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
+                    result.trace.error(
+                        f"[JOB]: {ls} Catch:\n\t{err.__class__.__name__}:"
+                        f"\n\t{err}"
+                    )
+                    context.update(
+                        {
+                            "errors": {
+                                "class": err,
+                                "name": err.__class__.__name__,
+                                "message": f"{err.__class__.__name__}: {err}",
+                            },
+                        },
+                    )
 
         return result.catch(status=status, context=context)