ddeutil-workflow 0.0.37__py3-none-any.whl → 0.0.38__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.37"
+__version__: str = "0.0.38"

ddeutil/workflow/__init__.py

@@ -39,6 +39,8 @@ from .job import (
     Job,
     RunsOn,
     Strategy,
+    local_execute,
+    local_execute_strategy,
 )
 from .logs import (
     TraceData,
@@ -69,6 +71,8 @@ from .stages import (
     BashStage,
     CallStage,
     EmptyStage,
+    ForEachStage,
+    ParallelStage,
     PyStage,
     Stage,
     TriggerStage,
@@ -89,7 +93,6 @@ from .templates import (
 from .utils import (
     batch,
     cross_product,
-    dash2underscore,
     delay,
     filter_func,
     gen_id,

ddeutil/workflow/api/routes/job.py

@@ -45,6 +45,7 @@ async def job_execute(
         run_id=result.run_id,
         parent_run_id=result.parent_run_id,
     )
+    context: DictData = {}
     try:
         job.set_outputs(
             job.execute(
@@ -52,7 +53,7 @@ async def job_execute(
                 run_id=rs.run_id,
                 parent_run_id=rs.parent_run_id,
             ).context,
-            to=params,
+            to=context,
         )
     except JobException as err:
         rs.trace.error(f"[WORKFLOW]: {err.__class__.__name__}: {err}")
@@ -70,4 +71,5 @@ async def job_execute(
             exclude_defaults=True,
         ),
         "params": params,
+        "context": context,
     }

ddeutil/workflow/api/routes/logs.py

@@ -6,7 +6,7 @@
 """This route include audit and trace log paths."""
 from __future__ import annotations
 
-from fastapi import APIRouter
+from fastapi import APIRouter, Path, Query
 from fastapi import status as st
 from fastapi.responses import UJSONResponse
 
@@ -27,12 +27,17 @@ log_route = APIRouter(
     summary="Read all trace logs.",
     tags=["trace"],
 )
-async def get_traces():
+async def get_traces(
+    offset: int = Query(default=0, gt=0),
+    limit: int = Query(default=100, gt=0),
+):
     """Return all trace logs from the current trace log path that config with
     `WORKFLOW_LOG_PATH` environment variable name.
     """
     return {
-        "message": "Getting trace logs",
+        "message": (
+            f"Getting trace logs with offset: {offset} and limit: {limit}"
+        ),
         "traces": [
             trace.model_dump(
                 by_alias=True,
@@ -117,7 +122,10 @@ async def get_audit_with_workflow(workflow: str):
     summary="Read all audit logs with specific workflow name and release date.",
     tags=["audit"],
 )
-async def get_audit_with_workflow_release(workflow: str, release: str):
+async def get_audit_with_workflow_release(
+    workflow: str = Path(...),
+    release: str = Path(...),
+):
     """Return all audit logs with specific workflow name and release date from
     the current audit log path that config with `WORKFLOW_AUDIT_PATH`
     environment variable name.

ddeutil/workflow/caller.py

@@ -26,6 +26,7 @@ T = TypeVar("T")
 P = ParamSpec("P")
 
 logger = logging.getLogger("ddeutil.workflow")
+logging.getLogger("asyncio").setLevel(logging.INFO)
 
 
 class TagFunc(Protocol):
@@ -60,10 +61,13 @@ def tag(
 
         @wraps(func)
         def wrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
-            # NOTE: Able to do anything before calling the call function.
             return func(*args, **kwargs)
 
-        return wrapped
+        @wraps(func)
+        async def awrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
+            return await func(*args, **kwargs)
+
+        return awrapped if inspect.iscoroutinefunction(func) else wrapped
 
     return func_internal
 

ddeutil/workflow/context.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from typing import Optional, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from .__types import DictData
+
+
+class ErrorContext(BaseModel):  # pragma: no cov
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    obj: Exception = Field(alias="class")
+    name: str = Field(description="A name of exception class.")
+    message: str = Field(description="A exception message.")
+
+
+class OutputContext(BaseModel):  # pragma: no cov
+    outputs: DictData = Field(default_factory=dict)
+    errors: Optional[ErrorContext] = Field(default=None)
+
+    def is_exception(self) -> bool:
+        return self.errors is not None
+
+
+class StageContext(BaseModel):  # pragma: no cov
+    stages: dict[str, OutputContext]
+    errors: Optional[ErrorContext] = Field(default=None)
+
+    def is_exception(self) -> bool:
+        return self.errors is not None
+
+
+class MatrixContext(StageContext):  # pragma: no cov
+    matrix: DictData = Field(default_factory=dict)
+
+
+MatrixStageContext = dict[
+    str, Union[MatrixContext, StageContext]
+]  # pragma: no cov
+
+
+class StrategyContext(BaseModel):  # pragma: no cov
+    strategies: MatrixStageContext
+    errors: Optional[ErrorContext] = Field(default=None)
+
+    def is_exception(self) -> bool:
+        return self.errors is not None
+
+
+StrategyMatrixContext = Union[
+    StrategyContext, MatrixStageContext
+]  # pragma: no cov
+
+
+class JobContext(BaseModel):  # pragma: no cov
+    params: DictData = Field(description="A parameterize value")
+    jobs: dict[str, StrategyMatrixContext]
+    errors: Optional[ErrorContext] = Field(default=None)

ddeutil/workflow/exceptions.py

@@ -9,8 +9,21 @@ annotate for handle error only.
 """
 from __future__ import annotations
 
+from typing import Any
 
-class BaseWorkflowException(Exception): ...
+
+def to_dict(exception: Exception) -> dict[str, Any]:  # pragma: no cov
+    return {
+        "class": exception,
+        "name": exception.__class__.__name__,
+        "message": str(exception),
+    }
+
+
+class BaseWorkflowException(Exception):
+
+    def to_dict(self) -> dict[str, Any]:
+        return to_dict(self)
 
 
 class UtilException(BaseWorkflowException): ...

ddeutil/workflow/job.py

@@ -27,7 +27,7 @@ from threading import Event
 from typing import Annotated, Any, Literal, Optional, Union
 
 from ddeutil.core import freeze_args
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Discriminator, Field, Tag
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
@@ -43,7 +43,6 @@ from .stages import Stage
 from .templates import has_template
 from .utils import (
     cross_product,
-    dash2underscore,
     filter_func,
     gen_id,
 )
@@ -155,7 +154,7 @@ class Strategy(BaseModel):
 
     fail_fast: bool = Field(
         default=False,
-        serialization_alias="fail-fast",
+        alias="fail-fast",
     )
     max_parallel: int = Field(
         default=1,
@@ -164,7 +163,7 @@ class Strategy(BaseModel):
             "The maximum number of executor thread pool that want to run "
             "parallel"
         ),
-        serialization_alias="max-parallel",
+        alias="max-parallel",
     )
     matrix: Matrix = Field(
         default_factory=dict,
@@ -181,18 +180,6 @@ class Strategy(BaseModel):
         description="A list of exclude matrix that want to filter-out.",
     )
 
-    @model_validator(mode="before")
-    def __prepare_keys(cls, values: DictData) -> DictData:
-        """Rename key that use dash to underscore because Python does not
-        support this character exist in any variable name.
-
-        :param values: A parsing values to these models
-        :rtype: DictData
-        """
-        dash2underscore("max-parallel", values)
-        dash2underscore("fail-fast", values)
-        return values
-
     def is_set(self) -> bool:
         """Return True if this strategy was set from yaml template.
 
@@ -268,13 +255,17 @@ class RunsOnK8s(BaseRunsOn):  # pragma: no cov
     type: Literal[RunsOnType.K8S] = Field(default=RunsOnType.K8S)
 
 
+def get_discriminator_runs_on(model: dict[str, Any]) -> str:
+    return model.get("type", "local")
+
+
 RunsOn = Annotated[
     Union[
-        RunsOnLocal,
-        RunsOnSelfHosted,
-        RunsOnK8s,
+        Annotated[RunsOnK8s, Tag(RunsOnType.K8S)],
+        Annotated[RunsOnSelfHosted, Tag(RunsOnType.SELF_HOSTED)],
+        Annotated[RunsOnLocal, Tag(RunsOnType.LOCAL)],
     ],
-    Field(discriminator="type"),
+    Discriminator(get_discriminator_runs_on),
 ]
 
 
@@ -319,7 +310,7 @@ class Job(BaseModel):
     runs_on: RunsOn = Field(
         default_factory=RunsOnLocal,
         description="A target node for this job to use for execution.",
-        serialization_alias="runs-on",
+        alias="runs-on",
     )
     stages: list[Stage] = Field(
         default_factory=list,
@@ -328,7 +319,7 @@ class Job(BaseModel):
     trigger_rule: TriggerRules = Field(
         default=TriggerRules.all_success,
         description="A trigger rule of tracking needed jobs.",
-        serialization_alias="trigger-rule",
+        alias="trigger-rule",
     )
     needs: list[str] = Field(
         default_factory=list,
@@ -339,18 +330,6 @@ class Job(BaseModel):
         description="A strategy matrix that want to generate.",
     )
 
-    @model_validator(mode="before")
-    def __prepare_keys__(cls, values: DictData) -> DictData:
-        """Rename key that use dash to underscore because Python does not
-        support this character exist in any variable name.
-
-        :param values: A passing value that coming for initialize this object.
-        :rtype: DictData
-        """
-        dash2underscore("runs-on", values)
-        dash2underscore("trigger-rule", values)
-        return values
-
     @field_validator("desc", mode="after")
     def ___prepare_desc__(cls, value: str) -> str:
         """Prepare description string that was created on a template.
@@ -441,15 +420,14 @@ class Job(BaseModel):
 
         :rtype: DictData
         """
+        if "jobs" not in to:
+            to["jobs"] = {}
+
         if self.id is None and not config.job_default_id:
             raise JobException(
                 "This job do not set the ID before setting execution output."
             )
 
-        # NOTE: Create jobs key to receive an output from the job execution.
-        if "jobs" not in to:
-            to["jobs"] = {}
-
         # NOTE: If the job ID did not set, it will use index of jobs key
         #   instead.
         _id: str = self.id or str(len(to["jobs"]) + 1)
@@ -458,11 +436,12 @@ class Job(BaseModel):
             {"errors": output.pop("errors", {})} if "errors" in output else {}
         )
 
-        to["jobs"][_id] = (
-            {"strategies": output, **errors}
-            if self.strategy.is_set()
-            else {**output.get(next(iter(output), "DUMMY"), {}), **errors}
-        )
+        if self.strategy.is_set():
+            to["jobs"][_id] = {"strategies": output, **errors}
+        else:
+            _output = output.get(next(iter(output), "FIRST"), {})
+            _output.pop("matrix", {})
+            to["jobs"][_id] = {**_output, **errors}
         return to
 
     def execute(
@@ -472,6 +451,7 @@ class Job(BaseModel):
         run_id: str | None = None,
         parent_run_id: str | None = None,
         result: Result | None = None,
+        event: Event | None = None,
     ) -> Result:
         """Job execution with passing dynamic parameters from the workflow
         execution. It will generate matrix values at the first step and run
@@ -482,23 +462,35 @@ class Job(BaseModel):
         :param parent_run_id: A parent workflow running ID for this release.
         :param result: (Result) A result object for keeping context and status
             data.
+        :param event: (Event) An event manager that pass to the
+            PoolThreadExecutor.
 
         :rtype: Result
         """
-        if result is None:  # pragma: no cov
-            result: Result = Result(
-                run_id=(run_id or gen_id(self.id or "", unique=True)),
-                parent_run_id=parent_run_id,
-            )
-        elif parent_run_id:  # pragma: no cov
-            result.set_parent_run_id(parent_run_id)
+        result: Result = Result.construct_with_rs_or_id(
+            result,
+            run_id=run_id,
+            parent_run_id=parent_run_id,
+            id_logic=(self.id or "not-set"),
+        )
 
         if self.runs_on.type == RunsOnType.LOCAL:
             return local_execute(
                 job=self,
                 params=params,
                 result=result,
+                event=event,
             )
+        elif self.runs_on.type == RunsOnType.SELF_HOSTED:  # pragma: no cov
+            pass
+        elif self.runs_on.type == RunsOnType.K8S:  # pragma: no cov
+            pass
+
+        # pragma: no cov
+        result.trace.error(
+            f"[JOB]: Job executor does not support for runs-on type: "
+            f"{self.runs_on.type} yet"
+        )
         raise NotImplementedError(
             f"The job runs-on other type: {self.runs_on.type} does not "
             f"support yet."
@@ -512,6 +504,7 @@ def local_execute_strategy(
     *,
     result: Result | None = None,
     event: Event | None = None,
+    raise_error: bool = False,
 ) -> Result:
     """Local job strategy execution with passing dynamic parameters from the
     workflow execution to strategy matrix.
@@ -533,11 +526,12 @@ def local_execute_strategy(
     :param result: (Result) A result object for keeping context and status
         data.
     :param event: (Event) An event manager that pass to the PoolThreadExecutor.
+    :param raise_error: (bool) A flag that all this method raise error
 
     :rtype: Result
     """
-    if result is None:  # pragma: no cov
-        result: Result = Result(run_id=gen_id(job.id or "", unique=True))
+    if result is None:
+        result: Result = Result(run_id=gen_id(job.id or "not-set", unique=True))
 
     strategy_id: str = gen_id(strategy)
 
@@ -556,62 +550,43 @@ def local_execute_strategy(
     context: DictData = copy.deepcopy(params)
     context.update({"matrix": strategy, "stages": {}})
 
+    if strategy:
+        result.trace.info(f"[JOB]: Execute Strategy ID: {strategy_id}")
+        result.trace.info(f"[JOB]: ... Matrix: {strategy_id}")
+
     # IMPORTANT: The stage execution only run sequentially one-by-one.
     for stage in job.stages:
 
         if stage.is_skipped(params=context):
-            result.trace.info(f"[JOB]: Skip stage: {stage.iden!r}")
+            result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
             continue
 
-        result.trace.info(f"[JOB]: Execute stage: {stage.iden!r}")
-
-        # NOTE: Logging a matrix that pass on this stage execution.
-        if strategy:
-            result.trace.info(f"[JOB]: ... Matrix: {strategy}")
-
-        # NOTE: Force stop this execution if event was set from main
-        #   execution.
         if event and event.is_set():
             error_msg: str = (
                 "Job strategy was canceled from event that had set before "
                 "strategy execution."
             )
             return result.catch(
-                status=1,
+                status=Status.FAILED,
                 context={
                     strategy_id: {
                         "matrix": strategy,
-                        # NOTE: If job strategy executor use multithreading,
-                        #   it will not filter function object from context.
-                        # ---
-                        # "stages": filter_func(context.pop("stages", {})),
-                        #
                         "stages": context.pop("stages", {}),
-                        "errors": {
-                            "class": JobException(error_msg),
-                            "name": "JobException",
-                            "message": error_msg,
-                        },
+                        "errors": JobException(error_msg).to_dict(),
                     },
                 },
             )
 
         # PARAGRAPH:
         #
-        #       I do not use below syntax because `params` dict be the
-        #   reference memory pointer, and it was changed when I action
-        #   anything like update or re-construct this.
-        #
-        #       ... params |= stage.execute(params=params)
-        #
-        #   This step will add the stage result to `stages` key in
-        #   that stage id. It will have structure like;
+        #       This step will add the stage result to `stages` key in that
+        #   stage id. It will have structure like;
         #
         #   {
         #       "params": { ... },
         #       "jobs": { ... },
         #       "matrix": { ... },
-        #       "stages": { { "stage-id-1": ... }, ... }
+        #       "stages": { { "stage-id-01": { "outputs": { ... } } }, ... }
         #   }
         #
         # IMPORTANT:
@@ -631,23 +606,19 @@ def local_execute_strategy(
             )
         except (StageException, UtilException) as err:
             result.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
-            if config.job_raise_error:
+            if raise_error or config.job_raise_error:
                 raise JobException(
                     f"Stage execution error: {err.__class__.__name__}: "
                     f"{err}"
                 ) from None
 
             return result.catch(
-                status=1,
+                status=Status.FAILED,
                 context={
                     strategy_id: {
                         "matrix": strategy,
                         "stages": context.pop("stages", {}),
-                        "errors": {
-                            "class": err,
-                            "name": err.__class__.__name__,
-                            "message": f"{err.__class__.__name__}: {err}",
-                        },
+                        "errors": err.to_dict(),
                     },
                 },
             )
@@ -673,44 +644,61 @@ def local_execute(
     run_id: str | None = None,
     parent_run_id: str | None = None,
     result: Result | None = None,
+    event: Event | None = None,
+    raise_error: bool = False,
 ) -> Result:
     """Local job execution with passing dynamic parameters from the workflow
     execution. It will generate matrix values at the first step and run
     multithread on this metrics to the `stages` field of this job.
 
-    :param job: A job model that want to execute.
-    :param params: An input parameters that use on job execution.
-    :param run_id: A job running ID for this execution.
-    :param parent_run_id: A parent workflow running ID for this release.
+        This method does not raise any JobException if it runs with
+    multi-threading strategy.
+
+    :param job: (Job) A job model that want to execute.
+    :param params: (DictData) An input parameters that use on job execution.
+    :param run_id: (str) A job running ID for this execution.
+    :param parent_run_id: (str) A parent workflow running ID for this release.
     :param result: (Result) A result object for keeping context and status
         data.
+    :param event: (Event) An event manager that pass to the PoolThreadExecutor.
+    :param raise_error: (bool) A flag that all this method raise error to the
+        strategy execution.
 
     :rtype: Result
     """
-    if result is None:  # pragma: no cov
-        result: Result = Result(
-            run_id=(run_id or gen_id(job.id or "", unique=True)),
-            parent_run_id=parent_run_id,
-        )
-    elif parent_run_id:  # pragma: no cov
-        result.set_parent_run_id(parent_run_id)
+    result: Result = Result.construct_with_rs_or_id(
+        result,
+        run_id=run_id,
+        parent_run_id=parent_run_id,
+        id_logic=(job.id or "not-set"),
+    )
+    event: Event = Event() if event is None else event
 
     # NOTE: Normal Job execution without parallel strategy matrix. It uses
     #   for-loop to control strategy execution sequentially.
     if (not job.strategy.is_set()) or job.strategy.max_parallel == 1:
 
         for strategy in job.strategy.make():
-            result: Result = local_execute_strategy(
+
+            # TODO: stop and raise error if the event was set.
+            local_execute_strategy(
                 job=job,
                 strategy=strategy,
                 params=params,
                 result=result,
+                event=event,
+                raise_error=raise_error,
             )
 
         return result.catch(status=Status.SUCCESS)
 
-    # NOTE: Create event for cancel executor by trigger stop running event.
-    event: Event = Event()
+    fail_fast_flag: bool = job.strategy.fail_fast
+    ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
+
+    result.trace.info(
+        f"[JOB]: Start multithreading: {job.strategy.max_parallel} threads "
+        f"with {ls} mode."
+    )
 
     # IMPORTANT: Start running strategy execution by multithreading because
     #   it will run by strategy values without waiting previous execution.
@@ -727,52 +715,43 @@ def local_execute(
                 params=params,
                 result=result,
                 event=event,
+                raise_error=raise_error,
             )
             for strategy in job.strategy.make()
         ]
 
         context: DictData = {}
         status: Status = Status.SUCCESS
-        fail_fast_flag: bool = job.strategy.fail_fast
 
         if not fail_fast_flag:
             done = as_completed(futures, timeout=1800)
         else:
-            # NOTE: Get results from a collection of tasks with a timeout
-            #   that has the first exception.
             done, not_done = wait(
                 futures, timeout=1800, return_when=FIRST_EXCEPTION
             )
-            nd: str = (
-                f", the strategies do not run is {not_done}" if not_done else ""
-            )
-            result.trace.debug(f"[JOB]: Strategy is set Fail Fast{nd}")
 
-            # NOTE: Stop all running tasks with setting the event manager
-            #   and cancel any scheduled tasks.
             if len(done) != len(futures):
+                result.trace.warning(
+                    "[JOB]: Set the event for stop running stage."
+                )
                 event.set()
                 for future in not_done:
                     future.cancel()
 
+            nd: str = (
+                f", the strategies do not run is {not_done}" if not_done else ""
+            )
+            result.trace.debug(f"[JOB]: Strategy is set Fail Fast{nd}")
+
         for future in done:
             try:
                 future.result()
             except JobException as err:
                 status = Status.FAILED
-                ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
                 result.trace.error(
                     f"[JOB]: {ls} Catch:\n\t{err.__class__.__name__}:"
                     f"\n\t{err}"
                 )
-                context.update(
-                    {
-                        "errors": {
-                            "class": err,
-                            "name": err.__class__.__name__,
-                            "message": f"{err.__class__.__name__}: {err}",
-                        },
-                    },
-                )
+                context.update({"errors": err.to_dict()})
 
     return result.catch(status=status, context=context)

ddeutil/workflow/logs.py

@@ -234,7 +234,7 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
         return f"({self.cut_id}) {message}"
 
     def writer(self, message: str, is_err: bool = False) -> None:
-        """ "Write a trace message after making to target file and write metadata
+        """Write a trace message after making to target file and write metadata
         in the same path of standard files.
 
             The path of logging data will store by format:
@@ -279,6 +279,11 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
                 + "\n"
             )
 
+    async def awriter(
+        self, message: str, is_err: bool = False
+    ):  # pragma: no cov
+        """TODO: Use `aiofiles` for make writer method support async."""
+
 
 class SQLiteTraceLog(BaseTraceLog):  # pragma: no cov
     """Trace Log object that write trace log to the SQLite database file."""