ddeutil-workflow 0.0.81__py3-none-any.whl → 0.0.83__py3-none-any.whl

ddeutil/workflow/job.py

@@ -48,7 +48,7 @@ from enum import Enum
 from functools import lru_cache
 from textwrap import dedent
 from threading import Event
-from typing import Annotated, Any, Optional, Union
+from typing import Annotated, Any, Literal, Optional, Union
 
 from ddeutil.core import freeze_args
 from pydantic import BaseModel, Discriminator, Field, SecretStr, Tag
@@ -72,8 +72,8 @@ from .result import (
 )
 from .reusables import has_template, param2template
 from .stages import Stage
-from .traces import TraceManager, get_trace
-from .utils import cross_product, filter_func, gen_id
+from .traces import Trace, get_trace
+from .utils import cross_product, extract_id, filter_func, gen_id
 
 MatrixFilter = list[dict[str, Union[str, int]]]
 
@@ -187,10 +187,8 @@ class Strategy(BaseModel):
         ),
         alias="fail-fast",
     )
-    max_parallel: int = Field(
+    max_parallel: Union[int, str] = Field(
         default=1,
-        gt=0,
-        lt=10,
         description=(
             "The maximum number of executor thread pool that want to run "
             "parallel. This value should gather than 0 and less than 10."
@@ -427,9 +425,9 @@ class OnGCPBatch(BaseRunsOn):  # pragma: no cov
     args: GCPBatchArgs = Field(alias="with")
 
 
-def get_discriminator_runs_on(model: dict[str, Any]) -> RunsOn:
+def get_discriminator_runs_on(data: dict[str, Any]) -> RunsOn:
     """Get discriminator of the RunsOn models."""
-    t: str = model.get("type")
+    t: str = data.get("type")
     return RunsOn(t) if t else LOCAL
 
 
@@ -538,13 +536,28 @@ class Job(BaseModel):
         description="An extra override config values.",
     )
 
+    @field_validator(
+        "runs_on",
+        mode="before",
+        json_schema_input_type=Union[RunsOnModel, Literal["local"]],
+    )
+    def __prepare_runs_on(cls, data: Any) -> Any:
+        """Prepare runs on value that was passed with string type."""
+        if isinstance(data, str):
+            if data != "local":
+                raise ValueError(
+                    "runs-on that pass with str type should be `local` only"
+                )
+            return {"type": data}
+        return data
+
     @field_validator("desc", mode="after")
-    def ___prepare_desc__(cls, value: str) -> str:
+    def ___prepare_desc__(cls, data: str) -> str:
         """Prepare description string that was created on a template.
 
         :rtype: str
         """
-        return dedent(value.lstrip("\n"))
+        return dedent(data.lstrip("\n"))
 
     @field_validator("stages", mode="after")
     def __validate_stage_id__(cls, value: list[Stage]) -> list[Stage]:
@@ -877,9 +890,10 @@ class Job(BaseModel):
             Result: Return Result object that create from execution context.
         """
         ts: float = time.monotonic()
-        parent_run_id: str = run_id
-        run_id: str = gen_id((self.id or "EMPTY"), unique=True)
-        trace: TraceManager = get_trace(
+        parent_run_id, run_id = extract_id(
+            (self.id or "EMPTY"), run_id=run_id, extras=self.extras
+        )
+        trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
         trace.info(
@@ -1016,7 +1030,7 @@ def local_execute_strategy(
 
     :rtype: tuple[Status, DictData]
     """
-    trace: TraceManager = get_trace(
+    trace: Trace = get_trace(
         run_id, parent_run_id=parent_run_id, extras=job.extras
     )
     if strategy:
@@ -1152,7 +1166,7 @@ def local_execute(
     ts: float = time.monotonic()
     parent_run_id: StrOrNone = run_id
     run_id: str = gen_id((job.id or "EMPTY"), unique=True)
-    trace: TraceManager = get_trace(
+    trace: Trace = get_trace(
         run_id, parent_run_id=parent_run_id, extras=job.extras
     )
     context: DictData = {"status": WAIT}
@@ -1174,11 +1188,52 @@ def local_execute(
 
     event: Event = event or Event()
     ls: str = "Fail-Fast" if job.strategy.fail_fast else "All-Completed"
-    workers: int = job.strategy.max_parallel
+    workers: Union[int, str] = job.strategy.max_parallel
+    if isinstance(workers, str):
+        try:
+            workers: int = int(
+                param2template(workers, params=params, extras=job.extras)
+            )
+        except Exception as err:
+            trace.exception(
+                "[JOB]: Got the error on call param2template to "
+                f"max-parallel value: {workers}"
+            )
+            return Result(
+                run_id=run_id,
+                parent_run_id=parent_run_id,
+                status=FAILED,
+                context=catch(
+                    context,
+                    status=FAILED,
+                    updated={"errors": to_dict(err)},
+                ),
+                info={"execution_time": time.monotonic() - ts},
+                extras=job.extras,
+            )
+    if workers >= 10:
+        err_msg: str = (
+            f"The max-parallel value should not more than 10, the current value "
+            f"was set: {workers}."
+        )
+        trace.error(f"[JOB]: {err_msg}")
+        return Result(
+            run_id=run_id,
+            parent_run_id=parent_run_id,
+            status=FAILED,
+            context=catch(
+                context,
+                status=FAILED,
+                updated={"errors": JobError(err_msg).to_dict()},
+            ),
+            info={"execution_time": time.monotonic() - ts},
+            extras=job.extras,
+        )
+
     strategies: list[DictStr] = job.strategy.make()
     len_strategy: int = len(strategies)
     trace.info(
-        f"[JOB]: ... Mode {ls}: {job.id!r} with {workers} "
+        f"[JOB]: Mode {ls}: {job.id!r} with {workers} "
         f"worker{'s' if workers > 1 else ''}."
     )
 
@@ -1218,7 +1273,6 @@ def local_execute(
 
         errors: DictData = {}
         statuses: list[Status] = [WAIT] * len_strategy
-        fail_fast: bool = False
 
         if not job.strategy.fail_fast:
             done: Iterator[Future] = as_completed(futures)
@@ -1243,7 +1297,6 @@ def local_execute(
                 )
                 trace.debug(f"[JOB]: ... Job was set Fail-Fast{nd}")
                 done: Iterator[Future] = as_completed(futures)
-                fail_fast: bool = True
 
         for i, future in enumerate(done, start=0):
             try:
@@ -1258,19 +1311,10 @@ def local_execute(
                 pass
 
     status: Status = validate_statuses(statuses)
-
-    # NOTE: Prepare status because it does not cancel from parent event but
-    #   cancel from failed item execution.
-    if fail_fast and status == CANCEL:
-        status = FAILED
-
-    return Result(
-        run_id=run_id,
-        parent_run_id=parent_run_id,
+    return Result.from_trace(trace).catch(
         status=status,
         context=catch(context, status=status, updated=errors),
         info={"execution_time": time.monotonic() - ts},
-        extras=job.extras,
     )
 
 
@@ -1295,7 +1339,7 @@ def self_hosted_execute(
     """
     parent_run_id: StrOrNone = run_id
     run_id: str = gen_id((job.id or "EMPTY"), unique=True)
-    trace: TraceManager = get_trace(
+    trace: Trace = get_trace(
         run_id, parent_run_id=parent_run_id, extras=job.extras
     )
     context: DictData = {"status": WAIT}
@@ -1378,7 +1422,7 @@ def docker_execution(
     """
     parent_run_id: StrOrNone = run_id
     run_id: str = gen_id((job.id or "EMPTY"), unique=True)
-    trace: TraceManager = get_trace(
+    trace: Trace = get_trace(
         run_id, parent_run_id=parent_run_id, extras=job.extras
     )
     context: DictData = {"status": WAIT}

ddeutil/workflow/result.py

@@ -8,25 +8,17 @@
 This module provides the core result and status management functionality for
 workflow execution tracking. It includes the Status enumeration for execution
 states and the Result dataclass for context transfer between workflow components.
-
-Classes:
-    Status: Enumeration for execution status tracking
-    Result: Dataclass for execution context and result management
-
-Functions:
-    validate_statuses: Determine final status from multiple status values
-    get_status_from_error: Convert exception types to appropriate status
 """
 from __future__ import annotations
 
 from dataclasses import field
 from enum import Enum
-from typing import Optional, Union
+from typing import Any, Optional, TypedDict, Union
 
 from pydantic import ConfigDict
 from pydantic.dataclasses import dataclass
 from pydantic.functional_validators import model_validator
-from typing_extensions import Self
+from typing_extensions import NotRequired, Self
 
 from . import (
     JobCancelError,
@@ -34,13 +26,16 @@ from . import (
     JobSkipError,
     StageCancelError,
     StageError,
+    StageNestedCancelError,
+    StageNestedError,
+    StageNestedSkipError,
     StageSkipError,
     WorkflowCancelError,
     WorkflowError,
 )
 from .__types import DictData
-from .audits import TraceManager, get_trace
-from .errors import ResultError
+from .audits import Trace, get_trace
+from .errors import ErrorData, ResultError
 from .utils import default_gen_id
 
 
@@ -123,10 +118,10 @@ def validate_statuses(statuses: list[Status]) -> Status:
         >>> validate_statuses([SUCCESS, SUCCESS, SUCCESS])
         >>> # Returns: SUCCESS
     """
-    if any(s == CANCEL for s in statuses):
-        return CANCEL
-    elif any(s == FAILED for s in statuses):
+    if any(s == FAILED for s in statuses):
         return FAILED
+    elif any(s == CANCEL for s in statuses):
+        return CANCEL
     elif any(s == WAIT for s in statuses):
         return WAIT
     for status in (SUCCESS, SKIP):
@@ -140,6 +135,9 @@ def get_status_from_error(
         StageError,
         StageCancelError,
         StageSkipError,
+        StageNestedCancelError,
+        StageNestedError,
+        StageNestedSkipError,
         JobError,
         JobCancelError,
         JobSkipError,
@@ -157,10 +155,16 @@ def get_status_from_error(
     Returns:
         Status: The status from the specific exception class.
     """
-    if isinstance(error, (StageSkipError, JobSkipError)):
+    if isinstance(error, (StageNestedSkipError, StageSkipError, JobSkipError)):
         return SKIP
     elif isinstance(
-        error, (StageCancelError, JobCancelError, WorkflowCancelError)
+        error,
+        (
+            StageNestedCancelError,
+            StageCancelError,
+            JobCancelError,
+            WorkflowCancelError,
+        ),
     ):
         return CANCEL
     return FAILED
@@ -188,9 +192,7 @@ class Result:
     info: DictData = field(default_factory=dict)
     run_id: str = field(default_factory=default_gen_id)
     parent_run_id: Optional[str] = field(default=None)
-    trace: Optional[TraceManager] = field(
-        default=None, compare=False, repr=False
-    )
+    trace: Optional[Trace] = field(default=None, compare=False, repr=False)
 
     @model_validator(mode="after")
     def __prepare_trace(self) -> Self:
@@ -199,7 +201,7 @@ class Result:
         :rtype: Self
         """
         if self.trace is None:  # pragma: no cov
-            self.trace: TraceManager = get_trace(
+            self.trace: Trace = get_trace(
                 self.run_id,
                 parent_run_id=self.parent_run_id,
                 extras=self.extras,
@@ -208,7 +210,7 @@ class Result:
         return self
 
     @classmethod
-    def from_trace(cls, trace: TraceManager):
+    def from_trace(cls, trace: Trace):
         """Construct the result model from trace for clean code objective."""
         return cls(
             run_id=trace.run_id,
@@ -274,6 +276,9 @@ def catch(
         context: A context data that want to be the current context.
         status: A status enum object.
         updated: A updated data that will update to the current context.
+
+    Returns:
+        DictData: A catch context data.
     """
     context.update(updated or {})
     context["status"] = Status(status) if isinstance(status, int) else status
@@ -291,3 +296,49 @@ def catch(
         else:
             raise ResultError(f"The key {k!r} does not exists on context data.")
     return context
+
+
+class Context(TypedDict):
+    """Context dict typed."""
+
+    status: Status
+    context: NotRequired[DictData]
+    errors: NotRequired[Union[list[ErrorData], ErrorData]]
+    info: NotRequired[DictData]
+
+
+class Layer(str, Enum):
+    WORKFLOW = "workflow"
+    JOB = "job"
+    STRATEGY = "strategy"
+    STAGE = "stage"
+
+
+def get_context_by_layer(
+    context: DictData,
+    key: str,
+    layer: Layer,
+    context_key: str,
+    *,
+    default: Optional[Any] = None,
+) -> Any:  # pragma: no cov
+    if layer == Layer.WORKFLOW:
+        return context.get("jobs", {}).get(key, {}).get(context_key, default)
+    elif layer == Layer.JOB:
+        return context.get("stages", {}).get(key, {}).get(context_key, default)
+    elif layer == Layer.STRATEGY:
+        return (
+            context.get("strategies", {}).get(key, {}).get(context_key, default)
+        )
+    return context.get(key, {}).get(context_key, default)
+
+
+def get_status(
+    context: DictData,
+    key: str,
+    layer: Layer,
+) -> Status:  # pragma: no cov
+    """Get status from context by a specific key and context layer."""
+    return get_context_by_layer(
+        context, key, layer, context_key="status", default=WAIT
+    )