ddeutil-workflow 0.0.80__py3-none-any.whl → 0.0.82__py3-none-any.whl

ddeutil/workflow/conf.py

@@ -22,19 +22,6 @@ Functions:
     pass_env: Process environment variable substitution
     api_config: Get API-specific configuration settings
 
-Example:
-    ```python
-    from ddeutil.workflow.conf import Config, YamlParser
-
-    # Load workflow configuration
-    parser = YamlParser("my-workflow")
-    workflow_config = parser.data
-
-    # Access dynamic configuration
-    from ddeutil.workflow.conf import dynamic
-    log_level = dynamic("log_level", default="INFO")
-    ```
-
 Note:
     Configuration files support environment variable substitution using
     ${VAR_NAME} syntax and provide extensive validation capabilities.
@@ -155,7 +142,7 @@ class Config:  # pragma: no cov
         )
 
     @property
-    def audit_conf(self) -> str:
+    def audit_conf(self) -> dict[str, Any]:
         return json.loads(
             env("LOG_AUDIT_URL", '{"type": "file", "path": "./audits"}')
         )
@@ -189,17 +176,6 @@ class YamlParser:
     """Base Load object that use to search config data by given some identity
     value like name of `Workflow` or `Crontab` templates.
 
-    :param name: (str) A name of key of config data that read with YAML
-        Environment object.
-    :param path: (Path) A config path object.
-    :param externals: (DictData) An external config data that want to add to
-        loaded config data.
-    :param extras: (DictDdata) An extra parameters that use to override core
-        config values.
-
-    :raise ValueError: If the data does not find on the config path with the
-        name parameter.
-
     Noted:
         The config data should have `type` key for modeling validation that
     make this loader know what is config should to do pass to.
@@ -222,6 +198,23 @@ class YamlParser:
         extras: Optional[DictData] = None,
         obj: Optional[Union[object, str]] = None,
     ) -> None:
+        """Main constructure function.
+
+        Args:
+            name (str): A name of key of config data that read with YAML
+                Environment object.
+            path (Path): A config path object.
+            externals (DictData): An external config data that want to add to
+                loaded config data.
+            extras (DictDdata): An extra parameters that use to override core
+                config values.
+            obj (object | str): An object that want to validate from the `type`
+                key before keeping the config data.
+
+        Raises:
+            ValueError: If the data does not find on the config path with the
+                name parameter.
+        """
         self.path: Path = Path(dynamic("conf_path", f=path, extras=extras))
         self.externals: DictData = externals or {}
         self.extras: DictData = extras or {}
@@ -255,17 +248,19 @@ class YamlParser:
         """Find data with specific key and return the latest modify date data if
         this key exists multiple files.
 
-        :param name: (str) A name of data that want to find.
-        :param path: (Path) A config path object.
-        :param paths: (list[Path]) A list of config path object.
-        :param obj: (object | str) An object that want to validate matching
-            before return.
-        :param extras: (DictData)  An extra parameter that use to override core
-            config values.
-        :param ignore_filename: (str) An ignore filename. Default is
-            ``.confignore`` filename.
+        Args:
+            name (str): A name of data that want to find.
+            path (Path): A config path object.
+            paths (list[Path]): A list of config path object.
+            obj (object | str): An object that want to validate matching
+                before return.
+            extras (DictData):  An extra parameter that use to override core
+                config values.
+            ignore_filename (str): An ignore filename. Default is
+                ``.confignore`` filename.
 
-        :rtype: DictData
+        Returns:
+            DictData: A config data that was found on the searching paths.
         """
         path: Path = dynamic("conf_path", f=path, extras=extras)
         if not paths:
@@ -288,9 +283,12 @@ class YamlParser:
                     continue
 
                 if data := cls.filter_yaml(file, name=name):
+
+                    # NOTE: Start adding file metadata.
                     file_stat: os.stat_result = file.lstat()
                     data["created_at"] = file_stat.st_ctime
                     data["updated_at"] = file_stat.st_mtime
+
                     if not obj_type:
                         all_data.append((file_stat.st_mtime, data))
                     elif (t := data.get("type")) and t == obj_type:
@@ -324,11 +322,12 @@ class YamlParser:
             extras: (DictData) An extra parameter that use to override core
                 config values.
             ignore_filename: (str) An ignore filename. Default is
-            ``.confignore`` filename.
-            tags: (list[str])
-                A list of tag that want to filter.
+                ``.confignore`` filename.
+            tags (list[str]): A list of tag that want to filter.
 
-        :rtype: Iterator[tuple[str, DictData]]
+        Returns:
+            Iterator[tuple[str, DictData]]: An iterator of config data that was
+                found on the searching paths.
         """
         excluded: list[str] = excluded or []
         tags: list[str] = tags or []
@@ -364,7 +363,12 @@ class YamlParser:
                     ):
                         continue
 
-                    if (t := data.get("type")) and t == obj_type:
+                    if (
+                        # isinstance(data, dict) and
+                        (t := data.get("type"))
+                        and t == obj_type
+                    ):
+                        # NOTE: Start adding file metadata.
                         file_stat: os.stat_result = file.lstat()
                         data["created_at"] = file_stat.st_ctime
                         data["updated_at"] = file_stat.st_mtime
@@ -372,6 +376,7 @@ class YamlParser:
                             file.lstat().st_mtime,
                             data,
                         )
+
                         if key in all_data:
                             all_data[key].append(marking)
                         else:
@@ -405,15 +410,30 @@ class YamlParser:
     def filter_yaml(cls, file: Path, name: Optional[str] = None) -> DictData:
         """Read a YAML file context from an input file path and specific name.
 
-        :param file: (Path) A file path that want to extract YAML context.
-        :param name: (str) A key name that search on a YAML context.
+        Notes:
+            The data that will return from reading context will map with config
+            name if an input searching name does not pass to this function.
+
+                input: {"name": "foo", "type": "Some"}
+                output: {"foo": {"name": "foo", "type": "Some"}}
 
-        :rtype: DictData
+        Args:
+            file (Path): A file path that want to extract YAML context.
+            name (str): A key name that search on a YAML context.
+
+        Returns:
+            DictData: A data that read from this file if it is YAML format.
         """
         if any(file.suffix.endswith(s) for s in (".yml", ".yaml")):
             values: DictData = YamlFlResolve(file).read()
             if values is not None:
-                return values.get(name, {}) if name else values
+                if name:
+                    if "name" in values and values.get("name") == name:
+                        return values
+                    return (
+                        values[name] | {"name": name} if name in values else {}
+                    )
+                return {values["name"]: values} if "name" in values else values
         return {}
 
     @cached_property

ddeutil/workflow/errors.py

@@ -135,11 +135,12 @@ class BaseError(Exception):
 
         Example:
             >>> error = BaseError("Something failed", refs="stage-1")
-            >>> # Simple format
+
+            Simple format
             >>> error.to_dict()
             >>> # Returns: {"name": "BaseError", "message": "Something failed"}
 
-            >>> # With reference mapping
+            With reference mapping
             >>> error.to_dict(with_refs=True)
             >>> # Returns: {"stage-1": {"name": "BaseError", "message": "Something failed"}}
             ```
@@ -165,6 +166,15 @@ class StageCancelError(StageError): ...
 class StageSkipError(StageError): ...
 
 
+class StageNestedError(StageError): ...
+
+
+class StageNestedCancelError(StageNestedError): ...
+
+
+class StageNestedSkipError(StageNestedError): ...
+
+
 class JobError(BaseError): ...
 
 

ddeutil/workflow/job.py

@@ -48,7 +48,7 @@ from enum import Enum
 from functools import lru_cache
 from textwrap import dedent
 from threading import Event
-from typing import Annotated, Any, Optional, Union
+from typing import Annotated, Any, Literal, Optional, Union
 
 from ddeutil.core import freeze_args
 from pydantic import BaseModel, Discriminator, Field, SecretStr, Tag
@@ -72,7 +72,7 @@ from .result import (
 )
 from .reusables import has_template, param2template
 from .stages import Stage
-from .traces import TraceManager, get_trace
+from .traces import Trace, get_trace
 from .utils import cross_product, filter_func, gen_id
 
 MatrixFilter = list[dict[str, Union[str, int]]]
@@ -187,10 +187,8 @@ class Strategy(BaseModel):
         ),
         alias="fail-fast",
     )
-    max_parallel: int = Field(
+    max_parallel: Union[int, str] = Field(
         default=1,
-        gt=0,
-        lt=10,
         description=(
             "The maximum number of executor thread pool that want to run "
             "parallel. This value should gather than 0 and less than 10."
@@ -427,9 +425,9 @@ class OnGCPBatch(BaseRunsOn):  # pragma: no cov
     args: GCPBatchArgs = Field(alias="with")
 
 
-def get_discriminator_runs_on(model: dict[str, Any]) -> RunsOn:
+def get_discriminator_runs_on(data: dict[str, Any]) -> RunsOn:
     """Get discriminator of the RunsOn models."""
-    t: str = model.get("type")
+    t: str = data.get("type")
     return RunsOn(t) if t else LOCAL
 
 
@@ -538,13 +536,28 @@ class Job(BaseModel):
         description="An extra override config values.",
     )
 
+    @field_validator(
+        "runs_on",
+        mode="before",
+        json_schema_input_type=Union[RunsOnModel, Literal["local"]],
+    )
+    def __prepare_runs_on(cls, data: Any) -> Any:
+        """Prepare runs on value that was passed with string type."""
+        if isinstance(data, str):
+            if data != "local":
+                raise ValueError(
+                    "runs-on that pass with str type should be `local` only"
+                )
+            return {"type": data}
+        return data
+
     @field_validator("desc", mode="after")
-    def ___prepare_desc__(cls, value: str) -> str:
+    def ___prepare_desc__(cls, data: str) -> str:
         """Prepare description string that was created on a template.
 
         :rtype: str
         """
-        return dedent(value.lstrip("\n"))
+        return dedent(data.lstrip("\n"))
 
     @field_validator("stages", mode="after")
     def __validate_stage_id__(cls, value: list[Stage]) -> list[Stage]:
@@ -879,7 +892,7 @@ class Job(BaseModel):
         ts: float = time.monotonic()
         parent_run_id: str = run_id
         run_id: str = gen_id((self.id or "EMPTY"), unique=True)
-        trace: TraceManager = get_trace(
+        trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
         trace.info(
@@ -1016,7 +1029,7 @@ def local_execute_strategy(
 
     :rtype: tuple[Status, DictData]
     """
-    trace: TraceManager = get_trace(
+    trace: Trace = get_trace(
         run_id, parent_run_id=parent_run_id, extras=job.extras
     )
     if strategy:
@@ -1152,7 +1165,7 @@ def local_execute(
     ts: float = time.monotonic()
     parent_run_id: StrOrNone = run_id
     run_id: str = gen_id((job.id or "EMPTY"), unique=True)
-    trace: TraceManager = get_trace(
+    trace: Trace = get_trace(
         run_id, parent_run_id=parent_run_id, extras=job.extras
     )
     context: DictData = {"status": WAIT}
@@ -1174,11 +1187,52 @@ def local_execute(
 
     event: Event = event or Event()
     ls: str = "Fail-Fast" if job.strategy.fail_fast else "All-Completed"
-    workers: int = job.strategy.max_parallel
+    workers: Union[int, str] = job.strategy.max_parallel
+    if isinstance(workers, str):
+        try:
+            workers: int = int(
+                param2template(workers, params=params, extras=job.extras)
+            )
+        except Exception as err:
+            trace.exception(
+                "[JOB]: Got the error on call param2template to "
+                f"max-parallel value: {workers}"
+            )
+            return Result(
+                run_id=run_id,
+                parent_run_id=parent_run_id,
+                status=FAILED,
+                context=catch(
+                    context,
+                    status=FAILED,
+                    updated={"errors": to_dict(err)},
+                ),
+                info={"execution_time": time.monotonic() - ts},
+                extras=job.extras,
+            )
+    if workers >= 10:
+        err_msg: str = (
+            f"The max-parallel value should not more than 10, the current value "
+            f"was set: {workers}."
+        )
+        trace.error(f"[JOB]: {err_msg}")
+        return Result(
+            run_id=run_id,
+            parent_run_id=parent_run_id,
+            status=FAILED,
+            context=catch(
+                context,
+                status=FAILED,
+                updated={"errors": JobError(err_msg).to_dict()},
+            ),
+            info={"execution_time": time.monotonic() - ts},
+            extras=job.extras,
+        )
+
     strategies: list[DictStr] = job.strategy.make()
     len_strategy: int = len(strategies)
     trace.info(
-        f"[JOB]: ... Mode {ls}: {job.id!r} with {workers} "
+        f"[JOB]: Mode {ls}: {job.id!r} with {workers} "
         f"worker{'s' if workers > 1 else ''}."
     )
 
@@ -1295,7 +1349,7 @@ def self_hosted_execute(
     """
     parent_run_id: StrOrNone = run_id
     run_id: str = gen_id((job.id or "EMPTY"), unique=True)
-    trace: TraceManager = get_trace(
+    trace: Trace = get_trace(
         run_id, parent_run_id=parent_run_id, extras=job.extras
     )
     context: DictData = {"status": WAIT}
@@ -1378,7 +1432,7 @@ def docker_execution(
     """
     parent_run_id: StrOrNone = run_id
     run_id: str = gen_id((job.id or "EMPTY"), unique=True)
-    trace: TraceManager = get_trace(
+    trace: Trace = get_trace(
         run_id, parent_run_id=parent_run_id, extras=job.extras
     )
     context: DictData = {"status": WAIT}

ddeutil/workflow/result.py

@@ -21,12 +21,12 @@ from __future__ import annotations
 
 from dataclasses import field
 from enum import Enum
-from typing import Optional, Union
+from typing import Optional, TypedDict, Union
 
 from pydantic import ConfigDict
 from pydantic.dataclasses import dataclass
 from pydantic.functional_validators import model_validator
-from typing_extensions import Self
+from typing_extensions import NotRequired, Self
 
 from . import (
     JobCancelError,
@@ -34,13 +34,16 @@ from . import (
     JobSkipError,
     StageCancelError,
     StageError,
+    StageNestedCancelError,
+    StageNestedError,
+    StageNestedSkipError,
     StageSkipError,
     WorkflowCancelError,
     WorkflowError,
 )
 from .__types import DictData
-from .audits import TraceManager, get_trace
-from .errors import ResultError
+from .audits import Trace, get_trace
+from .errors import ErrorData, ResultError
 from .utils import default_gen_id
 
 
@@ -140,6 +143,9 @@ def get_status_from_error(
         StageError,
         StageCancelError,
         StageSkipError,
+        StageNestedCancelError,
+        StageNestedError,
+        StageNestedSkipError,
         JobError,
         JobCancelError,
         JobSkipError,
@@ -157,10 +163,16 @@ def get_status_from_error(
     Returns:
         Status: The status from the specific exception class.
     """
-    if isinstance(error, (StageSkipError, JobSkipError)):
+    if isinstance(error, (StageNestedSkipError, StageSkipError, JobSkipError)):
         return SKIP
     elif isinstance(
-        error, (StageCancelError, JobCancelError, WorkflowCancelError)
+        error,
+        (
+            StageNestedCancelError,
+            StageCancelError,
+            JobCancelError,
+            WorkflowCancelError,
+        ),
     ):
         return CANCEL
     return FAILED
@@ -188,9 +200,7 @@ class Result:
     info: DictData = field(default_factory=dict)
     run_id: str = field(default_factory=default_gen_id)
     parent_run_id: Optional[str] = field(default=None)
-    trace: Optional[TraceManager] = field(
-        default=None, compare=False, repr=False
-    )
+    trace: Optional[Trace] = field(default=None, compare=False, repr=False)
 
     @model_validator(mode="after")
     def __prepare_trace(self) -> Self:
@@ -199,7 +209,7 @@ class Result:
         :rtype: Self
         """
         if self.trace is None:  # pragma: no cov
-            self.trace: TraceManager = get_trace(
+            self.trace: Trace = get_trace(
                 self.run_id,
                 parent_run_id=self.parent_run_id,
                 extras=self.extras,
@@ -208,7 +218,7 @@ class Result:
         return self
 
     @classmethod
-    def from_trace(cls, trace: TraceManager):
+    def from_trace(cls, trace: Trace):
         """Construct the result model from trace for clean code objective."""
         return cls(
             run_id=trace.run_id,
@@ -274,6 +284,9 @@ def catch(
         context: A context data that want to be the current context.
         status: A status enum object.
         updated: A updated data that will update to the current context.
+
+    Returns:
+        DictData: A catch context data.
     """
     context.update(updated or {})
     context["status"] = Status(status) if isinstance(status, int) else status
@@ -291,3 +304,12 @@ def catch(
         else:
             raise ResultError(f"The key {k!r} does not exists on context data.")
     return context
+
+
+class Context(TypedDict):
+    """Context dict typed."""
+
+    status: Status
+    context: NotRequired[DictData]
+    errors: NotRequired[Union[list[ErrorData], ErrorData]]
+    info: NotRequired[DictData]

ddeutil/workflow/reusables.py

@@ -25,22 +25,20 @@ Functions:
     create_model_from_caller: Generate Pydantic models from function signatures
 
 Example:
-    ```python
-    from ddeutil.workflow.reusables import tag
-
-    @tag("data-processing", alias="process-csv")
-    def process_csv_file(input_path: str, output_path: str) -> dict:
-        # Custom processing logic
-        return {"status": "completed", "rows_processed": 1000}
-
-    # Use in workflow YAML:
-    # stages:
-    #   - name: "Process data"
-    #     uses: "data-processing/process-csv@latest"
-    #     args:
-    #       input_path: "/data/input.csv"
-    #       output_path: "/data/output.csv"
-    ```
+
+    >>> from ddeutil.workflow.reusables import tag
+    >>>
+    >>> @tag("data-processing", alias="process-csv")
+    >>> def process_csv_file(input_path: str, output_path: str) -> dict:
+    >>>     return {"status": "completed", "rows_processed": 1000}
+
+    >>> # Use in workflow YAML:
+    >>> # stages:
+    >>> #   - name: "Process data"
+    >>> #     uses: "data-processing/process-csv@latest"
+    >>> #     args:
+    >>> #       input_path: "/data/input.csv"
+    >>> #       output_path: "/data/output.csv"
 
 Note:
     The registry system supports versioning and aliasing for better function
@@ -64,6 +62,7 @@ from typing import (
     Protocol,
     TypeVar,
     Union,
+    cast,
     get_type_hints,
 )
 
@@ -201,7 +200,7 @@ def get_args_const(
             f"Post-filter: {expr} does not valid because it raise syntax error."
         ) from None
 
-    body: list[Expr] = mod.body
+    body: list[Expr] = cast(list[Expr], mod.body)
     if len(body) > 1:
         raise UtilError(
             "Post-filter function should be only one calling per workflow."