ddeutil-workflow 0.0.81__tar.gz → 0.0.82__tar.gz

{ddeutil_workflow-0.0.81/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.82}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.81
+Version: 0.0.82
 Summary: Lightweight workflow orchestration with YAML template
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.81 → ddeutil_workflow-0.0.82}/pyproject.toml

@@ -91,7 +91,6 @@ omit = [
     "src/ddeutil/workflow/__cron.py",
     "src/ddeutil/workflow/__main__.py",
     "src/ddeutil/workflow/__types.py",
-    "src/ddeutil/workflow/cli.py",
     "src/ddeutil/workflow/api/__init__.py",
     "src/ddeutil/workflow/api/log_conf.py",
     "src/ddeutil/workflow/api/routes/__init__.py",

ddeutil_workflow-0.0.82/src/ddeutil/workflow/__about__.py

@@ -0,0 +1,2 @@
+__version__: str = "0.0.82"
+__python_version__: str = "3.9"

{ddeutil_workflow-0.0.81 → ddeutil_workflow-0.0.82}/src/ddeutil/workflow/__init__.py

@@ -50,11 +50,25 @@ Note:
 from .__cron import CronRunner
 from .__types import DictData, DictStr, Matrix, Re, TupleStr
 from .audits import (
+    EVENT,
+    FORCE,
+    NORMAL,
+    RERUN,
     Audit,
     FileAudit,
     get_audit,
 )
-from .conf import *
+from .conf import (
+    PREFIX,
+    CallerSecret,
+    Config,
+    YamlParser,
+    api_config,
+    config,
+    dynamic,
+    env,
+    pass_env,
+)
 from .errors import (
     BaseError,
     JobCancelError,
@@ -63,6 +77,9 @@ from .errors import (
     ResultError,
     StageCancelError,
     StageError,
+    StageNestedCancelError,
+    StageNestedError,
+    StageNestedSkipError,
     StageSkipError,
     UtilError,
     WorkflowCancelError,
@@ -132,15 +149,11 @@ from .stages import (
     VirtualPyStage,
 )
 from .traces import (
-    TraceManager,
+    Trace,
     get_trace,
 )
 from .utils import *
 from .workflow import (
-    EVENT,
-    FORCE,
-    NORMAL,
-    RERUN,
     ReleaseType,
     Workflow,
 )

ddeutil_workflow-0.0.81/src/ddeutil/workflow/cli.py → ddeutil_workflow-0.0.82/src/ddeutil/workflow/__main__.py

@@ -238,13 +238,12 @@ def workflow_execute(
     typer.echo(f"... with params: {params_dict}")
 
 
-WORKFLOW_TYPE = Literal["Workflow"]
-
-
 class WorkflowSchema(Workflow):
     """Override workflow model fields for generate JSON schema file."""
 
-    type: WORKFLOW_TYPE = Field(description="A type of workflow template.")
+    type: Literal["Workflow"] = Field(
+        description="A type of workflow template that should be `Workflow`."
+    )
     name: Optional[str] = Field(default=None, description="A workflow name.")
     params: dict[str, Union[Param, str]] = Field(
         default_factory=dict,

{ddeutil_workflow-0.0.81 → ddeutil_workflow-0.0.82}/src/ddeutil/workflow/api/routes/job.py

@@ -15,7 +15,7 @@ from fastapi.responses import UJSONResponse
 from ...__types import DictData
 from ...errors import JobError
 from ...job import Job
-from ...traces import TraceManager, get_trace
+from ...traces import Trace, get_trace
 from ...utils import gen_id
 
 logger = logging.getLogger("uvicorn.error")
@@ -41,7 +41,7 @@ async def job_execute(
     if extras:
         job.extras = extras
 
-    trace: TraceManager = get_trace(
+    trace: Trace = get_trace(
         run_id, parent_run_id=parent_run_id, extras=job.extras
     )
 

{ddeutil_workflow-0.0.81 → ddeutil_workflow-0.0.82}/src/ddeutil/workflow/api/routes/logs.py

@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""This route include audit and trace log paths."""
+"""This route include audit log path."""
 from __future__ import annotations
 
 from fastapi import APIRouter, Path, Query
@@ -11,7 +11,6 @@ from fastapi import status as st
 from fastapi.responses import UJSONResponse
 
 from ...audits import get_audit
-from ...result import Result
 
 router = APIRouter(
     prefix="/logs",
@@ -20,63 +19,6 @@ router = APIRouter(
 )
 
 
-@router.get(
-    path="/traces/",
-    response_class=UJSONResponse,
-    status_code=st.HTTP_200_OK,
-    summary="Read all trace logs.",
-    tags=["trace"],
-)
-async def get_traces(
-    offset: int = Query(default=0, gt=0),
-    limit: int = Query(default=100, gt=0),
-):
-    """Return all trace logs from the current trace log path that config with
-    `WORKFLOW_LOG_PATH` environment variable name.
-    """
-    result = Result()
-    return {
-        "message": (
-            f"Getting trace logs with offset: {offset} and limit: {limit}"
-        ),
-        "traces": [
-            trace.model_dump(
-                by_alias=True,
-                exclude_none=True,
-                exclude_unset=True,
-            )
-            for trace in result.trace.find_traces()
-        ],
-    }
-
-
-@router.get(
-    path="/traces/{run_id}",
-    response_class=UJSONResponse,
-    status_code=st.HTTP_200_OK,
-    summary="Read trace log with specific running ID.",
-    tags=["trace"],
-)
-async def get_trace_with_id(run_id: str):
-    """Return trace log with specific running ID from the current trace log path
-    that config with `WORKFLOW_LOG_PATH` environment variable name.
-
-    - **run_id**: A running ID that want to search a trace log from the log
-        path.
-    """
-    result = Result()
-    return {
-        "message": f"Getting trace log with specific running ID: {run_id}",
-        "trace": (
-            result.trace.find_trace_with_id(run_id).model_dump(
-                by_alias=True,
-                exclude_none=True,
-                exclude_unset=True,
-            )
-        ),
-    }
-
-
 @router.get(
     path="/audits/",
     response_class=UJSONResponse,
@@ -84,12 +26,17 @@ async def get_trace_with_id(run_id: str):
     summary="Read all audit logs.",
     tags=["audit"],
 )
-async def get_audits():
+async def get_audits(
+    offset: int = Query(default=0, gt=0),
+    limit: int = Query(default=100, gt=0),
+):
     """Return all audit logs from the current audit log path that config with
     `WORKFLOW_AUDIT_URL` environment variable name.
     """
     return {
-        "message": "Getting audit logs",
+        "message": (
+            f"Getting audit logs with offset: {offset} and limit: {limit}",
+        ),
         "audits": list(get_audit().find_audits(name="demo")),
     }
 

{ddeutil_workflow-0.0.81 → ddeutil_workflow-0.0.82}/src/ddeutil/workflow/audits.py

@@ -49,33 +49,65 @@ import zlib
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from datetime import datetime, timedelta
+from enum import Enum
 from pathlib import Path
 from typing import Annotated, Any, ClassVar, Literal, Optional, Union
 from urllib.parse import ParseResult, urlparse
 
-from pydantic import BaseModel, Field, TypeAdapter
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
 from .__types import DictData
 from .conf import dynamic
-from .traces import TraceManager, get_trace, set_logging
+from .traces import Trace, get_trace, set_logging
 
 logger = logging.getLogger("ddeutil.workflow")
 
 
+class ReleaseType(str, Enum):
+    """Release type enumeration for workflow execution modes.
+
+    This enum defines the different types of workflow releases that can be
+    triggered, each with specific behavior and use cases.
+
+    Attributes:
+        NORMAL: Standard workflow release execution
+        RERUN: Re-execution of previously failed workflow
+        EVENT: Event-triggered workflow execution
+        FORCE: Forced execution bypassing normal conditions
+    """
+
+    NORMAL = "normal"
+    RERUN = "rerun"
+    EVENT = "event"
+    FORCE = "force"
+
+
+NORMAL = ReleaseType.NORMAL
+RERUN = ReleaseType.RERUN
+EVENT = ReleaseType.EVENT
+FORCE = ReleaseType.FORCE
+
+
 class AuditData(BaseModel):
+    """Audit Data model."""
+
+    model_config = ConfigDict(use_enum_values=True)
+
     name: str = Field(description="A workflow name.")
     release: datetime = Field(description="A release datetime.")
-    type: str = Field(description="A running type before logging.")
+    type: ReleaseType = Field(
+        default=NORMAL, description="A running type before logging."
+    )
     context: DictData = Field(
         default_factory=dict,
         description="A context that receive from a workflow execution result.",
     )
+    run_id: str = Field(description="A running ID")
     parent_run_id: Optional[str] = Field(
         default=None, description="A parent running ID."
     )
-    run_id: str = Field(description="A running ID")
     runs_metadata: DictData = Field(
         default_factory=dict,
         description="A runs metadata that will use to tracking this audit log.",
@@ -122,7 +154,7 @@ class BaseAudit(BaseModel, ABC):
     @abstractmethod
     def is_pointed(
         self,
-        data: AuditData,
+        data: Any,
         *,
         extras: Optional[DictData] = None,
     ) -> bool:
@@ -328,21 +360,21 @@ class FileAudit(BaseAudit):
             return AuditData.model_validate(obj=json.load(f))
 
     def is_pointed(
-        self, data: AuditData, *, extras: Optional[DictData] = None
+        self,
+        data: Any,
+        *,
+        extras: Optional[DictData] = None,
     ) -> bool:
         """Check if the release log already exists at the destination log path.
 
         Args:
-            data: The workflow name.
+            data (str):
             extras: Optional extra parameters to override core config.
 
         Returns:
             bool: True if the release log exists, False otherwise.
         """
-        # NOTE: Return False if enable writing log flag does not set.
-        if not dynamic("enable_write_audit", extras=extras):
-            return False
-        return self.pointer(data).exists()
+        return self.pointer(AuditData.model_validate(data)).exists()
 
     def pointer(self, data: AuditData) -> Path:
         """Return release directory path generated from model data.
@@ -365,7 +397,7 @@ class FileAudit(BaseAudit):
             Self: The audit instance after saving.
         """
         audit = AuditData.model_validate(data)
-        trace: TraceManager = get_trace(
+        trace: Trace = get_trace(
             audit.run_id,
             parent_run_id=audit.parent_run_id,
             extras=self.extras,
@@ -655,7 +687,7 @@ class SQLiteAudit(BaseAudit):  # pragma: no cov
             ValueError: If SQLite database is not properly configured.
         """
         audit = AuditData.model_validate(data)
-        trace: TraceManager = get_trace(
+        trace: Trace = get_trace(
             audit.run_id,
             parent_run_id=audit.parent_run_id,
             extras=self.extras,
@@ -748,10 +780,7 @@ Audit = Annotated[
 ]
 
 
-def get_audit(
-    *,
-    extras: Optional[DictData] = None,
-) -> Audit:  # pragma: no cov
+def get_audit(extras: Optional[DictData] = None) -> Audit:  # pragma: no cov
     """Get an audit model dynamically based on the config audit path value.
 
     Args:

{ddeutil_workflow-0.0.81 → ddeutil_workflow-0.0.82}/src/ddeutil/workflow/conf.py

@@ -176,17 +176,6 @@ class YamlParser:
     """Base Load object that use to search config data by given some identity
     value like name of `Workflow` or `Crontab` templates.
 
-    :param name: (str) A name of key of config data that read with YAML
-        Environment object.
-    :param path: (Path) A config path object.
-    :param externals: (DictData) An external config data that want to add to
-        loaded config data.
-    :param extras: (DictDdata) An extra parameters that use to override core
-        config values.
-
-    :raise ValueError: If the data does not find on the config path with the
-        name parameter.
-
     Noted:
         The config data should have `type` key for modeling validation that
     make this loader know what is config should to do pass to.
@@ -209,6 +198,23 @@ class YamlParser:
         extras: Optional[DictData] = None,
         obj: Optional[Union[object, str]] = None,
     ) -> None:
+        """Main constructure function.
+
+        Args:
+            name (str): A name of key of config data that read with YAML
+                Environment object.
+            path (Path): A config path object.
+            externals (DictData): An external config data that want to add to
+                loaded config data.
+            extras (DictDdata): An extra parameters that use to override core
+                config values.
+            obj (object | str): An object that want to validate from the `type`
+                key before keeping the config data.
+
+        Raises:
+            ValueError: If the data does not find on the config path with the
+                name parameter.
+        """
         self.path: Path = Path(dynamic("conf_path", f=path, extras=extras))
         self.externals: DictData = externals or {}
         self.extras: DictData = extras or {}
@@ -242,17 +248,19 @@ class YamlParser:
         """Find data with specific key and return the latest modify date data if
         this key exists multiple files.
 
-        :param name: (str) A name of data that want to find.
-        :param path: (Path) A config path object.
-        :param paths: (list[Path]) A list of config path object.
-        :param obj: (object | str) An object that want to validate matching
-            before return.
-        :param extras: (DictData)  An extra parameter that use to override core
-            config values.
-        :param ignore_filename: (str) An ignore filename. Default is
-            ``.confignore`` filename.
+        Args:
+            name (str): A name of data that want to find.
+            path (Path): A config path object.
+            paths (list[Path]): A list of config path object.
+            obj (object | str): An object that want to validate matching
+                before return.
+            extras (DictData):  An extra parameter that use to override core
+                config values.
+            ignore_filename (str): An ignore filename. Default is
+                ``.confignore`` filename.
 
-        :rtype: DictData
+        Returns:
+            DictData: A config data that was found on the searching paths.
         """
         path: Path = dynamic("conf_path", f=path, extras=extras)
         if not paths:
@@ -317,7 +325,9 @@ class YamlParser:
                 ``.confignore`` filename.
             tags (list[str]): A list of tag that want to filter.
 
-        :rtype: Iterator[tuple[str, DictData]]
+        Returns:
+            Iterator[tuple[str, DictData]]: An iterator of config data that was
+                found on the searching paths.
         """
         excluded: list[str] = excluded or []
         tags: list[str] = tags or []
@@ -353,8 +363,11 @@ class YamlParser:
                     ):
                         continue
 
-                    if (t := data.get("type")) and t == obj_type:
-
+                    if (
+                        # isinstance(data, dict) and
+                        (t := data.get("type"))
+                        and t == obj_type
+                    ):
                         # NOTE: Start adding file metadata.
                         file_stat: os.stat_result = file.lstat()
                         data["created_at"] = file_stat.st_ctime
@@ -397,6 +410,13 @@ class YamlParser:
     def filter_yaml(cls, file: Path, name: Optional[str] = None) -> DictData:
         """Read a YAML file context from an input file path and specific name.
 
+        Notes:
+            The data that will return from reading context will map with config
+            name if an input searching name does not pass to this function.
+
+                input: {"name": "foo", "type": "Some"}
+                output: {"foo": {"name": "foo", "type": "Some"}}
+
         Args:
             file (Path): A file path that want to extract YAML context.
             name (str): A key name that search on a YAML context.
@@ -413,7 +433,7 @@ class YamlParser:
                     return (
                         values[name] | {"name": name} if name in values else {}
                     )
-                return values
+                return {values["name"]: values} if "name" in values else values
         return {}
 
     @cached_property

{ddeutil_workflow-0.0.81 → ddeutil_workflow-0.0.82}/src/ddeutil/workflow/errors.py

@@ -166,6 +166,15 @@ class StageCancelError(StageError): ...
 class StageSkipError(StageError): ...
 
 
+class StageNestedError(StageError): ...
+
+
+class StageNestedCancelError(StageNestedError): ...
+
+
+class StageNestedSkipError(StageNestedError): ...
+
+
 class JobError(BaseError): ...
 
 

{ddeutil_workflow-0.0.81 → ddeutil_workflow-0.0.82}/src/ddeutil/workflow/job.py

@@ -48,7 +48,7 @@ from enum import Enum
 from functools import lru_cache
 from textwrap import dedent
 from threading import Event
-from typing import Annotated, Any, Optional, Union
+from typing import Annotated, Any, Literal, Optional, Union
 
 from ddeutil.core import freeze_args
 from pydantic import BaseModel, Discriminator, Field, SecretStr, Tag
@@ -72,7 +72,7 @@ from .result import (
 )
 from .reusables import has_template, param2template
 from .stages import Stage
-from .traces import TraceManager, get_trace
+from .traces import Trace, get_trace
 from .utils import cross_product, filter_func, gen_id
 
 MatrixFilter = list[dict[str, Union[str, int]]]
@@ -187,10 +187,8 @@ class Strategy(BaseModel):
         ),
         alias="fail-fast",
     )
-    max_parallel: int = Field(
+    max_parallel: Union[int, str] = Field(
         default=1,
-        gt=0,
-        lt=10,
         description=(
             "The maximum number of executor thread pool that want to run "
             "parallel. This value should gather than 0 and less than 10."
@@ -427,9 +425,9 @@ class OnGCPBatch(BaseRunsOn):  # pragma: no cov
     args: GCPBatchArgs = Field(alias="with")
 
 
-def get_discriminator_runs_on(model: dict[str, Any]) -> RunsOn:
+def get_discriminator_runs_on(data: dict[str, Any]) -> RunsOn:
     """Get discriminator of the RunsOn models."""
-    t: str = model.get("type")
+    t: str = data.get("type")
     return RunsOn(t) if t else LOCAL
 
 
@@ -538,13 +536,28 @@ class Job(BaseModel):
         description="An extra override config values.",
     )
 
+    @field_validator(
+        "runs_on",
+        mode="before",
+        json_schema_input_type=Union[RunsOnModel, Literal["local"]],
+    )
+    def __prepare_runs_on(cls, data: Any) -> Any:
+        """Prepare runs on value that was passed with string type."""
+        if isinstance(data, str):
+            if data != "local":
+                raise ValueError(
+                    "runs-on that pass with str type should be `local` only"
+                )
+            return {"type": data}
+        return data
+
     @field_validator("desc", mode="after")
-    def ___prepare_desc__(cls, value: str) -> str:
+    def ___prepare_desc__(cls, data: str) -> str:
         """Prepare description string that was created on a template.
 
         :rtype: str
         """
-        return dedent(value.lstrip("\n"))
+        return dedent(data.lstrip("\n"))
 
     @field_validator("stages", mode="after")
     def __validate_stage_id__(cls, value: list[Stage]) -> list[Stage]:
@@ -879,7 +892,7 @@ class Job(BaseModel):
         ts: float = time.monotonic()
         parent_run_id: str = run_id
         run_id: str = gen_id((self.id or "EMPTY"), unique=True)
-        trace: TraceManager = get_trace(
+        trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
         trace.info(
@@ -1016,7 +1029,7 @@ def local_execute_strategy(
 
     :rtype: tuple[Status, DictData]
     """
-    trace: TraceManager = get_trace(
+    trace: Trace = get_trace(
         run_id, parent_run_id=parent_run_id, extras=job.extras
     )
     if strategy:
@@ -1152,7 +1165,7 @@ def local_execute(
     ts: float = time.monotonic()
     parent_run_id: StrOrNone = run_id
     run_id: str = gen_id((job.id or "EMPTY"), unique=True)
-    trace: TraceManager = get_trace(
+    trace: Trace = get_trace(
         run_id, parent_run_id=parent_run_id, extras=job.extras
     )
     context: DictData = {"status": WAIT}
@@ -1174,11 +1187,52 @@ def local_execute(
 
     event: Event = event or Event()
     ls: str = "Fail-Fast" if job.strategy.fail_fast else "All-Completed"
-    workers: int = job.strategy.max_parallel
+    workers: Union[int, str] = job.strategy.max_parallel
+    if isinstance(workers, str):
+        try:
+            workers: int = int(
+                param2template(workers, params=params, extras=job.extras)
+            )
+        except Exception as err:
+            trace.exception(
+                "[JOB]: Got the error on call param2template to "
+                f"max-parallel value: {workers}"
+            )
+            return Result(
+                run_id=run_id,
+                parent_run_id=parent_run_id,
+                status=FAILED,
+                context=catch(
+                    context,
+                    status=FAILED,
+                    updated={"errors": to_dict(err)},
+                ),
+                info={"execution_time": time.monotonic() - ts},
+                extras=job.extras,
+            )
+    if workers >= 10:
+        err_msg: str = (
+            f"The max-parallel value should not more than 10, the current value "
+            f"was set: {workers}."
+        )
+        trace.error(f"[JOB]: {err_msg}")
+        return Result(
+            run_id=run_id,
+            parent_run_id=parent_run_id,
+            status=FAILED,
+            context=catch(
+                context,
+                status=FAILED,
+                updated={"errors": JobError(err_msg).to_dict()},
+            ),
+            info={"execution_time": time.monotonic() - ts},
+            extras=job.extras,
+        )
+
     strategies: list[DictStr] = job.strategy.make()
     len_strategy: int = len(strategies)
     trace.info(
-        f"[JOB]: ... Mode {ls}: {job.id!r} with {workers} "
+        f"[JOB]: Mode {ls}: {job.id!r} with {workers} "
         f"worker{'s' if workers > 1 else ''}."
     )
 
@@ -1295,7 +1349,7 @@ def self_hosted_execute(
     """
     parent_run_id: StrOrNone = run_id
     run_id: str = gen_id((job.id or "EMPTY"), unique=True)
-    trace: TraceManager = get_trace(
+    trace: Trace = get_trace(
         run_id, parent_run_id=parent_run_id, extras=job.extras
     )
     context: DictData = {"status": WAIT}
@@ -1378,7 +1432,7 @@ def docker_execution(
     """
     parent_run_id: StrOrNone = run_id
     run_id: str = gen_id((job.id or "EMPTY"), unique=True)
-    trace: TraceManager = get_trace(
+    trace: Trace = get_trace(
         run_id, parent_run_id=parent_run_id, extras=job.extras
     )
     context: DictData = {"status": WAIT}