ddeutil-workflow 0.0.72__py3-none-any.whl → 0.0.74__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.72"
+__version__: str = "0.0.74"

ddeutil/workflow/__cron.py

@@ -38,8 +38,11 @@ class YearReachLimit(Exception):
 def str2cron(value: str) -> str:  # pragma: no cov
     """Convert Special String with the @ prefix to Crontab value.
 
-    :param value: (str) A string value that want to convert to cron value.
-    :rtype: str
+    Args:
+        value: A string value that want to convert to cron value.
+
+    Returns:
+        str: The converted cron expression.
 
     Table:
 
@@ -165,9 +168,10 @@ CRON_UNITS_YEAR: Units = CRON_UNITS + (
 class CronPart:
     """Part of Cron object that represent a collection of positive integers.
 
-    :param unit: A Unit dataclass object.
-    :param values: A crontab values that want to validate
-    :param options: A Options dataclass object.
+    Args:
+        unit: A Unit dataclass object.
+        values: A crontab values that want to validate
+        options: A Options dataclass object.
     """
 
     __slots__: tuple[str, ...] = (
@@ -288,7 +292,11 @@ class CronPart:
         """Parses a string as a range of positive integers. The string should
         include only `-` and `,` special strings.
 
-        :param value: (str) A string value that want to parse
+        Args:
+            value: A string value that want to parse
+
+        Returns:
+            tuple[int, ...]: Parsed range of integers.
 
         TODO: support for `L`, `W`, and `#`
         ---
@@ -334,8 +342,6 @@ class CronPart:
             -   15 10 ? * 6L 2002-2005
                 Run at 10:15am UTC on the last Friday of each month during the
                 years 2002 to 2005
-
-        :rtype: tuple[int, ...]
         """
         interval_list: list[list[int]] = []
         # NOTE: Start replace alternative like JAN to FEB or MON to SUN.

ddeutil/workflow/__init__.py

@@ -3,19 +3,108 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-from .__cron import CronJob, CronRunner
+"""DDE Workflow - Lightweight Workflow Orchestration Package.
+
+This package provides a comprehensive workflow orchestration system with YAML template
+support. It enables developers to create, manage, and execute complex workflows with
+minimal configuration.
+
+Key Features:
+    - YAML-based workflow configuration
+    - Job and stage execution management
+    - Scheduling with cron-like syntax
+    - Parallel and sequential execution support
+    - Comprehensive error handling and logging
+    - Extensible stage types (Bash, Python, Docker, etc.)
+    - Matrix strategy for parameterized workflows
+    - Audit and tracing capabilities
+
+Main Classes:
+    Workflow: Core workflow orchestration class
+    Job: Execution unit containing stages
+    Stage: Individual task execution unit
+    CronJob: Scheduled workflow execution
+    Audit: Execution tracking and logging
+    Result: Execution status and output management
+
+Example:
+    Basic workflow usage:
+
+    ```python
+    from ddeutil.workflow import Workflow
+
+    # Load workflow from configuration
+    workflow = Workflow.from_conf('my-workflow')
+
+    # Execute with parameters
+    result = workflow.execute({'param1': 'value1'})
+
+    if result.status == 'SUCCESS':
+        print("Workflow completed successfully")
+    ```
+
+Note:
+    This package requires Python 3.9+ and supports both synchronous and
+    asynchronous execution patterns.
+"""
+from .__cron import CronRunner
 from .__types import DictData, DictStr, Matrix, Re, TupleStr
 from .audits import (
     Audit,
-    AuditModel,
     FileAudit,
-    get_audit,
+    get_audit_model,
 )
 from .conf import *
-from .errors import *
-from .event import *
-from .job import *
-from .params import *
+from .errors import (
+    BaseError,
+    JobCancelError,
+    JobError,
+    JobSkipError,
+    ResultError,
+    StageCancelError,
+    StageError,
+    StageSkipError,
+    UtilError,
+    WorkflowCancelError,
+    WorkflowError,
+    WorkflowTimeoutError,
+    to_dict,
+)
+from .event import (
+    Cron,
+    CronJob,
+    CronJobYear,
+    Crontab,
+    CrontabValue,
+    CrontabYear,
+    Event,
+    Interval,
+)
+from .job import (
+    Job,
+    OnAzBatch,
+    OnDocker,
+    OnLocal,
+    OnSelfHosted,
+    Rule,
+    RunsOnModel,
+    Strategy,
+    docker_execution,
+    local_execute,
+    local_execute_strategy,
+    self_hosted_execute,
+)
+from .params import (
+    ArrayParam,
+    DateParam,
+    DatetimeParam,
+    DecimalParam,
+    FloatParam,
+    IntParam,
+    MapParam,
+    Param,
+    StrParam,
+)
 from .result import (
     CANCEL,
     FAILED,
@@ -26,15 +115,35 @@ from .result import (
     Status,
 )
 from .reusables import *
-from .stages import *
+from .stages import (
+    BashStage,
+    CallStage,
+    CaseStage,
+    DockerStage,
+    EmptyStage,
+    ForEachStage,
+    ParallelStage,
+    PyStage,
+    RaiseStage,
+    Stage,
+    TriggerStage,
+    UntilStage,
+    VirtualPyStage,
+)
 from .traces import (
     ConsoleTrace,
     FileTrace,
     Trace,
     TraceData,
     TraceMeta,
-    TraceModel,
     get_trace,
 )
 from .utils import *
-from .workflow import *
+from .workflow import (
+    EVENT,
+    FORCE,
+    NORMAL,
+    RERUN,
+    ReleaseType,
+    Workflow,
+)

ddeutil/workflow/__types.py

@@ -27,6 +27,47 @@ DictData = dict[str, Any]
 DictStr = dict[str, str]
 Matrix = dict[str, Union[list[str], list[int]]]
 
+# Pre-compile regex patterns for better performance
+_RE_CALLER_PATTERN = r"""
+    \$                                                      # start with $
+    {{                                                      # value open with {{
+        \s*                                                 # whitespace or not
+        (?P<caller>
+            (?P<caller_prefix>(?:[a-zA-Z_-]+\??\.)*)
+            (?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+\??)
+        )
+        \s*                                                 # whitespace or not
+        (?P<post_filters>
+            (?:
+                \|\s*
+                (?:
+                    [a-zA-Z0-9_]{3,}
+                    [a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]*
+                )\s*
+            )*
+        )
+    }}                                                      # value close with }}
+"""
+
+_RE_TASK_FMT_PATTERN = r"""
+    ^                               # start task format
+        (?P<path>[^/@]+)
+        /                           # start get function with /
+        (?P<func>[^@]+)
+        @                           # start tag with @
+        (?P<tag>.+)
+    $                               # end task format
+"""
+
+# Compile patterns at module level for better performance
+RE_CALLER: Pattern = re.compile(
+    _RE_CALLER_PATTERN, MULTILINE | IGNORECASE | UNICODE | VERBOSE
+)
+
+RE_TASK_FMT: Pattern = re.compile(
+    _RE_TASK_FMT_PATTERN, MULTILINE | IGNORECASE | UNICODE | VERBOSE
+)
+
 
 class Context(TypedDict):
     """TypeDict support the Context."""
@@ -51,10 +92,12 @@ class CallerRe:
     def from_regex(cls, match: Match[str]) -> Self:
         """Class construct from matching result.
 
-        :param match: A match string object for contract this Caller regex data
-            class.
+        Args:
+            match: A match string object for contract this Caller regex data
+                class.
 
-        :rtype: Self
+        Returns:
+            Self: The constructed CallerRe instance from regex match.
         """
         return cls(full=match.group(0), **match.groupdict())
 
@@ -79,29 +122,7 @@ class Re:
     #       - ${{ params.datetime | fmt('%Y-%m-%d') }}
     #       - ${{ params.source?.schema }}
     #
-    __re_caller: str = r"""
-        \$                                                      # start with $
-        {{                                                      # value open with {{
-            \s*                                                 # whitespace or not
-            (?P<caller>
-                (?P<caller_prefix>(?:[a-zA-Z_-]+\??\.)*)
-                (?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+\??)
-            )
-            \s*                                                 # whitespace or not
-            (?P<post_filters>
-                (?:
-                    \|\s*
-                    (?:
-                        [a-zA-Z0-9_]{3,}
-                        [a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]*
-                    )\s*
-                )*
-            )
-        }}                                                      # value close with }}
-    """
-    RE_CALLER: Pattern = re.compile(
-        __re_caller, MULTILINE | IGNORECASE | UNICODE | VERBOSE
-    )
+    RE_CALLER: Pattern = RE_CALLER
 
     # NOTE:
     #   Regular expression:
@@ -111,28 +132,19 @@ class Re:
     #   Examples:
     #       - tasks/function@dummy
     #
-    __re_task_fmt: str = r"""
-        ^                               # start task format
-            (?P<path>[^/@]+)
-            /                           # start get function with /
-            (?P<func>[^@]+)
-            @                           # start tag with @
-            (?P<tag>.+)
-        $                               # end task format
-    """
-    RE_TASK_FMT: Pattern = re.compile(
-        __re_task_fmt, MULTILINE | IGNORECASE | UNICODE | VERBOSE
-    )
+    RE_TASK_FMT: Pattern = RE_TASK_FMT
 
     @classmethod
     def finditer_caller(cls, value: str) -> Iterator[CallerRe]:
         """Generate CallerRe object that create from matching object that
         extract with re.finditer function.
 
-        :param value: (str) A string value that want to finditer with the caller
-            regular expression.
+        Args:
+            value: A string value that want to finditer with the caller
+                regular expression.
 
-        :rtype: Iterator[CallerRe]
+        Yields:
+            CallerRe: CallerRe objects created from regex matches.
         """
         for found in cls.RE_CALLER.finditer(value):
             yield CallerRe.from_regex(found)

ddeutil/workflow/api/__init__.py

@@ -1,3 +1,30 @@
+"""FastAPI Web Application for Workflow Management.
+
+This module provides a RESTful API interface for workflow orchestration using
+FastAPI. It enables remote workflow management, execution monitoring, and
+provides endpoints for workflow operations.
+
+The API supports:
+    - Workflow execution and management
+    - Job status monitoring
+    - Log streaming and access
+    - Result retrieval and analysis
+
+Example:
+    ```python
+    from ddeutil.workflow.api import app
+
+    # Run the API server
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)
+    ```
+
+Routes:
+    - /workflows: Workflow management endpoints
+    - /jobs: Job execution and monitoring
+    - /logs: Log access and streaming
+"""
+
 # ------------------------------------------------------------------------------
 # Copyright (c) 2022 Korawich Anuttra. All rights reserved.
 # Licensed under the MIT License. See LICENSE in the project root for
@@ -28,7 +55,17 @@ logger = logging.getLogger("uvicorn.error")
 
 @contextlib.asynccontextmanager
 async def lifespan(_: FastAPI) -> AsyncIterator[dict[str, list]]:
-    """Lifespan function for the FastAPI application."""
+    """FastAPI application lifespan management.
+
+    Manages the startup and shutdown lifecycle of the FastAPI application.
+    Currently yields an empty dictionary for future extension.
+
+    Args:
+        _: FastAPI application instance (unused)
+
+    Yields:
+        dict: Empty dictionary for future lifespan data
+    """
     yield {}
 
 
@@ -59,7 +96,20 @@ app.add_middleware(
 
 @app.get(path="/", response_class=UJSONResponse)
 async def health() -> UJSONResponse:
-    """Index view that not return any template without json status."""
+    """Health check endpoint for API status monitoring.
+
+    Provides a simple health check endpoint to verify the API is running
+    and responding correctly. Returns a JSON response with health status.
+
+    Returns:
+        UJSONResponse: JSON response confirming healthy API status
+
+    Example:
+        ```bash
+        curl http://localhost:8000/
+        # Returns: {"message": "Workflow already start up with healthy status."}
+        ```
+    """
     logger.info("[API]: Workflow API Application already running ...")
     return UJSONResponse(
         content={"message": "Workflow already start up with healthy status."},
@@ -78,7 +128,28 @@ async def validation_exception_handler(
     request: Request,
     exc: RequestValidationError,
 ) -> UJSONResponse:
-    """Error Handler for model validate does not valid."""
+    """Handle request validation errors from Pydantic models.
+
+    Provides standardized error responses for request validation failures,
+    including detailed error information for debugging and client feedback.
+
+    Args:
+        request: The FastAPI request object (unused)
+        exc: The validation exception containing error details
+
+    Returns:
+        UJSONResponse: Standardized error response with validation details
+
+    Example:
+        When a request fails validation:
+        ```json
+        {
+            "message": "Body does not parsing with model.",
+            "detail": [...],
+            "body": {...}
+        }
+        ```
+    """
     _ = request
     return UJSONResponse(
         status_code=st.HTTP_422_UNPROCESSABLE_ENTITY,

ddeutil/workflow/api/routes/job.py

@@ -8,72 +8,58 @@ from __future__ import annotations
 import logging
 from typing import Any, Optional
 
-from fastapi import APIRouter
+from fastapi import APIRouter, Body
 from fastapi import status as st
 from fastapi.responses import UJSONResponse
-from pydantic import BaseModel, Field
 
 from ...__types import DictData
 from ...errors import JobError
 from ...job import Job
-from ...result import Result
+from ...traces import Trace, get_trace
+from ...utils import gen_id
 
 logger = logging.getLogger("uvicorn.error")
 router = APIRouter(prefix="/job", tags=["job"])
 
 
-class ResultCreate(BaseModel):
-    """Create Result model for receive running IDs to create the Result
-    dataclass.
-    """
-
-    run_id: str = Field(description="A running ID.")
-    parent_run_id: Optional[str] = Field(
-        default=None, description="A parent running ID."
-    )
-
-
 @router.post(
     path="/execute/",
     response_class=UJSONResponse,
     status_code=st.HTTP_200_OK,
 )
 async def job_execute(
-    result: ResultCreate,
     job: Job,
     params: dict[str, Any],
-    extras: Optional[dict[str, Any]] = None,
+    run_id: str = Body(...),
+    extras: Optional[dict[str, Any]] = Body(default=None),
 ) -> UJSONResponse:
     """Execute job via RestAPI with execute route path."""
     logger.info("[API]: Start execute job ...")
-    rs: Result = Result(
-        run_id=result.run_id,
-        parent_run_id=result.parent_run_id,
-        extras=extras or {},
-    )
+    parent_run_id: str = run_id
+    run_id = gen_id(job.id, unique=True)
 
     if extras:
         job.extras = extras
 
+    trace: Trace = get_trace(
+        run_id, parent_run_id=parent_run_id, extras=job.extras
+    )
+
     context: DictData = {}
     try:
         job.set_outputs(
             job.execute(
                 params=params,
-                run_id=rs.run_id,
-                parent_run_id=rs.parent_run_id,
+                run_id=parent_run_id,
             ).context,
             to=context,
         )
     except JobError as err:
-        rs.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
+        trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
         return UJSONResponse(
             content={
                 "message": str(err),
-                "result": {
-                    "run_id": rs.run_id,
-                    "parent_run_id": rs.parent_run_id,
-                },
+                "run_id": parent_run_id,
                 "job": job.model_dump(
                     by_alias=True,
                     exclude_none=False,
@@ -88,7 +74,7 @@ async def job_execute(
     return UJSONResponse(
         content={
             "message": "Execute job via RestAPI successful.",
-            "result": {"run_id": rs.run_id, "parent_run_id": rs.parent_run_id},
+            "run_id": parent_run_id,
             "job": job.model_dump(
                 by_alias=True,
                 exclude_none=False,

ddeutil/workflow/api/routes/logs.py

@@ -10,7 +10,7 @@ from fastapi import APIRouter, Path, Query
 from fastapi import status as st
 from fastapi.responses import UJSONResponse
 
-from ...audits import get_audit
+from ...audits import get_audit_model
 from ...result import Result
 
 router = APIRouter(
@@ -86,11 +86,11 @@ async def get_trace_with_id(run_id: str):
 )
 async def get_audits():
     """Return all audit logs from the current audit log path that config with
-    `WORKFLOW_AUDIT_PATH` environment variable name.
+    `WORKFLOW_AUDIT_URL` environment variable name.
     """
     return {
         "message": "Getting audit logs",
-        "audits": list(get_audit().find_audits(name="demo")),
+        "audits": list(get_audit_model().find_audits(name="demo")),
     }
 
 
@@ -103,13 +103,13 @@ async def get_audits():
 )
 async def get_audit_with_workflow(workflow: str):
     """Return all audit logs with specific workflow name from the current audit
-    log path that config with `WORKFLOW_AUDIT_PATH` environment variable name.
+    log path that config with `WORKFLOW_AUDIT_URL` environment variable name.
 
     - **workflow**: A specific workflow name that want to find audit logs.
     """
     return {
         "message": f"Getting audit logs with workflow name {workflow}",
-        "audits": list(get_audit().find_audits(name="demo")),
+        "audits": list(get_audit_model().find_audits(name="demo")),
     }
 
 
@@ -125,7 +125,7 @@ async def get_audit_with_workflow_release(
     release: str = Path(...),
 ):
     """Return all audit logs with specific workflow name and release date from
-    the current audit log path that config with `WORKFLOW_AUDIT_PATH`
+    the current audit log path that config with `WORKFLOW_AUDIT_URL`
     environment variable name.
 
     - **workflow**: A specific workflow name that want to find audit logs.
@@ -136,7 +136,7 @@ async def get_audit_with_workflow_release(
             f"Getting audit logs with workflow name {workflow} and release "
             f"{release}"
         ),
-        "audits": list(get_audit().find_audits(name="demo")),
+        "audits": list(get_audit_model().find_audits(name="demo")),
     }
 
 
@@ -154,7 +154,7 @@ async def get_audit_with_workflow_release_run_id(
     workflow: str, release: str, run_id: str
 ):
     """Return all audit logs with specific workflow name and release date from
-    the current audit log path that config with `WORKFLOW_AUDIT_PATH`
+    the current audit log path that config with `WORKFLOW_AUDIT_URL`
     environment variable name.
 
     - **workflow**: A specific workflow name that want to find audit logs.
@@ -167,5 +167,5 @@ async def get_audit_with_workflow_release_run_id(
             f"Getting audit logs with workflow name {workflow}, release "
             f"{release}, and running ID {run_id}"
         ),
-        "audits": list(get_audit().find_audits(name="demo")),
+        "audits": list(get_audit_model().find_audits(name="demo")),
     }

ddeutil/workflow/api/routes/workflows.py

@@ -16,7 +16,7 @@ from fastapi.responses import UJSONResponse
 from pydantic import BaseModel
 
 from ...__types import DictData
-from ...audits import AuditModel, get_audit
+from ...audits import Audit, get_audit_model
 from ...conf import YamlParser
 from ...result import Result
 from ...workflow import Workflow
@@ -100,7 +100,7 @@ async def get_workflow_audits(name: str):
                     exclude_none=False,
                     exclude_unset=True,
                 )
-                for audit in get_audit().find_audits(name=name)
+                for audit in get_audit_model().find_audits(name=name)
             ],
         }
     except FileNotFoundError:
@@ -114,7 +114,7 @@ async def get_workflow_audits(name: str):
 async def get_workflow_release_audit(name: str, release: str):
     """Get Workflow audit log with an input release value."""
     try:
-        audit: AuditModel = get_audit().find_audit_with_release(
+        audit: Audit = get_audit_model().find_audit_with_release(
             name=name,
             release=datetime.strptime(release, "%Y%m%d%H%M%S"),
         )