ddeutil-workflow 0.0.81__py3-none-any.whl → 0.0.82__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1,2 @@
-__version__: str = "0.0.81"
+__version__: str = "0.0.82"
+__python_version__: str = "3.9"

ddeutil/workflow/__init__.py

@@ -50,11 +50,25 @@ Note:
 from .__cron import CronRunner
 from .__types import DictData, DictStr, Matrix, Re, TupleStr
 from .audits import (
+    EVENT,
+    FORCE,
+    NORMAL,
+    RERUN,
     Audit,
     FileAudit,
     get_audit,
 )
-from .conf import *
+from .conf import (
+    PREFIX,
+    CallerSecret,
+    Config,
+    YamlParser,
+    api_config,
+    config,
+    dynamic,
+    env,
+    pass_env,
+)
 from .errors import (
     BaseError,
     JobCancelError,
@@ -63,6 +77,9 @@ from .errors import (
     ResultError,
     StageCancelError,
     StageError,
+    StageNestedCancelError,
+    StageNestedError,
+    StageNestedSkipError,
     StageSkipError,
     UtilError,
     WorkflowCancelError,
@@ -132,15 +149,11 @@ from .stages import (
     VirtualPyStage,
 )
 from .traces import (
-    TraceManager,
+    Trace,
     get_trace,
 )
 from .utils import *
 from .workflow import (
-    EVENT,
-    FORCE,
-    NORMAL,
-    RERUN,
     ReleaseType,
     Workflow,
 )

ddeutil/workflow/__main__.py

@@ -1,4 +1,283 @@
-from .cli import app
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from platform import python_version
+from textwrap import dedent
+from typing import Annotated, Any, Literal, Optional, Union
+
+import typer
+from pydantic import Field, TypeAdapter
+
+from .__about__ import __version__
+from .__types import DictData
+from .conf import config
+from .errors import JobError
+from .job import Job
+from .params import Param
+from .workflow import Workflow
+
+app = typer.Typer(pretty_exceptions_enable=True)
+
+
+@app.callback()
+def callback() -> None:
+    """Manage Workflow Orchestration CLI.
+
+    Use it with the interface workflow engine.
+    """
+
+
+@app.command()
+def version() -> None:
+    """Get the ddeutil-workflow package version."""
+    typer.echo(f"ddeutil-workflow=={__version__}")
+    typer.echo(f"python-version=={python_version()}")
+
+
+@app.command()
+def init() -> None:
+    """Initialize a Workflow structure on the current context."""
+    config.conf_path.mkdir(exist_ok=True)
+    (config.conf_path / ".confignore").touch()
+
+    conf_example_path: Path = config.conf_path / "examples"
+    conf_example_path.mkdir(exist_ok=True)
+
+    example_template: Path = conf_example_path / "wf_examples.yml"
+    example_template.write_text(
+        dedent(
+            """
+        # Example workflow template.
+        name: wf-example:
+        type: Workflow
+        desc: |
+          An example workflow template that provide the demo of workflow.
+        params:
+          name:
+              type: str
+              default: "World"
+        jobs:
+          first-job:
+            stages:
+
+              - name: "Hello Stage"
+                echo: "Start say hi to the console"
+
+              - name: "Call tasks"
+                uses: tasks/say-hello-func@example
+                with:
+                  name: ${{ params.name }}
+
+          second-job:
+
+              - name: "Hello Env"
+                echo: "Start say hi with ${ WORKFLOW_DEMO_HELLO }"
+        """
+        ).lstrip("\n")
+    )
+
+    if "." in config.registry_caller:
+        task_path = Path("./tasks")
+        task_path.mkdir(exist_ok=True)
+
+        dummy_tasks_path = task_path / "example.py"
+        dummy_tasks_path.write_text(
+            dedent(
+                """
+            from typing import Any, Optional
+
+            from ddeutil.workflow import Result, tag
+
+            @tag(name="example", alias="say-hello-func")
+            def hello_world_task(name: str, rs: Result, extras: Optional[dict[str, Any]] = None) -> dict[str, str]:
+                \"\"\"Logging hello task function\"\"\"
+                _extras = extras or {}
+                # NOTE: I will use custom newline logging if you pass `||`.
+                rs.trace.info(
+                    f"Hello, {name}||"
+                    f"> running ID: {rs.run_id}"
+                    f"> extras: {_extras}"
+                )
+                return {"name": name}
+            """
+            ).lstrip("\n")
+        )
+
+        init_path = task_path / "__init__.py"
+        init_path.write_text("from .example import hello_world_task\n")
+
+    dotenv_file = Path(".env")
+    mode: str = "a" if dotenv_file.exists() else "w"
+    with dotenv_file.open(mode=mode) as f:
+        f.write("\n# Workflow Environment Variables\n")
+        f.write(
+            "WORKFLOW_DEMO_HELLO=foo\n"
+            "WORKFLOW_CORE_DEBUG_MODE=true\n"
+            "WORKFLOW_LOG_TIMEZONE=Asia/Bangkok\n"
+            'WORKFLOW_LOG_TRACE_HANDLERS=\'[{"type": "console"}]\'\n'
+            'WORKFLOW_LOG_AUDIT_CONF=\'{"type": "file", "path": "./audits"}\''
+            "WORKFLOW_LOG_AUDIT_ENABLE_WRITE=true\n"
+        )
+
+    typer.echo("Starter command:")
+    typer.echo(
+        ">>> `source .env && workflow-cli workflows execute --name=wf-example`"
+    )
+
+
+@app.command(name="job")
+def execute_job(
+    params: Annotated[str, typer.Option(help="A job execute parameters")],
+    job: Annotated[str, typer.Option(help="A job model")],
+    run_id: Annotated[str, typer.Option(help="A running ID")],
+) -> None:
+    """Job execution on the local.
+
+    Example:
+        ... workflow-cli job --params \"{\\\"test\\\": 1}\"
+    """
+    try:
+        params_dict: dict[str, Any] = json.loads(params)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Params does not support format: {params!r}.") from e
+
+    try:
+        job_dict: dict[str, Any] = json.loads(job)
+        _job: Job = Job.model_validate(obj=job_dict)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Jobs does not support format: {job!r}.") from e
+
+    typer.echo(f"Job params: {params_dict}")
+    context: DictData = {}
+    try:
+        _job.set_outputs(
+            _job.execute(params=params_dict, run_id=run_id).context,
+            to=context,
+        )
+        typer.echo("[JOB]: Context result:")
+        typer.echo(json.dumps(context, default=str, indent=0))
+    except JobError as err:
+        typer.echo(f"[JOB]: {err.__class__.__name__}: {err}")
+
+
+@app.command()
+def api(
+    host: Annotated[str, typer.Option(help="A host url.")] = "0.0.0.0",
+    port: Annotated[int, typer.Option(help="A port url.")] = 80,
+    debug: Annotated[bool, typer.Option(help="A debug mode flag")] = True,
+    workers: Annotated[int, typer.Option(help="A worker number")] = None,
+    reload: Annotated[bool, typer.Option(help="A reload flag")] = False,
+) -> None:
+    """
+    Provision API application from the FastAPI.
+    """
+    import uvicorn
+
+    from .api import app as fastapp
+    from .api.log_conf import LOGGING_CONFIG
+
+    # LOGGING_CONFIG = {}
+
+    uvicorn.run(
+        fastapp,
+        host=host,
+        port=port,
+        log_config=uvicorn.config.LOGGING_CONFIG | LOGGING_CONFIG,
+        # NOTE: Logging level of uvicorn should be lowered case.
+        log_level=("debug" if debug else "info"),
+        workers=workers,
+        reload=reload,
+    )
+
+
+@app.command()
+def make(
+    name: Annotated[Path, typer.Argument()],
+) -> None:
+    """
+    Create Workflow YAML template.
+
+    :param name:
+    """
+    typer.echo(f"Start create YAML template filename: {name.resolve()}")
+
+
+workflow_app = typer.Typer()
+app.add_typer(workflow_app, name="workflows", help="An Only Workflow CLI.")
+
+
+@workflow_app.callback()
+def workflow_callback():
+    """Manage Only Workflow CLI."""
+
+
+@workflow_app.command(name="execute")
+def workflow_execute(
+    name: Annotated[
+        str,
+        typer.Option(help="A name of workflow template."),
+    ],
+    params: Annotated[
+        str,
+        typer.Option(help="A workflow execute parameters"),
+    ] = "{}",
+):
+    """Execute workflow by passing a workflow template name."""
+    try:
+        params_dict: dict[str, Any] = json.loads(params)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Params does not support format: {params!r}.") from e
+
+    typer.echo(f"Start execute workflow template: {name}")
+    typer.echo(f"... with params: {params_dict}")
+
+
+class WorkflowSchema(Workflow):
+    """Override workflow model fields for generate JSON schema file."""
+
+    type: Literal["Workflow"] = Field(
+        description="A type of workflow template that should be `Workflow`."
+    )
+    name: Optional[str] = Field(default=None, description="A workflow name.")
+    params: dict[str, Union[Param, str]] = Field(
+        default_factory=dict,
+        description="A parameters that need to use on this workflow.",
+    )
+
+
+@workflow_app.command(name="json-schema")
+def workflow_json_schema(
+    output: Annotated[
+        Path,
+        typer.Option(help="An output file to export the JSON schema."),
+    ] = Path("./json-schema.json"),
+) -> None:
+    """Generate JSON schema file from the Workflow model."""
+    template = dict[str, WorkflowSchema]
+    json_schema = TypeAdapter(template).json_schema(by_alias=True)
+    template_schema: dict[str, str] = {
+        "$schema": "http://json-schema.org/draft-07/schema#",
+        "title": "Workflow Configuration JSON Schema",
+        "version": __version__,
+    }
+    with open(output, mode="w", encoding="utf-8") as f:
+        json.dump(template_schema | json_schema, f, indent=2)
+
+
+log_app = typer.Typer()
+app.add_typer(log_app, name="logs", help="An Only Log CLI.")
+
+
+@log_app.callback()
+def log_callback():
+    """Manage Only Log CLI."""
+
 
 if __name__ == "__main__":
     app()

ddeutil/workflow/api/routes/job.py

@@ -15,7 +15,7 @@ from fastapi.responses import UJSONResponse
 from ...__types import DictData
 from ...errors import JobError
 from ...job import Job
-from ...traces import TraceManager, get_trace
+from ...traces import Trace, get_trace
 from ...utils import gen_id
 
 logger = logging.getLogger("uvicorn.error")
@@ -41,7 +41,7 @@ async def job_execute(
     if extras:
         job.extras = extras
 
-    trace: TraceManager = get_trace(
+    trace: Trace = get_trace(
         run_id, parent_run_id=parent_run_id, extras=job.extras
     )
 

ddeutil/workflow/api/routes/logs.py

@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""This route include audit and trace log paths."""
+"""This route include audit log path."""
 from __future__ import annotations
 
 from fastapi import APIRouter, Path, Query
@@ -11,7 +11,6 @@ from fastapi import status as st
 from fastapi.responses import UJSONResponse
 
 from ...audits import get_audit
-from ...result import Result
 
 router = APIRouter(
     prefix="/logs",
@@ -20,63 +19,6 @@ router = APIRouter(
 )
 
 
-@router.get(
-    path="/traces/",
-    response_class=UJSONResponse,
-    status_code=st.HTTP_200_OK,
-    summary="Read all trace logs.",
-    tags=["trace"],
-)
-async def get_traces(
-    offset: int = Query(default=0, gt=0),
-    limit: int = Query(default=100, gt=0),
-):
-    """Return all trace logs from the current trace log path that config with
-    `WORKFLOW_LOG_PATH` environment variable name.
-    """
-    result = Result()
-    return {
-        "message": (
-            f"Getting trace logs with offset: {offset} and limit: {limit}"
-        ),
-        "traces": [
-            trace.model_dump(
-                by_alias=True,
-                exclude_none=True,
-                exclude_unset=True,
-            )
-            for trace in result.trace.find_traces()
-        ],
-    }
-
-
-@router.get(
-    path="/traces/{run_id}",
-    response_class=UJSONResponse,
-    status_code=st.HTTP_200_OK,
-    summary="Read trace log with specific running ID.",
-    tags=["trace"],
-)
-async def get_trace_with_id(run_id: str):
-    """Return trace log with specific running ID from the current trace log path
-    that config with `WORKFLOW_LOG_PATH` environment variable name.
-
-    - **run_id**: A running ID that want to search a trace log from the log
-        path.
-    """
-    result = Result()
-    return {
-        "message": f"Getting trace log with specific running ID: {run_id}",
-        "trace": (
-            result.trace.find_trace_with_id(run_id).model_dump(
-                by_alias=True,
-                exclude_none=True,
-                exclude_unset=True,
-            )
-        ),
-    }
-
-
 @router.get(
     path="/audits/",
     response_class=UJSONResponse,
@@ -84,12 +26,17 @@ async def get_trace_with_id(run_id: str):
     summary="Read all audit logs.",
     tags=["audit"],
 )
-async def get_audits():
+async def get_audits(
+    offset: int = Query(default=0, gt=0),
+    limit: int = Query(default=100, gt=0),
+):
     """Return all audit logs from the current audit log path that config with
     `WORKFLOW_AUDIT_URL` environment variable name.
     """
     return {
-        "message": "Getting audit logs",
+        "message": (
+            f"Getting audit logs with offset: {offset} and limit: {limit}",
+        ),
         "audits": list(get_audit().find_audits(name="demo")),
     }
 

ddeutil/workflow/audits.py

@@ -49,33 +49,65 @@ import zlib
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from datetime import datetime, timedelta
+from enum import Enum
 from pathlib import Path
 from typing import Annotated, Any, ClassVar, Literal, Optional, Union
 from urllib.parse import ParseResult, urlparse
 
-from pydantic import BaseModel, Field, TypeAdapter
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
 from .__types import DictData
 from .conf import dynamic
-from .traces import TraceManager, get_trace, set_logging
+from .traces import Trace, get_trace, set_logging
 
 logger = logging.getLogger("ddeutil.workflow")
 
 
+class ReleaseType(str, Enum):
+    """Release type enumeration for workflow execution modes.
+
+    This enum defines the different types of workflow releases that can be
+    triggered, each with specific behavior and use cases.
+
+    Attributes:
+        NORMAL: Standard workflow release execution
+        RERUN: Re-execution of previously failed workflow
+        EVENT: Event-triggered workflow execution
+        FORCE: Forced execution bypassing normal conditions
+    """
+
+    NORMAL = "normal"
+    RERUN = "rerun"
+    EVENT = "event"
+    FORCE = "force"
+
+
+NORMAL = ReleaseType.NORMAL
+RERUN = ReleaseType.RERUN
+EVENT = ReleaseType.EVENT
+FORCE = ReleaseType.FORCE
+
+
 class AuditData(BaseModel):
+    """Audit Data model."""
+
+    model_config = ConfigDict(use_enum_values=True)
+
     name: str = Field(description="A workflow name.")
     release: datetime = Field(description="A release datetime.")
-    type: str = Field(description="A running type before logging.")
+    type: ReleaseType = Field(
+        default=NORMAL, description="A running type before logging."
+    )
     context: DictData = Field(
         default_factory=dict,
         description="A context that receive from a workflow execution result.",
     )
+    run_id: str = Field(description="A running ID")
     parent_run_id: Optional[str] = Field(
         default=None, description="A parent running ID."
     )
-    run_id: str = Field(description="A running ID")
     runs_metadata: DictData = Field(
         default_factory=dict,
         description="A runs metadata that will use to tracking this audit log.",
@@ -122,7 +154,7 @@ class BaseAudit(BaseModel, ABC):
     @abstractmethod
     def is_pointed(
         self,
-        data: AuditData,
+        data: Any,
         *,
         extras: Optional[DictData] = None,
     ) -> bool:
@@ -328,21 +360,21 @@ class FileAudit(BaseAudit):
             return AuditData.model_validate(obj=json.load(f))
 
     def is_pointed(
-        self, data: AuditData, *, extras: Optional[DictData] = None
+        self,
+        data: Any,
+        *,
+        extras: Optional[DictData] = None,
     ) -> bool:
         """Check if the release log already exists at the destination log path.
 
         Args:
-            data: The workflow name.
+            data (str):
             extras: Optional extra parameters to override core config.
 
         Returns:
             bool: True if the release log exists, False otherwise.
         """
-        # NOTE: Return False if enable writing log flag does not set.
-        if not dynamic("enable_write_audit", extras=extras):
-            return False
-        return self.pointer(data).exists()
+        return self.pointer(AuditData.model_validate(data)).exists()
 
     def pointer(self, data: AuditData) -> Path:
         """Return release directory path generated from model data.
@@ -365,7 +397,7 @@ class FileAudit(BaseAudit):
             Self: The audit instance after saving.
         """
         audit = AuditData.model_validate(data)
-        trace: TraceManager = get_trace(
+        trace: Trace = get_trace(
             audit.run_id,
             parent_run_id=audit.parent_run_id,
             extras=self.extras,
@@ -655,7 +687,7 @@ class SQLiteAudit(BaseAudit):  # pragma: no cov
             ValueError: If SQLite database is not properly configured.
         """
         audit = AuditData.model_validate(data)
-        trace: TraceManager = get_trace(
+        trace: Trace = get_trace(
             audit.run_id,
             parent_run_id=audit.parent_run_id,
             extras=self.extras,
@@ -748,10 +780,7 @@ Audit = Annotated[
 ]
 
 
-def get_audit(
-    *,
-    extras: Optional[DictData] = None,
-) -> Audit:  # pragma: no cov
+def get_audit(extras: Optional[DictData] = None) -> Audit:  # pragma: no cov
     """Get an audit model dynamically based on the config audit path value.
 
     Args: