ddeutil-workflow 0.0.81__py3-none-any.whl → 0.0.83__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1,2 @@
-__version__: str = "0.0.81"
+__version__: str = "0.0.83"
+__python_version__: str = "3.9"

ddeutil/workflow/__cron.py

@@ -715,7 +715,7 @@ class CronJob:
         self,
         date: Optional[datetime] = None,
         *,
-        tz: Optional[str] = None,
+        tz: Optional[Union[str, ZoneInfo]] = None,
     ) -> CronRunner:
         """Returns CronRunner instance that be datetime runner with this
         cronjob. It can use `next`, `prev`, or `reset` methods to generate

ddeutil/workflow/__init__.py

@@ -50,19 +50,37 @@ Note:
 from .__cron import CronRunner
 from .__types import DictData, DictStr, Matrix, Re, TupleStr
 from .audits import (
+    DRYRUN,
+    FORCE,
+    NORMAL,
+    RERUN,
     Audit,
-    FileAudit,
+    LocalFileAudit,
     get_audit,
 )
-from .conf import *
+from .conf import (
+    PREFIX,
+    CallerSecret,
+    Config,
+    YamlParser,
+    api_config,
+    config,
+    dynamic,
+    env,
+    pass_env,
+)
 from .errors import (
     BaseError,
+    EventError,
     JobCancelError,
     JobError,
     JobSkipError,
     ResultError,
     StageCancelError,
     StageError,
+    StageNestedCancelError,
+    StageNestedError,
+    StageNestedSkipError,
     StageSkipError,
     UtilError,
     WorkflowCancelError,
@@ -132,15 +150,11 @@ from .stages import (
     VirtualPyStage,
 )
 from .traces import (
-    TraceManager,
+    Trace,
     get_trace,
 )
 from .utils import *
 from .workflow import (
-    EVENT,
-    FORCE,
-    NORMAL,
-    RERUN,
     ReleaseType,
     Workflow,
 )

ddeutil/workflow/__main__.py

@@ -1,4 +1,283 @@
-from .cli import app
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from platform import python_version
+from textwrap import dedent
+from typing import Annotated, Any, Literal, Optional, Union
+
+import typer
+from pydantic import Field, TypeAdapter
+
+from .__about__ import __version__
+from .__types import DictData
+from .conf import config
+from .errors import JobError
+from .job import Job
+from .params import Param
+from .workflow import Workflow
+
+app = typer.Typer(pretty_exceptions_enable=True)
+
+
+@app.callback()
+def callback() -> None:
+    """Manage Workflow Orchestration CLI.
+
+    Use it with the interface workflow engine.
+    """
+
+
+@app.command()
+def version() -> None:
+    """Get the ddeutil-workflow package version."""
+    typer.echo(f"ddeutil-workflow=={__version__}")
+    typer.echo(f"python-version=={python_version()}")
+
+
+@app.command()
+def init() -> None:
+    """Initialize a Workflow structure on the current context."""
+    config.conf_path.mkdir(exist_ok=True)
+    (config.conf_path / ".confignore").touch()
+
+    conf_example_path: Path = config.conf_path / "examples"
+    conf_example_path.mkdir(exist_ok=True)
+
+    example_template: Path = conf_example_path / "wf_examples.yml"
+    example_template.write_text(
+        dedent(
+            """
+        # Example workflow template.
+        name: wf-example:
+        type: Workflow
+        desc: |
+          An example workflow template that provide the demo of workflow.
+        params:
+          name:
+              type: str
+              default: "World"
+        jobs:
+          first-job:
+            stages:
+
+              - name: "Hello Stage"
+                echo: "Start say hi to the console"
+
+              - name: "Call tasks"
+                uses: tasks/say-hello-func@example
+                with:
+                  name: ${{ params.name }}
+
+          second-job:
+
+              - name: "Hello Env"
+                echo: "Start say hi with ${ WORKFLOW_DEMO_HELLO }"
+        """
+        ).lstrip("\n")
+    )
+
+    if "." in config.registry_caller:
+        task_path = Path("./tasks")
+        task_path.mkdir(exist_ok=True)
+
+        dummy_tasks_path = task_path / "example.py"
+        dummy_tasks_path.write_text(
+            dedent(
+                """
+            from typing import Any, Optional
+
+            from ddeutil.workflow import Result, tag
+
+            @tag(name="example", alias="say-hello-func")
+            def hello_world_task(name: str, rs: Result, extras: Optional[dict[str, Any]] = None) -> dict[str, str]:
+                \"\"\"Logging hello task function\"\"\"
+                _extras = extras or {}
+                # NOTE: I will use custom newline logging if you pass `||`.
+                rs.trace.info(
+                    f"Hello, {name}||"
+                    f"> running ID: {rs.run_id}"
+                    f"> extras: {_extras}"
+                )
+                return {"name": name}
+            """
+            ).lstrip("\n")
+        )
+
+        init_path = task_path / "__init__.py"
+        init_path.write_text("from .example import hello_world_task\n")
+
+    dotenv_file = Path(".env")
+    mode: str = "a" if dotenv_file.exists() else "w"
+    with dotenv_file.open(mode=mode) as f:
+        f.write("\n# Workflow Environment Variables\n")
+        f.write(
+            "WORKFLOW_DEMO_HELLO=foo\n"
+            "WORKFLOW_CORE_DEBUG_MODE=true\n"
+            "WORKFLOW_LOG_TIMEZONE=Asia/Bangkok\n"
+            'WORKFLOW_LOG_TRACE_HANDLERS=\'[{"type": "console"}]\'\n'
+            'WORKFLOW_LOG_AUDIT_CONF=\'{"type": "file", "path": "./audits"}\''
+            "WORKFLOW_LOG_AUDIT_ENABLE_WRITE=true\n"
+        )
+
+    typer.echo("Starter command:")
+    typer.echo(
+        ">>> `source .env && workflow-cli workflows execute --name=wf-example`"
+    )
+
+
+@app.command(name="job")
+def execute_job(
+    params: Annotated[str, typer.Option(help="A job execute parameters")],
+    job: Annotated[str, typer.Option(help="A job model")],
+    run_id: Annotated[str, typer.Option(help="A running ID")],
+) -> None:
+    """Job execution on the local.
+
+    Example:
+        ... workflow-cli job --params \"{\\\"test\\\": 1}\"
+    """
+    try:
+        params_dict: dict[str, Any] = json.loads(params)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Params does not support format: {params!r}.") from e
+
+    try:
+        job_dict: dict[str, Any] = json.loads(job)
+        _job: Job = Job.model_validate(obj=job_dict)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Jobs does not support format: {job!r}.") from e
+
+    typer.echo(f"Job params: {params_dict}")
+    context: DictData = {}
+    try:
+        _job.set_outputs(
+            _job.execute(params=params_dict, run_id=run_id).context,
+            to=context,
+        )
+        typer.echo("[JOB]: Context result:")
+        typer.echo(json.dumps(context, default=str, indent=0))
+    except JobError as err:
+        typer.echo(f"[JOB]: {err.__class__.__name__}: {err}")
+
+
+@app.command()
+def api(
+    host: Annotated[str, typer.Option(help="A host url.")] = "0.0.0.0",
+    port: Annotated[int, typer.Option(help="A port url.")] = 80,
+    debug: Annotated[bool, typer.Option(help="A debug mode flag")] = True,
+    workers: Annotated[int, typer.Option(help="A worker number")] = None,
+    reload: Annotated[bool, typer.Option(help="A reload flag")] = False,
+) -> None:
+    """
+    Provision API application from the FastAPI.
+    """
+    import uvicorn
+
+    from .api import app as fastapp
+    from .api.log_conf import LOGGING_CONFIG
+
+    # LOGGING_CONFIG = {}
+
+    uvicorn.run(
+        fastapp,
+        host=host,
+        port=port,
+        log_config=uvicorn.config.LOGGING_CONFIG | LOGGING_CONFIG,
+        # NOTE: Logging level of uvicorn should be lowered case.
+        log_level=("debug" if debug else "info"),
+        workers=workers,
+        reload=reload,
+    )
+
+
+@app.command()
+def make(
+    name: Annotated[Path, typer.Argument()],
+) -> None:
+    """
+    Create Workflow YAML template.
+
+    :param name:
+    """
+    typer.echo(f"Start create YAML template filename: {name.resolve()}")
+
+
+workflow_app = typer.Typer()
+app.add_typer(workflow_app, name="workflows", help="An Only Workflow CLI.")
+
+
+@workflow_app.callback()
+def workflow_callback():
+    """Manage Only Workflow CLI."""
+
+
+@workflow_app.command(name="execute")
+def workflow_execute(
+    name: Annotated[
+        str,
+        typer.Option(help="A name of workflow template."),
+    ],
+    params: Annotated[
+        str,
+        typer.Option(help="A workflow execute parameters"),
+    ] = "{}",
+):
+    """Execute workflow by passing a workflow template name."""
+    try:
+        params_dict: dict[str, Any] = json.loads(params)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Params does not support format: {params!r}.") from e
+
+    typer.echo(f"Start execute workflow template: {name}")
+    typer.echo(f"... with params: {params_dict}")
+
+
+class WorkflowSchema(Workflow):
+    """Override workflow model fields for generate JSON schema file."""
+
+    type: Literal["Workflow"] = Field(
+        description="A type of workflow template that should be `Workflow`."
+    )
+    name: Optional[str] = Field(default=None, description="A workflow name.")
+    params: dict[str, Union[Param, str]] = Field(
+        default_factory=dict,
+        description="A parameters that need to use on this workflow.",
+    )
+
+
+@workflow_app.command(name="json-schema")
+def workflow_json_schema(
+    output: Annotated[
+        Path,
+        typer.Option(help="An output file to export the JSON schema."),
+    ] = Path("./json-schema.json"),
+) -> None:
+    """Generate JSON schema file from the Workflow model."""
+    template = dict[str, WorkflowSchema]
+    json_schema = TypeAdapter(template).json_schema(by_alias=True)
+    template_schema: dict[str, str] = {
+        "$schema": "http://json-schema.org/draft-07/schema#",
+        "title": "Workflow Configuration JSON Schema",
+        "version": __version__,
+    }
+    with open(output, mode="w", encoding="utf-8") as f:
+        json.dump(template_schema | json_schema, f, indent=2)
+
+
+log_app = typer.Typer()
+app.add_typer(log_app, name="logs", help="An Only Log CLI.")
+
+
+@log_app.callback()
+def log_callback():
+    """Manage Only Log CLI."""
+
 
 if __name__ == "__main__":
     app()

ddeutil/workflow/__types.py

@@ -16,17 +16,26 @@ from re import (
     Match,
     Pattern,
 )
-from typing import Any, Optional, TypedDict, Union
+from typing import Any, Optional, TypedDict, Union, cast
 
 from typing_extensions import Self
 
 StrOrNone = Optional[str]
 StrOrInt = Union[str, int]
 TupleStr = tuple[str, ...]
+ListStr = list[str]
+ListInt = list[int]
 DictData = dict[str, Any]
+DictRange = dict[int, Any]
 DictStr = dict[str, str]
 Matrix = dict[str, Union[list[str], list[int]]]
 
+
+def cast_dict(value: TypedDict[...]) -> DictData:
+    """Cast any TypedDict object to DictData type."""
+    return cast(DictData, value)
+
+
 # Pre-compile regex patterns for better performance
 _RE_CALLER_PATTERN = r"""
     \$                                                      # start with $

ddeutil/workflow/api/routes/job.py

@@ -15,7 +15,7 @@ from fastapi.responses import UJSONResponse
 from ...__types import DictData
 from ...errors import JobError
 from ...job import Job
-from ...traces import TraceManager, get_trace
+from ...traces import Trace, get_trace
 from ...utils import gen_id
 
 logger = logging.getLogger("uvicorn.error")
@@ -41,7 +41,7 @@ async def job_execute(
     if extras:
         job.extras = extras
 
-    trace: TraceManager = get_trace(
+    trace: Trace = get_trace(
         run_id, parent_run_id=parent_run_id, extras=job.extras
     )
 

ddeutil/workflow/api/routes/logs.py

@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""This route include audit and trace log paths."""
+"""This route include audit log path."""
 from __future__ import annotations
 
 from fastapi import APIRouter, Path, Query
@@ -11,7 +11,6 @@ from fastapi import status as st
 from fastapi.responses import UJSONResponse
 
 from ...audits import get_audit
-from ...result import Result
 
 router = APIRouter(
     prefix="/logs",
@@ -20,63 +19,6 @@ router = APIRouter(
 )
 
 
-@router.get(
-    path="/traces/",
-    response_class=UJSONResponse,
-    status_code=st.HTTP_200_OK,
-    summary="Read all trace logs.",
-    tags=["trace"],
-)
-async def get_traces(
-    offset: int = Query(default=0, gt=0),
-    limit: int = Query(default=100, gt=0),
-):
-    """Return all trace logs from the current trace log path that config with
-    `WORKFLOW_LOG_PATH` environment variable name.
-    """
-    result = Result()
-    return {
-        "message": (
-            f"Getting trace logs with offset: {offset} and limit: {limit}"
-        ),
-        "traces": [
-            trace.model_dump(
-                by_alias=True,
-                exclude_none=True,
-                exclude_unset=True,
-            )
-            for trace in result.trace.find_traces()
-        ],
-    }
-
-
-@router.get(
-    path="/traces/{run_id}",
-    response_class=UJSONResponse,
-    status_code=st.HTTP_200_OK,
-    summary="Read trace log with specific running ID.",
-    tags=["trace"],
-)
-async def get_trace_with_id(run_id: str):
-    """Return trace log with specific running ID from the current trace log path
-    that config with `WORKFLOW_LOG_PATH` environment variable name.
-
-    - **run_id**: A running ID that want to search a trace log from the log
-        path.
-    """
-    result = Result()
-    return {
-        "message": f"Getting trace log with specific running ID: {run_id}",
-        "trace": (
-            result.trace.find_trace_with_id(run_id).model_dump(
-                by_alias=True,
-                exclude_none=True,
-                exclude_unset=True,
-            )
-        ),
-    }
-
-
 @router.get(
     path="/audits/",
     response_class=UJSONResponse,
@@ -84,12 +26,17 @@ async def get_trace_with_id(run_id: str):
     summary="Read all audit logs.",
     tags=["audit"],
 )
-async def get_audits():
+async def get_audits(
+    offset: int = Query(default=0, gt=0),
+    limit: int = Query(default=100, gt=0),
+):
     """Return all audit logs from the current audit log path that config with
     `WORKFLOW_AUDIT_URL` environment variable name.
     """
     return {
-        "message": "Getting audit logs",
+        "message": (
+            f"Getting audit logs with offset: {offset} and limit: {limit}",
+        ),
         "audits": list(get_audit().find_audits(name="demo")),
     }