ddeutil-workflow 0.0.34__py3-none-any.whl → 0.0.35__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.34"
+__version__: str = "0.0.35"

ddeutil/workflow/__init__.py

@@ -9,7 +9,7 @@ from .audit import (
     Audit,
     get_audit,
 )
-from .call import (
+from .caller import (
     ReturnTagFunc,
     TagFunc,
     extract_call,
@@ -39,6 +39,11 @@ from .job import (
     Job,
     Strategy,
 )
+from .logs import (
+    TraceLog,
+    get_dt_tznow,
+    get_trace,
+)
 from .params import (
     ChoiceParam,
     DatetimeParam,
@@ -49,9 +54,7 @@ from .params import (
 from .result import (
     Result,
     Status,
-    TraceLog,
     default_gen_id,
-    get_dt_tznow,
 )
 from .scheduler import (
     Schedule,

ddeutil/workflow/api/api.py

@@ -20,6 +20,7 @@ from ..conf import config, get_logger
 from ..scheduler import ReleaseThread, ReleaseThreads
 from ..workflow import ReleaseQueue, WorkflowTask
 from .repeat import repeat_at
+from .routes import log
 
 load_dotenv()
 logger = get_logger("ddeutil.workflow")
@@ -77,22 +78,26 @@ async def health():
     return {"message": "Workflow API already start up"}
 
 
-# NOTE: Enable the workflow route.
+# NOTE Add the logs route by default.
+app.include_router(log, prefix=config.prefix_path)
+
+
+# NOTE: Enable the workflows route.
 if config.enable_route_workflow:
-    from .route import workflow_route
+    from .routes import workflow
 
-    app.include_router(workflow_route, prefix=config.prefix_path)
+    app.include_router(workflow, prefix=config.prefix_path)
 
 
-# NOTE: Enable the schedule route.
+# NOTE: Enable the schedules route.
 if config.enable_route_schedule:
     from ..audit import get_audit
     from ..scheduler import schedule_task
-    from .route import schedule_route
+    from .routes import schedule
 
-    app.include_router(schedule_route, prefix=config.prefix_path)
+    app.include_router(schedule, prefix=config.prefix_path)
 
-    @schedule_route.on_event("startup")
+    @schedule.on_event("startup")
     @repeat_at(cron="* * * * *", delay=2)
     def scheduler_listener():
         """Schedule broker every minute at 02 second."""
@@ -109,7 +114,7 @@ if config.enable_route_schedule:
                 log=get_audit(),
             )
 
-    @schedule_route.on_event("startup")
+    @schedule.on_event("startup")
     @repeat_at(cron="*/5 * * * *", delay=10)
     def monitoring():
         logger.debug("[MONITOR]: Start monitoring threading.")

ddeutil/workflow/api/routes/__init__.py

@@ -0,0 +1,8 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from .logs import log_route as log
+from .schedules import schedule_route as schedule
+from .workflows import workflow_route as workflow

ddeutil/workflow/api/routes/logs.py

@@ -0,0 +1,36 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+from fastapi import APIRouter
+from fastapi.responses import UJSONResponse
+
+from ...conf import get_logger
+from ...logs import get_trace_obj
+
+logger = get_logger("ddeutil.workflow")
+
+
+# NOTE: Start create the schedule routes.
+#
+log_route = APIRouter(
+    prefix="/logs",
+    tags=["logs"],
+    default_response_class=UJSONResponse,
+)
+
+
+@log_route.get(path="/")
+async def get_logs():
+    return {
+        "message": "Getting logs",
+        "audits": list(get_trace_obj().find_logs()),
+    }
+
+
+@log_route.get(path="/{run_id}")
+async def get_log_with_run_id(run_id: str):
+    return get_trace_obj().find_log_with_id(run_id)

ddeutil/workflow/api/{route.py → routes/schedules.py}

@@ -6,30 +6,17 @@
 from __future__ import annotations
 
 import copy
-from dataclasses import asdict
 from datetime import datetime, timedelta
-from typing import Any
 
 from fastapi import APIRouter, HTTPException, Request
 from fastapi import status as st
 from fastapi.responses import UJSONResponse
-from pydantic import BaseModel
 
-from ..__types import DictData
-from ..audit import Audit, get_audit
-from ..conf import Loader, config, get_logger
-from ..result import Result
-from ..scheduler import Schedule
-from ..workflow import Workflow
+from ...conf import config, get_logger
+from ...scheduler import Schedule
 
 logger = get_logger("ddeutil.workflow")
 
-workflow_route = APIRouter(
-    prefix="/workflows",
-    tags=["workflows"],
-    default_response_class=UJSONResponse,
-)
-
 schedule_route = APIRouter(
     prefix="/schedules",
     tags=["schedules"],
@@ -37,122 +24,6 @@ schedule_route = APIRouter(
 )
 
 
-@workflow_route.get(path="/")
-async def get_workflows() -> DictData:
-    """Return all workflow workflows that exists in config path."""
-    workflows: DictData = dict(Loader.finds(Workflow))
-    return {
-        "message": f"Getting all workflows: {len(workflows)}",
-        "count": len(workflows),
-        "workflows": workflows,
-    }
-
-
-@workflow_route.get(path="/{name}")
-async def get_workflow_by_name(name: str) -> DictData:
-    """Return model of workflow that passing an input workflow name."""
-    try:
-        workflow: Workflow = Workflow.from_loader(name=name, externals={})
-    except ValueError as err:
-        logger.exception(err)
-        raise HTTPException(
-            status_code=st.HTTP_404_NOT_FOUND,
-            detail=(
-                f"Workflow workflow name: {name!r} does not found in /conf path"
-            ),
-        ) from None
-    return workflow.model_dump(
-        by_alias=True,
-        exclude_none=True,
-        exclude_unset=True,
-        exclude_defaults=True,
-    )
-
-
-class ExecutePayload(BaseModel):
-    params: dict[str, Any]
-
-
-@workflow_route.post(path="/{name}/execute", status_code=st.HTTP_202_ACCEPTED)
-async def execute_workflow(name: str, payload: ExecutePayload) -> DictData:
-    """Return model of workflow that passing an input workflow name."""
-    try:
-        workflow: Workflow = Workflow.from_loader(name=name, externals={})
-    except ValueError:
-        raise HTTPException(
-            status_code=st.HTTP_404_NOT_FOUND,
-            detail=(
-                f"Workflow workflow name: {name!r} does not found in /conf path"
-            ),
-        ) from None
-
-    # NOTE: Start execute manually
-    try:
-        result: Result = workflow.execute(params=payload.params)
-    except Exception as err:
-        raise HTTPException(
-            status_code=st.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"{type(err)}: {err}",
-        ) from None
-
-    return asdict(result)
-
-
-@workflow_route.get(path="/{name}/logs")
-async def get_workflow_logs(name: str):
-    try:
-        return {
-            "message": f"Getting workflow {name!r} logs",
-            "logs": [
-                log.model_dump(
-                    by_alias=True,
-                    exclude_none=True,
-                    exclude_unset=True,
-                    exclude_defaults=True,
-                )
-                for log in get_audit().find_audits(name=name)
-            ],
-        }
-    except FileNotFoundError:
-        raise HTTPException(
-            status_code=st.HTTP_404_NOT_FOUND,
-            detail=f"Does not found log for workflow {name!r}",
-        ) from None
-
-
-@workflow_route.get(path="/{name}/logs/{release}")
-async def get_workflow_release_log(name: str, release: str):
-    try:
-        log: Audit = get_audit().find_audit_with_release(
-            name=name, release=datetime.strptime(release, "%Y%m%d%H%M%S")
-        )
-    except FileNotFoundError:
-        raise HTTPException(
-            status_code=st.HTTP_404_NOT_FOUND,
-            detail=(
-                f"Does not found log for workflow {name!r} "
-                f"with release {release!r}"
-            ),
-        ) from None
-    return {
-        "message": f"Getting workflow {name!r} log in release {release}",
-        "log": log.model_dump(
-            by_alias=True,
-            exclude_none=True,
-            exclude_unset=True,
-            exclude_defaults=True,
-        ),
-    }
-
-
-@workflow_route.delete(
-    path="/{name}/logs/{release}",
-    status_code=st.HTTP_204_NO_CONTENT,
-)
-async def del_workflow_release_log(name: str, release: str):
-    return {"message": f"Deleted workflow {name!r} log in release {release}"}
-
-
 @schedule_route.get(path="/{name}")
 async def get_schedules(name: str):
     try:

ddeutil/workflow/api/routes/workflows.py

@@ -0,0 +1,137 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+from dataclasses import asdict
+from datetime import datetime
+from typing import Any
+
+from fastapi import APIRouter, HTTPException
+from fastapi import status as st
+from fastapi.responses import UJSONResponse
+from pydantic import BaseModel
+
+from ...__types import DictData
+from ...audit import Audit, get_audit
+from ...conf import Loader, get_logger
+from ...result import Result
+from ...workflow import Workflow
+
+logger = get_logger("ddeutil.workflow")
+
+workflow_route = APIRouter(
+    prefix="/workflows",
+    tags=["workflows"],
+    default_response_class=UJSONResponse,
+)
+
+
+@workflow_route.get(path="/")
+async def get_workflows() -> DictData:
+    """Return all workflow workflows that exists in config path."""
+    workflows: DictData = dict(Loader.finds(Workflow))
+    return {
+        "message": f"Getting all workflows: {len(workflows)}",
+        "count": len(workflows),
+        "workflows": workflows,
+    }
+
+
+@workflow_route.get(path="/{name}")
+async def get_workflow_by_name(name: str) -> DictData:
+    """Return model of workflow that passing an input workflow name."""
+    try:
+        workflow: Workflow = Workflow.from_loader(name=name, externals={})
+    except ValueError as err:
+        logger.exception(err)
+        raise HTTPException(
+            status_code=st.HTTP_404_NOT_FOUND,
+            detail=(
+                f"Workflow workflow name: {name!r} does not found in /conf path"
+            ),
+        ) from None
+    return workflow.model_dump(
+        by_alias=True,
+        exclude_none=True,
+        exclude_unset=True,
+        exclude_defaults=True,
+    )
+
+
+class ExecutePayload(BaseModel):
+    params: dict[str, Any]
+
+
+@workflow_route.post(path="/{name}/execute", status_code=st.HTTP_202_ACCEPTED)
+async def execute_workflow(name: str, payload: ExecutePayload) -> DictData:
+    """Return model of workflow that passing an input workflow name."""
+    try:
+        workflow: Workflow = Workflow.from_loader(name=name, externals={})
+    except ValueError:
+        raise HTTPException(
+            status_code=st.HTTP_404_NOT_FOUND,
+            detail=(
+                f"Workflow workflow name: {name!r} does not found in /conf path"
+            ),
+        ) from None
+
+    # NOTE: Start execute manually
+    try:
+        result: Result = workflow.execute(params=payload.params)
+    except Exception as err:
+        raise HTTPException(
+            status_code=st.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"{type(err)}: {err}",
+        ) from None
+
+    return asdict(result)
+
+
+@workflow_route.get(path="/{name}/audits")
+async def get_workflow_audits(name: str):
+    try:
+        return {
+            "message": f"Getting workflow {name!r} audits",
+            "audits": [
+                audit.model_dump(
+                    by_alias=True,
+                    exclude_none=True,
+                    exclude_unset=True,
+                    exclude_defaults=True,
+                )
+                for audit in get_audit().find_audits(name=name)
+            ],
+        }
+    except FileNotFoundError:
+        raise HTTPException(
+            status_code=st.HTTP_404_NOT_FOUND,
+            detail=f"Does not found audit for workflow {name!r}",
+        ) from None
+
+
+@workflow_route.get(path="/{name}/audits/{release}")
+async def get_workflow_release_audit(name: str, release: str):
+    try:
+        audit: Audit = get_audit().find_audit_with_release(
+            name=name, release=datetime.strptime(release, "%Y%m%d%H%M%S")
+        )
+    except FileNotFoundError:
+        raise HTTPException(
+            status_code=st.HTTP_404_NOT_FOUND,
+            detail=(
+                f"Does not found audit for workflow {name!r} "
+                f"with release {release!r}"
+            ),
+        ) from None
+    return {
+        "message": f"Getting workflow {name!r} audit in release {release}",
+        "audit": audit.model_dump(
+            by_alias=True,
+            exclude_none=True,
+            exclude_unset=True,
+            exclude_defaults=True,
+        ),
+    }

ddeutil/workflow/audit.py

@@ -20,7 +20,7 @@ from typing_extensions import Self
 
 from .__types import DictData, TupleStr
 from .conf import config
-from .result import TraceLog
+from .logs import TraceLog, get_trace
 
 __all__: TupleStr = (
     "get_audit",
@@ -174,7 +174,7 @@ class FileAudit(BaseAudit):
 
         :rtype: Self
         """
-        trace: TraceLog = TraceLog(self.run_id, self.parent_run_id)
+        trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
 
         # NOTE: Check environ variable was set for real writing.
         if not config.enable_write_audit:
@@ -214,7 +214,7 @@ class SQLiteAudit(BaseAudit):  # pragma: no cov
         """Save logging data that receive a context data from a workflow
         execution result.
         """
-        trace: TraceLog = TraceLog(self.run_id, self.parent_run_id)
+        trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
 
         # NOTE: Check environ variable was set for real writing.
         if not config.enable_write_audit:

ddeutil/workflow/{call.py → caller.py}

@@ -60,7 +60,7 @@ def tag(
 
         @wraps(func)
         def wrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
-            # NOTE: Able to do anything before calling call function.
+            # NOTE: Able to do anything before calling the call function.
             return func(*args, **kwargs)
 
         return wrapped
@@ -150,7 +150,7 @@ def extract_call(call: str) -> Callable[[], TagFunc]:
     """
     if not (found := Re.RE_TASK_FMT.search(call)):
         raise ValueError(
-            f"Call {call!r} does not match with call format regex."
+            f"Call {call!r} does not match with the call regex format."
         )
 
     # NOTE: Pass the searching call string to `path`, `func`, and `tag`.
@@ -160,13 +160,13 @@ def extract_call(call: str) -> Callable[[], TagFunc]:
     rgt: dict[str, Registry] = make_registry(f"{call.path}")
     if call.func not in rgt:
         raise NotImplementedError(
-            f"``REGISTER-MODULES.{call.path}.registries`` does not "
+            f"`REGISTER-MODULES.{call.path}.registries` does not "
             f"implement registry: {call.func!r}."
         )
 
     if call.tag not in rgt[call.func]:
         raise NotImplementedError(
             f"tag: {call.tag!r} does not found on registry func: "
-            f"``REGISTER-MODULES.{call.path}.registries.{call.func}``"
+            f"`REGISTER-MODULES.{call.path}.registries.{call.func}`"
         )
     return rgt[call.func][call.tag]

ddeutil/workflow/job.py

@@ -5,7 +5,7 @@
 # ------------------------------------------------------------------------------
 """Job Model that use for keeping stages and node that running its stages.
 The job handle the lineage of stages and location of execution of stages that
-mean the job model able to define ``runs-on`` key that allow you to run this
+mean the job model able to define `runs-on` key that allow you to run this
 job.
 
     This module include Strategy Model that use on the job strategy field.
@@ -24,10 +24,10 @@ from enum import Enum
 from functools import lru_cache
 from textwrap import dedent
 from threading import Event
-from typing import Any, Optional, Union
+from typing import Annotated, Any, Literal, Optional, Union
 
 from ddeutil.core import freeze_args
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
@@ -56,6 +56,11 @@ __all__: TupleStr = (
     "Strategy",
     "Job",
     "TriggerRules",
+    "RunsOn",
+    "RunsOnLocal",
+    "RunsOnSelfHosted",
+    "RunsOnDocker",
+    "RunsOnK8s",
     "make",
 )
 
@@ -216,13 +221,60 @@ class TriggerRules(str, Enum):
     none_skipped: str = "none_skipped"
 
 
-class RunsOn(str, Enum):
+class RunsOnType(str, Enum):
     """Runs-On enum object."""
 
-    local: str = "local"
-    docker: str = "docker"
-    self_hosted: str = "self_hosted"
-    k8s: str = "k8s"
+    LOCAL: str = "local"
+    DOCKER: str = "docker"
+    SELF_HOSTED: str = "self_hosted"
+    K8S: str = "k8s"
+
+
+class BaseRunsOn(BaseModel):
+    model_config = ConfigDict(use_enum_values=True)
+
+    type: Literal[RunsOnType.LOCAL]
+    args: DictData = Field(
+        default_factory=dict,
+        alias="with",
+    )
+
+
+class RunsOnLocal(BaseRunsOn):
+    """Runs-on local."""
+
+    type: Literal[RunsOnType.LOCAL] = Field(default=RunsOnType.LOCAL)
+
+
+class RunsOnSelfHosted(BaseRunsOn):
+    """Runs-on self-hosted."""
+
+    type: Literal[RunsOnType.SELF_HOSTED] = Field(
+        default=RunsOnType.SELF_HOSTED
+    )
+
+
+class RunsOnDocker(BaseRunsOn):
+    """Runs-on local Docker."""
+
+    type: Literal[RunsOnType.DOCKER] = Field(default=RunsOnType.DOCKER)
+
+
+class RunsOnK8s(BaseRunsOn):
+    """Runs-on Kubernetes."""
+
+    type: Literal[RunsOnType.K8S] = Field(default=RunsOnType.K8S)
+
+
+RunsOn = Annotated[
+    Union[
+        RunsOnLocal,
+        RunsOnSelfHosted,
+        RunsOnDocker,
+        RunsOnK8s,
+    ],
+    Field(discriminator="type"),
+]
 
 
 class Job(BaseModel):
@@ -263,9 +315,9 @@ class Job(BaseModel):
         default=None,
         description="A job description that can be string of markdown content.",
     )
-    runs_on: Optional[str] = Field(
-        default=None,
-        description="A target executor node for this job use to execution.",
+    runs_on: RunsOn = Field(
+        default_factory=RunsOnLocal,
+        description="A target node for this job to use for execution.",
         serialization_alias="runs-on",
     )
     stages: list[Stage] = Field(
@@ -359,7 +411,7 @@ class Job(BaseModel):
 
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
         """Set an outputs from execution process to the received context. The
-        result from execution will pass to value of ``strategies`` key.
+        result from execution will pass to value of `strategies` key.
 
             For example of setting output method, If you receive execute output
         and want to set on the `to` like;
@@ -424,14 +476,14 @@ class Job(BaseModel):
         workflow execution to strategy matrix.
 
             This execution is the minimum level of execution of this job model.
-        It different with ``self.execute`` because this method run only one
+        It different with `self.execute` because this method run only one
         strategy and return with context of this strategy data.
 
             The result of this execution will return result with strategy ID
         that generated from the `gen_id` function with an input strategy value.
 
-        :raise JobException: If it has any error from ``StageException`` or
-            ``UtilException``.
+        :raise JobException: If it has any error from `StageException` or
+            `UtilException`.
 
         :param strategy: A strategy metrix value that use on this execution.
             This value will pass to the `matrix` key for templating.
@@ -510,7 +562,7 @@ class Job(BaseModel):
             #
             #       ... params |= stage.execute(params=params)
             #
-            #   This step will add the stage result to ``stages`` key in
+            #   This step will add the stage result to `stages` key in
             #   that stage id. It will have structure like;
             #
             #   {
@@ -581,7 +633,7 @@ class Job(BaseModel):
     ) -> Result:
         """Job execution with passing dynamic parameters from the workflow
         execution. It will generate matrix values at the first step and run
-        multithread on this metrics to the ``stages`` field of this job.
+        multithread on this metrics to the `stages` field of this job.
 
         :param params: An input parameters that use on job execution.
         :param run_id: A job running ID for this execution.
@@ -591,15 +643,12 @@ class Job(BaseModel):
 
         :rtype: Result
         """
-
-        # NOTE: I use this condition because this method allow passing empty
-        #   params and I do not want to create new dict object.
         if result is None:  # pragma: no cov
             result: Result = Result(
                 run_id=(run_id or gen_id(self.id or "", unique=True)),
                 parent_run_id=parent_run_id,
             )
-        elif parent_run_id:
+        elif parent_run_id:  # pragma: no cov
             result.set_parent_run_id(parent_run_id)
 
         # NOTE: Normal Job execution without parallel strategy matrix. It uses