ddeutil-workflow 0.0.54__py3-none-any.whl → 0.0.55__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.54"
+__version__: str = "0.0.55"

ddeutil/workflow/api/__init__.py

@@ -1 +1,170 @@
-from .api import app
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import contextlib
+from collections.abc import AsyncIterator
+from datetime import datetime, timedelta
+from typing import TypedDict
+
+from dotenv import load_dotenv
+from fastapi import FastAPI, Request
+from fastapi import status as st
+from fastapi.encoders import jsonable_encoder
+from fastapi.exceptions import RequestValidationError
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.middleware.gzip import GZipMiddleware
+from fastapi.responses import UJSONResponse
+
+from ..__about__ import __version__
+from ..conf import api_config, config
+from ..logs import get_logger
+from ..scheduler import ReleaseThread, ReleaseThreads
+from ..workflow import ReleaseQueue, WorkflowTask
+from .routes import job, log
+from .utils import repeat_at
+
+load_dotenv()
+logger = get_logger("uvicorn.error")
+
+
+class State(TypedDict):
+    """TypeDict for State of FastAPI application."""
+
+    scheduler: list[str]
+    workflow_threads: ReleaseThreads
+    workflow_tasks: list[WorkflowTask]
+    workflow_queue: dict[str, ReleaseQueue]
+
+
+@contextlib.asynccontextmanager
+async def lifespan(a: FastAPI) -> AsyncIterator[State]:
+    """Lifespan function for the FastAPI application."""
+    a.state.scheduler = []
+    a.state.workflow_threads = {}
+    a.state.workflow_tasks = []
+    a.state.workflow_queue = {}
+
+    yield {
+        # NOTE: Scheduler value should be contained a key of workflow and
+        #   list of datetime of queue and running.
+        #
+        #   ... {
+        #   ...     '<workflow-name>': (
+        #   ...         [<running-datetime>, ...], [<queue-datetime>, ...]
+        #   ...     )
+        #   ... }
+        #
+        "scheduler": a.state.scheduler,
+        "workflow_queue": a.state.workflow_queue,
+        "workflow_threads": a.state.workflow_threads,
+        "workflow_tasks": a.state.workflow_tasks,
+    }
+
+
+app = FastAPI(
+    titile="Workflow",
+    description=(
+        "This is a workflow FastAPI application that use to manage manual "
+        "execute, logging, and schedule workflow via RestAPI."
+    ),
+    version=__version__,
+    lifespan=lifespan,
+    default_response_class=UJSONResponse,
+)
+app.add_middleware(GZipMiddleware, minimum_size=1000)
+origins: list[str] = [
+    "http://localhost",
+    "http://localhost:88",
+    "http://localhost:80",
+]
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.get(path="/", response_class=UJSONResponse)
+async def health():
+    """Index view that not return any template without json status."""
+    return {"message": "Workflow already start up with healthy status."}
+
+
+# NOTE Add the jobs and logs routes by default.
+app.include_router(job, prefix=api_config.prefix_path)
+app.include_router(log, prefix=api_config.prefix_path)
+
+
+# NOTE: Enable the workflows route.
+if api_config.enable_route_workflow:
+    from .routes import workflow
+
+    app.include_router(workflow, prefix=api_config.prefix_path)
+
+
+# NOTE: Enable the schedules route.
+if api_config.enable_route_schedule:
+    from ..logs import get_audit
+    from ..scheduler import schedule_task
+    from .routes import schedule
+
+    app.include_router(schedule, prefix=api_config.prefix_path)
+
+    @schedule.on_event("startup")
+    @repeat_at(cron="* * * * *", delay=2)
+    def scheduler_listener():
+        """Schedule broker every minute at 02 second."""
+        logger.debug(
+            f"[SCHEDULER]: Start listening schedule from queue "
+            f"{app.state.scheduler}"
+        )
+        if app.state.workflow_tasks:
+            schedule_task(
+                app.state.workflow_tasks,
+                stop=datetime.now(config.tz) + timedelta(minutes=1),
+                queue=app.state.workflow_queue,
+                threads=app.state.workflow_threads,
+                audit=get_audit(),
+            )
+
+    @schedule.on_event("startup")
+    @repeat_at(cron="*/5 * * * *", delay=10)
+    def monitoring():
+        """Monitoring workflow thread that running in the background."""
+        logger.debug("[MONITOR]: Start monitoring threading.")
+        snapshot_threads: list[str] = list(app.state.workflow_threads.keys())
+        for t_name in snapshot_threads:
+
+            thread_release: ReleaseThread = app.state.workflow_threads[t_name]
+
+            # NOTE: remove the thread that running success.
+            if not thread_release["thread"].is_alive():
+                app.state.workflow_threads.pop(t_name)
+
+
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(
+    request: Request, exc: RequestValidationError
+):
+    _ = request
+    return UJSONResponse(
+        status_code=st.HTTP_422_UNPROCESSABLE_ENTITY,
+        content=jsonable_encoder({"detail": exc.errors(), "body": exc.body}),
+    )
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(
+        app,
+        host="0.0.0.0",
+        port=80,
+        log_level="DEBUG",
+    )

ddeutil/workflow/api/routes/job.py

@@ -9,7 +9,7 @@ from typing import Any, Optional
 
 from fastapi import APIRouter
 from fastapi.responses import UJSONResponse
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 from ...__types import DictData
 from ...exceptions import JobException
@@ -18,33 +18,37 @@ from ...logs import get_logger
 from ...result import Result
 
 logger = get_logger("uvicorn.error")
+job_route = APIRouter(prefix="/job", tags=["job"])
 
 
-job_route = APIRouter(
-    prefix="/job",
-    tags=["job"],
-    default_response_class=UJSONResponse,
-)
+class ResultCreate(BaseModel):
+    """Create Result model for receive running IDs to create the Result
+    dataclass.
+    """
 
-
-class ResultPost(BaseModel):
-    context: DictData
-    run_id: str
-    parent_run_id: Optional[str] = None
+    run_id: str = Field(description="A running ID.")
+    parent_run_id: Optional[str] = Field(
+        default=None, description="A parent running ID."
+    )
 
 
-@job_route.post(path="/execute/")
+@job_route.post(path="/execute/", response_class=UJSONResponse)
 async def job_execute(
-    result: ResultPost,
+    result: ResultCreate,
     job: Job,
     params: dict[str, Any],
+    extras: Optional[dict[str, Any]] = None,
 ):
-    """Execute job via RestAPI."""
+    """Execute job via RestAPI with execute route path."""
     rs: Result = Result(
-        context=result.context,
         run_id=result.run_id,
         parent_run_id=result.parent_run_id,
+        extras=extras or {},
     )
+
+    if extras:
+        job.extras = extras
+
     context: DictData = {}
     try:
         job.set_outputs(
@@ -59,14 +63,11 @@ async def job_execute(
         rs.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
 
     return {
-        "message": "Start execute job via API.",
-        "result": {
-            "run_id": rs.run_id,
-            "parent_run_id": rs.parent_run_id,
-        },
+        "message": "Execute job via RestAPI.",
+        "result": {"run_id": rs.run_id, "parent_run_id": rs.parent_run_id},
         "job": job.model_dump(
             by_alias=True,
-            exclude_none=True,
+            exclude_none=False,
             exclude_unset=True,
             exclude_defaults=True,
         ),

ddeutil/workflow/api/routes/schedules.py

@@ -17,7 +17,6 @@ from ...logs import get_logger
 from ...scheduler import Schedule
 
 logger = get_logger("uvicorn.error")
-
 schedule_route = APIRouter(
     prefix="/schedules",
     tags=["schedules"],
@@ -108,7 +107,6 @@ async def add_deploy_scheduler(request: Request, name: str):
         schedule.tasks(
             start_date_waiting,
             queue=request.state.workflow_queue,
-            extras={},
         ),
     )
     return {

ddeutil/workflow/api/routes/workflows.py

@@ -21,7 +21,6 @@ from ...result import Result
 from ...workflow import Workflow
 
 logger = get_logger("uvicorn.error")
-
 workflow_route = APIRouter(
     prefix="/workflows",
     tags=["workflows"],
@@ -55,7 +54,7 @@ async def get_workflow_by_name(name: str) -> DictData:
         ) from None
     return workflow.model_dump(
         by_alias=True,
-        exclude_none=True,
+        exclude_none=False,
         exclude_unset=True,
         exclude_defaults=True,
     )
@@ -98,7 +97,7 @@ async def get_workflow_audits(name: str):
             "audits": [
                 audit.model_dump(
                     by_alias=True,
-                    exclude_none=True,
+                    exclude_none=False,
                     exclude_unset=True,
                     exclude_defaults=True,
                 )
@@ -132,7 +131,7 @@ async def get_workflow_release_audit(name: str, release: str):
         "message": f"Getting workflow {name!r} audit in release {release}",
         "audit": audit.model_dump(
             by_alias=True,
-            exclude_none=True,
+            exclude_none=False,
             exclude_unset=True,
             exclude_defaults=True,
         ),

ddeutil/workflow/job.py

@@ -540,6 +540,11 @@ class Job(BaseModel):
                         }
                     }
 
+            The keys that will set to the received context is `strategies`,
+        `errors`, and `skipped` keys. The `errors` and `skipped` keys will
+        extract from the result context if it exists. If it does not found, it
+        will not set on the received context.
+
         :raise JobException: If the job's ID does not set and the setting
             default job ID flag does not set.
 
@@ -599,7 +604,7 @@ class Job(BaseModel):
 
         :param params: (DictData) A parameter data.
         :param run_id: (str) A job running ID.
-        :param parent_run_id: (str) A parent workflow running ID.
+        :param parent_run_id: (str) A parent running ID.
         :param event: (Event) An Event manager instance that use to cancel this
             execution if it forces stopped by parent execution.
 
@@ -667,15 +672,15 @@ def local_execute_strategy(
     `set_outputs` method for reconstruct result context data.
 
     :param job: (Job) A job model that want to execute.
-    :param strategy: A strategy metrix value that use on this execution.
-        This value will pass to the `matrix` key for templating.
+    :param strategy: (DictData) A strategy metrix value. This value will pass
+        to the `matrix` key for templating in context data.
     :param params: (DictData) A parameter data.
     :param result: (Result) A Result instance for return context and status.
     :param event: (Event) An Event manager instance that use to cancel this
         execution if it forces stopped by parent execution.
 
-    :raise JobException: If it has any error from `StageException` or
-        `UtilException`.
+    :raise JobException: If stage execution raise any error as `StageException`
+        or `UtilException`.
 
     :rtype: Result
     """
@@ -683,17 +688,16 @@ def local_execute_strategy(
         run_id=gen_id(job.id or "not-set", unique=True),
         extras=job.extras,
     )
-
-    strategy_id: str = gen_id(strategy)
-    context: DictData = copy.deepcopy(params)
-    context.update({"matrix": strategy, "stages": {}})
-
     if strategy:
+        strategy_id: str = gen_id(strategy)
         result.trace.info(f"[JOB]: Start Strategy: {strategy_id!r}")
         result.trace.info(f"[JOB]: ... matrix: {strategy!r}")
     else:
-        result.trace.info("[JOB]: Start Strategy: EMPTY")
+        strategy_id: str = "EMPTY"
+        result.trace.info("[JOB]: Start Strategy: 'EMPTY'")
 
+    context: DictData = copy.deepcopy(params)
+    context.update({"matrix": strategy, "stages": {}})
     for stage in job.stages:
 
         if job.extras:
@@ -707,7 +711,7 @@ def local_execute_strategy(
         if event and event.is_set():
             error_msg: str = (
                 "Job strategy was canceled from event that had set before "
-                "strategy execution."
+                "job strategy execution."
             )
             return result.catch(
                 status=CANCEL,
@@ -820,7 +824,7 @@ def local_execute(
             context={
                 "errors": JobException(
                     "Job was canceled from event that had set before "
-                    "local execution."
+                    "local job execution."
                 ).to_dict()
             },
         )

ddeutil/workflow/result.py

@@ -98,6 +98,7 @@ class Result:
             return cls(
                 run_id=(run_id or gen_id(id_logic or "", unique=True)),
                 parent_run_id=parent_run_id,
+                ts=get_dt_now(dynamic("tz", extras=extras)),
                 extras=(extras or {}),
             )
         elif parent_run_id:

ddeutil/workflow/scheduler.py

@@ -535,9 +535,7 @@ def schedule_task(
         current_release: datetime = current_date.replace(
             second=0, microsecond=0
         )
-        if (
-            first_date := q.first_queue.date
-        ) > current_release:  # pragma: no cov
+        if (first_date := q.queue[0].date) > current_release:  # pragma: no cov
             result.trace.debug(
                 f"[WORKFLOW]: Skip schedule "
                 f"{first_date:%Y-%m-%d %H:%M:%S} for : {task.alias!r}"