ddeutil-workflow 0.0.53__py3-none-any.whl → 0.0.55__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.53"
+__version__: str = "0.0.55"

ddeutil/workflow/api/__init__.py

@@ -1 +1,170 @@
-from .api import app
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import contextlib
+from collections.abc import AsyncIterator
+from datetime import datetime, timedelta
+from typing import TypedDict
+
+from dotenv import load_dotenv
+from fastapi import FastAPI, Request
+from fastapi import status as st
+from fastapi.encoders import jsonable_encoder
+from fastapi.exceptions import RequestValidationError
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.middleware.gzip import GZipMiddleware
+from fastapi.responses import UJSONResponse
+
+from ..__about__ import __version__
+from ..conf import api_config, config
+from ..logs import get_logger
+from ..scheduler import ReleaseThread, ReleaseThreads
+from ..workflow import ReleaseQueue, WorkflowTask
+from .routes import job, log
+from .utils import repeat_at
+
+load_dotenv()
+logger = get_logger("uvicorn.error")
+
+
+class State(TypedDict):
+    """TypeDict for State of FastAPI application."""
+
+    scheduler: list[str]
+    workflow_threads: ReleaseThreads
+    workflow_tasks: list[WorkflowTask]
+    workflow_queue: dict[str, ReleaseQueue]
+
+
+@contextlib.asynccontextmanager
+async def lifespan(a: FastAPI) -> AsyncIterator[State]:
+    """Lifespan function for the FastAPI application."""
+    a.state.scheduler = []
+    a.state.workflow_threads = {}
+    a.state.workflow_tasks = []
+    a.state.workflow_queue = {}
+
+    yield {
+        # NOTE: Scheduler value should be contained a key of workflow and
+        #   list of datetime of queue and running.
+        #
+        #   ... {
+        #   ...     '<workflow-name>': (
+        #   ...         [<running-datetime>, ...], [<queue-datetime>, ...]
+        #   ...     )
+        #   ... }
+        #
+        "scheduler": a.state.scheduler,
+        "workflow_queue": a.state.workflow_queue,
+        "workflow_threads": a.state.workflow_threads,
+        "workflow_tasks": a.state.workflow_tasks,
+    }
+
+
+app = FastAPI(
+    titile="Workflow",
+    description=(
+        "This is a workflow FastAPI application that use to manage manual "
+        "execute, logging, and schedule workflow via RestAPI."
+    ),
+    version=__version__,
+    lifespan=lifespan,
+    default_response_class=UJSONResponse,
+)
+app.add_middleware(GZipMiddleware, minimum_size=1000)
+origins: list[str] = [
+    "http://localhost",
+    "http://localhost:88",
+    "http://localhost:80",
+]
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.get(path="/", response_class=UJSONResponse)
+async def health():
+    """Index view that not return any template without json status."""
+    return {"message": "Workflow already start up with healthy status."}
+
+
+# NOTE Add the jobs and logs routes by default.
+app.include_router(job, prefix=api_config.prefix_path)
+app.include_router(log, prefix=api_config.prefix_path)
+
+
+# NOTE: Enable the workflows route.
+if api_config.enable_route_workflow:
+    from .routes import workflow
+
+    app.include_router(workflow, prefix=api_config.prefix_path)
+
+
+# NOTE: Enable the schedules route.
+if api_config.enable_route_schedule:
+    from ..logs import get_audit
+    from ..scheduler import schedule_task
+    from .routes import schedule
+
+    app.include_router(schedule, prefix=api_config.prefix_path)
+
+    @schedule.on_event("startup")
+    @repeat_at(cron="* * * * *", delay=2)
+    def scheduler_listener():
+        """Schedule broker every minute at 02 second."""
+        logger.debug(
+            f"[SCHEDULER]: Start listening schedule from queue "
+            f"{app.state.scheduler}"
+        )
+        if app.state.workflow_tasks:
+            schedule_task(
+                app.state.workflow_tasks,
+                stop=datetime.now(config.tz) + timedelta(minutes=1),
+                queue=app.state.workflow_queue,
+                threads=app.state.workflow_threads,
+                audit=get_audit(),
+            )
+
+    @schedule.on_event("startup")
+    @repeat_at(cron="*/5 * * * *", delay=10)
+    def monitoring():
+        """Monitoring workflow thread that running in the background."""
+        logger.debug("[MONITOR]: Start monitoring threading.")
+        snapshot_threads: list[str] = list(app.state.workflow_threads.keys())
+        for t_name in snapshot_threads:
+
+            thread_release: ReleaseThread = app.state.workflow_threads[t_name]
+
+            # NOTE: remove the thread that running success.
+            if not thread_release["thread"].is_alive():
+                app.state.workflow_threads.pop(t_name)
+
+
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(
+    request: Request, exc: RequestValidationError
+):
+    _ = request
+    return UJSONResponse(
+        status_code=st.HTTP_422_UNPROCESSABLE_ENTITY,
+        content=jsonable_encoder({"detail": exc.errors(), "body": exc.body}),
+    )
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(
+        app,
+        host="0.0.0.0",
+        port=80,
+        log_level="DEBUG",
+    )

ddeutil/workflow/api/routes/job.py

@@ -9,7 +9,7 @@ from typing import Any, Optional
 
 from fastapi import APIRouter
 from fastapi.responses import UJSONResponse
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 from ...__types import DictData
 from ...exceptions import JobException
@@ -18,33 +18,37 @@ from ...logs import get_logger
 from ...result import Result
 
 logger = get_logger("uvicorn.error")
+job_route = APIRouter(prefix="/job", tags=["job"])
 
 
-job_route = APIRouter(
-    prefix="/job",
-    tags=["job"],
-    default_response_class=UJSONResponse,
-)
+class ResultCreate(BaseModel):
+    """Create Result model for receive running IDs to create the Result
+    dataclass.
+    """
 
-
-class ResultPost(BaseModel):
-    context: DictData
-    run_id: str
-    parent_run_id: Optional[str] = None
+    run_id: str = Field(description="A running ID.")
+    parent_run_id: Optional[str] = Field(
+        default=None, description="A parent running ID."
+    )
 
 
-@job_route.post(path="/execute/")
+@job_route.post(path="/execute/", response_class=UJSONResponse)
 async def job_execute(
-    result: ResultPost,
+    result: ResultCreate,
     job: Job,
     params: dict[str, Any],
+    extras: Optional[dict[str, Any]] = None,
 ):
-    """Execute job via API."""
+    """Execute job via RestAPI with execute route path."""
     rs: Result = Result(
-        context=result.context,
         run_id=result.run_id,
         parent_run_id=result.parent_run_id,
+        extras=extras or {},
     )
+
+    if extras:
+        job.extras = extras
+
     context: DictData = {}
     try:
         job.set_outputs(
@@ -56,17 +60,14 @@ async def job_execute(
             to=context,
         )
     except JobException as err:
-        rs.trace.error(f"[WORKFLOW]: {err.__class__.__name__}: {err}")
+        rs.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
 
     return {
-        "message": "Start execute job via API.",
-        "result": {
-            "run_id": rs.run_id,
-            "parent_run_id": rs.parent_run_id,
-        },
+        "message": "Execute job via RestAPI.",
+        "result": {"run_id": rs.run_id, "parent_run_id": rs.parent_run_id},
         "job": job.model_dump(
             by_alias=True,
-            exclude_none=True,
+            exclude_none=False,
             exclude_unset=True,
             exclude_defaults=True,
         ),

ddeutil/workflow/api/routes/schedules.py

@@ -17,7 +17,6 @@ from ...logs import get_logger
 from ...scheduler import Schedule
 
 logger = get_logger("uvicorn.error")
-
 schedule_route = APIRouter(
     prefix="/schedules",
     tags=["schedules"],
@@ -108,7 +107,6 @@ async def add_deploy_scheduler(request: Request, name: str):
         schedule.tasks(
             start_date_waiting,
             queue=request.state.workflow_queue,
-            extras={},
         ),
     )
     return {

ddeutil/workflow/api/routes/workflows.py

@@ -21,7 +21,6 @@ from ...result import Result
 from ...workflow import Workflow
 
 logger = get_logger("uvicorn.error")
-
 workflow_route = APIRouter(
     prefix="/workflows",
     tags=["workflows"],
@@ -55,7 +54,7 @@ async def get_workflow_by_name(name: str) -> DictData:
         ) from None
     return workflow.model_dump(
         by_alias=True,
-        exclude_none=True,
+        exclude_none=False,
         exclude_unset=True,
         exclude_defaults=True,
     )
@@ -98,7 +97,7 @@ async def get_workflow_audits(name: str):
             "audits": [
                 audit.model_dump(
                     by_alias=True,
-                    exclude_none=True,
+                    exclude_none=False,
                     exclude_unset=True,
                     exclude_defaults=True,
                 )
@@ -132,7 +131,7 @@ async def get_workflow_release_audit(name: str, release: str):
         "message": f"Getting workflow {name!r} audit in release {release}",
         "audit": audit.model_dump(
             by_alias=True,
-            exclude_none=True,
+            exclude_none=False,
             exclude_unset=True,
             exclude_defaults=True,
         ),