ddeutil-workflow 0.0.23__py3-none-any.whl → 0.0.25__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.23"
+__version__: str = "0.0.25"

ddeutil/workflow/__cron.py

@@ -736,6 +736,12 @@ class CronRunner:
         self.is_year: bool = isinstance(cron, CronJobYear)
         self.reset_flag: bool = True
 
+    def __repr__(self) -> str:
+        return (
+            f"{self.__class__.__name__}(CronJob('{self.cron}'), "
+            f"{self.date:%Y-%m-%d %H:%M:%S}, tz='{self.tz}')"
+        )
+
     def reset(self) -> None:
         """Resets the iterator to start time."""
         self.date: datetime = self.__start_date

ddeutil/workflow/__init__.py

@@ -3,11 +3,17 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+from .__cron import CronRunner
 from .conf import (
     Config,
     FileLog,
     Loader,
 )
+from .cron import (
+    On,
+    YearOn,
+    interval2crontab,
+)
 from .exceptions import (
     JobException,
     ParamValueException,
@@ -19,11 +25,6 @@ from .job import (
     Job,
     Strategy,
 )
-from .on import (
-    On,
-    YearOn,
-    interval2crontab,
-)
 from .params import (
     ChoiceParam,
     DatetimeParam,
@@ -35,6 +36,7 @@ from .result import Result
 from .scheduler import (
     Schedule,
     WorkflowSchedule,
+    schedule_runner,
 )
 from .stage import (
     BashStage,
@@ -68,11 +70,10 @@ from .utils import (
     map_post_filter,
     not_in_template,
     param2template,
-    queue2str,
     str2template,
     tag,
 )
 from .workflow import (
     Workflow,
-    WorkflowTaskData,
+    WorkflowTask,
 )

ddeutil/workflow/api/__init__.py

@@ -0,0 +1 @@
+from .api import app

ddeutil/workflow/{api.py → api/api.py}

@@ -11,7 +11,6 @@ import uuid
 from collections.abc import AsyncIterator
 from datetime import datetime, timedelta
 from queue import Empty, Queue
-from threading import Thread
 from typing import TypedDict
 
 from dotenv import load_dotenv
@@ -20,34 +19,39 @@ from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import UJSONResponse
 from pydantic import BaseModel
 
-from .__about__ import __version__
-from .conf import config, get_logger
+from ..__about__ import __version__
+from ..conf import config, get_logger
+from ..scheduler import ReleaseThread, ReleaseThreads
+from ..workflow import WorkflowQueue, WorkflowTask
 from .repeat import repeat_at, repeat_every
-from .workflow import WorkflowTaskData
 
 load_dotenv()
 logger = get_logger("ddeutil.workflow")
 
 
 class State(TypedDict):
+    """TypeDict for State of FastAPI application."""
+
+    # NOTE: For upper queue route.
     upper_queue: Queue
     upper_result: dict[str, str]
+
+    # NOTE: For schedule listener.
     scheduler: list[str]
-    workflow_threads: dict[str, Thread]
-    workflow_tasks: list[WorkflowTaskData]
-    workflow_queue: dict[str, list[datetime]]
-    workflow_running: dict[str, list[datetime]]
+    workflow_threads: ReleaseThreads
+    workflow_tasks: list[WorkflowTask]
+    workflow_queue: dict[str, WorkflowQueue]
 
 
 @contextlib.asynccontextmanager
 async def lifespan(a: FastAPI) -> AsyncIterator[State]:
+    """Lifespan function for the FastAPI application."""
     a.state.upper_queue = Queue()
     a.state.upper_result = {}
     a.state.scheduler = []
     a.state.workflow_threads = {}
     a.state.workflow_tasks = []
     a.state.workflow_queue = {}
-    a.state.workflow_running = {}
 
     await asyncio.create_task(broker_upper_messages())
 
@@ -65,7 +69,6 @@ async def lifespan(a: FastAPI) -> AsyncIterator[State]:
         #
         "scheduler": a.state.scheduler,
         "workflow_queue": a.state.workflow_queue,
-        "workflow_running": a.state.workflow_running,
         "workflow_threads": a.state.workflow_threads,
         "workflow_tasks": a.state.workflow_tasks,
     }
@@ -114,12 +117,11 @@ async def get_result(request_id: str) -> dict[str, str]:
 
 
 @app.get("/")
-@app.get("/api")
 async def health():
     return {"message": "Workflow API already start up"}
 
 
-@app.post("/api")
+@app.post(f"{config.prefix_path}/upper")
 async def message_upper(payload: Payload):
     """Convert message from any case to the upper case."""
     request_id: str = str(uuid.uuid4())
@@ -129,27 +131,47 @@ async def message_upper(payload: Payload):
     return await get_result(request_id)
 
 
+# NOTE: Enable the workflow route.
 if config.enable_route_workflow:
-    from .route import workflow
+    from .route import workflow_route
+
+    app.include_router(workflow_route, prefix=config.prefix_path)
 
-    app.include_router(workflow)
 
+# NOTE: Enable the schedule route.
 if config.enable_route_schedule:
-    from .route import schedule
-    from .scheduler import workflow_task
+    from ..conf import FileLog
+    from ..scheduler import schedule_task
+    from .route import schedule_route
 
-    app.include_router(schedule)
+    app.include_router(schedule_route, prefix=config.prefix_path)
 
-    @schedule.on_event("startup")
+    @schedule_route.on_event("startup")
     @repeat_at(cron="* * * * *", delay=2)
-    def schedule_broker_up():
+    def scheduler_listener():
+        """Schedule broker every minute at 02 second."""
         logger.debug(
             f"[SCHEDULER]: Start listening schedule from queue "
             f"{app.state.scheduler}"
         )
         if app.state.workflow_tasks:
-            workflow_task(
+            schedule_task(
                 app.state.workflow_tasks,
-                stop=datetime.now() + timedelta(minutes=1),
+                stop=datetime.now(config.tz) + timedelta(minutes=1),
+                queue=app.state.workflow_queue,
                 threads=app.state.workflow_threads,
+                log=FileLog,
             )
+
+    @schedule_route.on_event("startup")
+    @repeat_at(cron="*/5 * * * *")
+    def monitoring():
+        logger.debug("[MONITOR]: Start monitoring threading.")
+        snapshot_threads: list[str] = list(app.state.workflow_threads.keys())
+        for t_name in snapshot_threads:
+
+            thread_release: ReleaseThread = app.state.workflow_threads[t_name]
+
+            # NOTE: remove the thread that running success.
+            if not thread_release["thread"].is_alive():
+                app.state.workflow_threads.pop(t_name)

ddeutil/workflow/{repeat.py → api/repeat.py}

@@ -12,8 +12,8 @@ from functools import wraps
 
 from starlette.concurrency import run_in_threadpool
 
-from .__cron import CronJob
-from .conf import config, get_logger
+from ..__cron import CronJob
+from ..conf import config, get_logger
 
 logger = get_logger("ddeutil.workflow")
 

ddeutil/workflow/{route.py → api/route.py}

@@ -6,6 +6,7 @@
 from __future__ import annotations
 
 import copy
+from dataclasses import asdict
 from datetime import datetime, timedelta
 from typing import Any
 
@@ -14,41 +15,43 @@ from fastapi import status as st
 from fastapi.responses import UJSONResponse
 from pydantic import BaseModel
 
-from . import Workflow
-from .__types import DictData
-from .conf import Loader, config, get_logger
-from .result import Result
-from .scheduler import Schedule
+from ..__types import DictData
+from ..conf import Loader, config, get_logger
+from ..result import Result
+from ..scheduler import Schedule
+from ..workflow import Workflow
 
 logger = get_logger("ddeutil.workflow")
-workflow = APIRouter(
-    prefix="/api/workflow",
-    tags=["workflow"],
+
+workflow_route = APIRouter(
+    prefix="/workflows",
+    tags=["workflows"],
     default_response_class=UJSONResponse,
 )
-schedule = APIRouter(
-    prefix="/api/schedule",
-    tags=["schedule"],
+
+schedule_route = APIRouter(
+    prefix="/schedules",
+    tags=["schedules"],
     default_response_class=UJSONResponse,
 )
 
-ListDate = list[datetime]
 
-
-@workflow.get("/")
-async def get_workflows():
+@workflow_route.get(path="/")
+async def get_workflows() -> DictData:
     """Return all workflow workflows that exists in config path."""
-    workflows: DictData = Loader.finds(Workflow)
+    workflows: DictData = dict(Loader.finds(Workflow))
     return {
-        "message": f"getting all workflows: {workflows}",
+        "message": f"Getting all workflows: {len(workflows)}",
+        "count": len(workflows),
+        "workflows": workflows,
     }
 
 
-@workflow.get("/{name}")
-async def get_workflow(name: str) -> DictData:
+@workflow_route.get(path="/{name}")
+async def get_workflow_by_name(name: str) -> DictData:
     """Return model of workflow that passing an input workflow name."""
     try:
-        wf: Workflow = Workflow.from_loader(name=name, externals={})
+        workflow: Workflow = Workflow.from_loader(name=name, externals={})
     except ValueError as err:
         logger.exception(err)
         raise HTTPException(
@@ -57,7 +60,7 @@ async def get_workflow(name: str) -> DictData:
                 f"Workflow workflow name: {name!r} does not found in /conf path"
             ),
         ) from None
-    return wf.model_dump(
+    return workflow.model_dump(
         by_alias=True,
         exclude_none=True,
         exclude_unset=True,
@@ -69,11 +72,11 @@ class ExecutePayload(BaseModel):
     params: dict[str, Any]
 
 
-@workflow.post("/{name}/execute", status_code=st.HTTP_202_ACCEPTED)
+@workflow_route.post(path="/{name}/execute", status_code=st.HTTP_202_ACCEPTED)
 async def execute_workflow(name: str, payload: ExecutePayload) -> DictData:
     """Return model of workflow that passing an input workflow name."""
     try:
-        wf: Workflow = Workflow.from_loader(name=name, externals={})
+        workflow: Workflow = Workflow.from_loader(name=name, externals={})
     except ValueError:
         raise HTTPException(
             status_code=st.HTTP_404_NOT_FOUND,
@@ -83,36 +86,45 @@ async def execute_workflow(name: str, payload: ExecutePayload) -> DictData:
         ) from None
 
     # NOTE: Start execute manually
-    rs: Result = wf.execute(params=payload.params)
+    try:
+        result: Result = workflow.execute(params=payload.params)
+    except Exception as err:
+        raise HTTPException(
+            status_code=st.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"{type(err)}: {err}",
+        ) from None
 
-    return dict(rs)
+    return asdict(result)
 
 
-@workflow.get("/{name}/logs")
+@workflow_route.get(path="/{name}/logs")
 async def get_workflow_logs(name: str):
-    return {"message": f"getting workflow {name!r} logs"}
+    return {"message": f"Getting workflow {name!r} logs"}
 
 
-@workflow.get("/{name}/logs/{release}")
+@workflow_route.get(path="/{name}/logs/{release}")
 async def get_workflow_release_log(name: str, release: str):
-    return {"message": f"getting workflow {name!r} log in release {release}"}
+    return {"message": f"Getting workflow {name!r} log in release {release}"}
 
 
-@workflow.delete("/{name}/logs/{release}", status_code=st.HTTP_204_NO_CONTENT)
+@workflow_route.delete(
+    path="/{name}/logs/{release}",
+    status_code=st.HTTP_204_NO_CONTENT,
+)
 async def del_workflow_release_log(name: str, release: str):
-    return {"message": f"deleted workflow {name!r} log in release {release}"}
+    return {"message": f"Deleted workflow {name!r} log in release {release}"}
 
 
-@schedule.get("/{name}")
-async def get_schedule(name: str):
+@schedule_route.get(path="/{name}")
+async def get_schedules(name: str):
     try:
-        sch: Schedule = Schedule.from_loader(name=name, externals={})
+        schedule: Schedule = Schedule.from_loader(name=name, externals={})
     except ValueError:
         raise HTTPException(
             status_code=st.HTTP_404_NOT_FOUND,
             detail=f"Schedule name: {name!r} does not found in /conf path",
         ) from None
-    return sch.model_dump(
+    return schedule.model_dump(
         by_alias=True,
         exclude_none=True,
         exclude_unset=True,
@@ -120,32 +132,32 @@ async def get_schedule(name: str):
     )
 
 
-@schedule.get("/deploy")
+@schedule_route.get(path="/deploy")
 async def get_deploy_schedulers(request: Request):
     snapshot = copy.deepcopy(request.state.scheduler)
     return {"schedule": snapshot}
 
 
-@schedule.get("/deploy/{name}")
+@schedule_route.get(path="/deploy/{name}")
 async def get_deploy_scheduler(request: Request, name: str):
     if name in request.state.scheduler:
         sch = Schedule.from_loader(name)
         getter: list[dict[str, dict[str, list[datetime]]]] = []
-        for wf in sch.workflows:
+        for workflow in sch.workflows:
             getter.append(
                 {
-                    wf.name: {
+                    workflow.name: {
                         "queue": copy.deepcopy(
-                            request.state.workflow_queue[wf.name]
+                            request.state.workflow_queue[workflow.name]
                         ),
                         "running": copy.deepcopy(
-                            request.state.workflow_running[wf.name]
+                            request.state.workflow_running[workflow.name]
                         ),
                     }
                 }
             )
         return {
-            "message": f"getting {name!r} to schedule listener.",
+            "message": f"Getting {name!r} to schedule listener.",
             "scheduler": getter,
         }
     raise HTTPException(
@@ -154,13 +166,13 @@ async def get_deploy_scheduler(request: Request, name: str):
     )
 
 
-@schedule.post("/deploy/{name}")
+@schedule_route.post(path="/deploy/{name}")
 async def add_deploy_scheduler(request: Request, name: str):
     """Adding schedule name to application state store."""
     if name in request.state.scheduler:
         raise HTTPException(
             status_code=st.HTTP_302_FOUND,
-            detail="This schedule already exists in scheduler list.",
+            detail=f"This schedule {name!r} already exists in scheduler list.",
         )
 
     request.state.scheduler.append(name)
@@ -172,42 +184,49 @@ async def add_deploy_scheduler(request: Request, name: str):
 
     # NOTE: Create pair of workflow and on from schedule model.
     try:
-        sch = Schedule.from_loader(name)
-    except ValueError as e:
+        schedule: Schedule = Schedule.from_loader(name)
+    except ValueError as err:
         request.state.scheduler.remove(name)
-        logger.exception(e)
+        logger.exception(err)
         raise HTTPException(
             status_code=st.HTTP_404_NOT_FOUND,
-            detail=str(e),
+            detail=str(err),
         ) from None
+
     request.state.workflow_tasks.extend(
-        sch.tasks(
+        schedule.tasks(
             start_date_waiting,
             queue=request.state.workflow_queue,
-            running=request.state.workflow_running,
+            externals={},
         ),
     )
-    return {"message": f"adding {name!r} to schedule listener."}
+    return {
+        "message": f"Adding {name!r} to schedule listener.",
+        "start_date": start_date_waiting,
+    }
 
 
-@schedule.delete("/deploy/{name}")
+@schedule_route.delete(path="/deploy/{name}")
 async def del_deploy_scheduler(request: Request, name: str):
+    """Delete workflow task on the schedule listener."""
     if name in request.state.scheduler:
+
+        # NOTE: Remove current schedule name from the state.
         request.state.scheduler.remove(name)
-        sche = Schedule.from_loader(name)
-        for workflow_task in sche.tasks(datetime.now(), {}, {}):
-            request.state.workflow_tasks.remove(workflow_task)
 
-        for wf in sche.workflows:
-            if wf in request.state.workflow_queue:
-                request.state.workflow_queue.pop(wf, {})
+        schedule: Schedule = Schedule.from_loader(name)
 
-            if wf in request.state.workflow_running:
-                request.state.workflow_running.pop(wf, {})
+        for task in schedule.tasks(datetime.now(tz=config.tz), queue={}):
+            if task in request.state.workflow_tasks:
+                request.state.workflow_tasks.remove(task)
 
-        return {"message": f"deleted {name!r} to schedule listener."}
+        for workflow in schedule.workflows:
+            if workflow.alias in request.state.workflow_queue:
+                request.state.workflow_queue.pop(workflow.alias)
+
+        return {"message": f"Deleted schedule {name!r} in listener."}
 
     raise HTTPException(
         status_code=st.HTTP_404_NOT_FOUND,
-        detail=f"Does not found {name!r} in schedule listener",
+        detail=f"Does not found schedule {name!r} in listener",
     )

ddeutil/workflow/cli.py

@@ -6,23 +6,16 @@
 from __future__ import annotations
 
 import json
+import sys
 from datetime import datetime
-from enum import Enum
 from typing import Annotated, Optional
 
 from ddeutil.core import str2list
-from typer import Argument, Option, Typer
+from typer import Argument, Typer, echo
 
-from .conf import config, get_logger
+from .conf import config
 
-logger = get_logger("ddeutil.workflow")
 cli: Typer = Typer()
-cli_log: Typer = Typer()
-cli.add_typer(
-    cli_log,
-    name="log",
-    help="Logging of workflow CLI",
-)
 
 
 @cli.command()
@@ -34,15 +27,28 @@ def run(
     params: Annotated[
         str,
         Argument(
-            help="A json string for parameters of this workflow execution."
+            help="A json string for parameters of this workflow execution.",
         ),
     ],
 ):
     """Run workflow workflow manually with an input custom parameters that able
     to receive with workflow params config.
     """
-    logger.info(f"Running workflow name: {workflow}")
-    logger.info(f"... with Parameters: {json.dumps(json.loads(params))}")
+    echo(f"Running workflow name: ({type(workflow)}) {workflow!r}")
+    echo(f"... with Parameters: ({type(params)}) {params!r}")
+
+    from .result import Result
+    from .workflow import Workflow
+
+    try:
+        wf: Workflow = Workflow.from_loader(name=workflow)
+        rs: Result = wf.execute(params=json.loads(params))
+    except Exception as err:
+        echo(str(err))
+        sys.exit(1)
+
+    echo(f"Result: {rs}")
+    sys.exit(0)
 
 
 @cli.command()
@@ -69,54 +75,26 @@ def schedule(
     module.
     """
     excluded: list[str] = str2list(excluded) if excluded else []
+    echo(f"... with Excluded Parameters: {excluded!r}")
     externals: str = externals or "{}"
+
+    # NOTE: Convert timezone on the stop date.
     if stop:
         stop: datetime = stop.astimezone(tz=config.tz)
 
-    from .scheduler import workflow_runner
-
-    # NOTE: Start running workflow scheduler application.
-    workflow_rs: list[str] = workflow_runner(
-        stop=stop, excluded=excluded, externals=json.loads(externals)
-    )
-    logger.info(f"Application run success: {workflow_rs}")
+    from .scheduler import schedule_runner
 
+    try:
+        # NOTE: Start running workflow scheduler application.
+        workflow_rs: list[str] = schedule_runner(
+            stop=stop, excluded=excluded, externals=json.loads(externals)
+        )
+        echo(f"Schedule with CLI run success with: {workflow_rs}")
+    except Exception as err:
+        echo(str(err))
+        sys.exit(1)
 
-@cli_log.command("workflow-get")
-def workflow_log_get(
-    name: Annotated[
-        str,
-        Argument(help="A workflow name that want to getting log"),
-    ],
-    limit: Annotated[
-        int,
-        Argument(help="A number of the limitation of logging"),
-    ] = 100,
-    desc: Annotated[
-        bool,
-        Option(
-            "--desc",
-            help="A descending flag that order by logging release datetime.",
-        ),
-    ] = True,
-):
-    logger.info(f"{name} : limit {limit} : desc: {desc}")
-    return [""]
-
-
-class LogMode(str, Enum):
-    get = "get"
-    delete = "delete"
-
-
-@cli_log.command("workflow-delete")
-def workflow_log_delete(
-    mode: Annotated[
-        LogMode,
-        Argument(case_sensitive=True),
-    ]
-):
-    logger.info(mode)
+    sys.exit(0)
 
 
 @cli.callback()