ddeutil-workflow 0.0.10__py3-none-any.whl → 0.0.12__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.10"
+__version__: str = "0.0.12"

ddeutil/workflow/__init__.py

@@ -6,12 +6,13 @@
 from .exceptions import (
     JobException,
     ParamValueException,
-    PipelineException,
     StageException,
     UtilException,
+    WorkflowException,
 )
+from .job import Job, Strategy
 from .on import On, interval2crontab
-from .pipeline import Job, Pipeline, Strategy
+from .scheduler import Workflow
 from .stage import Stage, handler_result
 from .utils import (
     Param,

ddeutil/workflow/api.py

@@ -6,9 +6,14 @@
 from __future__ import annotations
 
 import asyncio
+import contextlib
 import os
 import uuid
+from collections.abc import AsyncIterator
+from datetime import datetime, timedelta
 from queue import Empty, Queue
+from threading import Thread
+from typing import TypedDict
 
 from ddeutil.core import str2bool
 from dotenv import load_dotenv
@@ -19,12 +24,55 @@ from pydantic import BaseModel
 
 from .__about__ import __version__
 from .log import get_logger
-from .repeat import repeat_every
+from .repeat import repeat_at, repeat_every
+from .scheduler import WorkflowTask
 
 load_dotenv()
 logger = get_logger("ddeutil.workflow")
 
 
+class State(TypedDict):
+    upper_queue: Queue
+    upper_result: dict[str, str]
+    scheduler: list[str]
+    workflow_threads: dict[str, Thread]
+    workflow_tasks: list[WorkflowTask]
+    workflow_queue: dict[str, list[datetime]]
+    workflow_running: dict[str, list[datetime]]
+
+
+@contextlib.asynccontextmanager
+async def lifespan(a: FastAPI) -> AsyncIterator[State]:
+    a.state.upper_queue = Queue()
+    a.state.upper_result = {}
+    a.state.scheduler = []
+    a.state.workflow_threads = {}
+    a.state.workflow_tasks = []
+    a.state.workflow_queue = {}
+    a.state.workflow_running = {}
+
+    await asyncio.create_task(broker_upper_messages())
+
+    yield {
+        "upper_queue": a.state.upper_queue,
+        "upper_result": a.state.upper_result,
+        # NOTE: Scheduler value should be contain a key of workflow workflow and
+        #   list of datetime of queue and running.
+        #
+        #   ... {
+        #   ...     '<workflow-name>': (
+        #   ...         [<running-datetime>, ...], [<queue-datetime>, ...]
+        #   ...     )
+        #   ... }
+        #
+        "scheduler": a.state.scheduler,
+        "workflow_queue": a.state.workflow_queue,
+        "workflow_running": a.state.workflow_running,
+        "workflow_threads": a.state.workflow_threads,
+        "workflow_tasks": a.state.workflow_tasks,
+    }
+
+
 app = FastAPI(
     titile="Workflow API",
     description=(
@@ -32,47 +80,52 @@ app = FastAPI(
         "execute or schedule workflow via RestAPI."
     ),
     version=__version__,
+    lifespan=lifespan,
+    default_response_class=UJSONResponse,
 )
 app.add_middleware(GZipMiddleware, minimum_size=1000)
-app.queue = Queue()
-app.output_dict = {}
-app.queue_limit = 5
 
 
-@app.on_event("startup")
 @repeat_every(seconds=10)
-def broker_upper_messages():
+async def broker_upper_messages():
     """Broker for receive message from the `/upper` path and change it to upper
     case. This broker use interval running in background every 10 seconds.
     """
-    for _ in range(app.queue_limit):
+    for _ in range(10):
         try:
-            obj = app.queue.get_nowait()
-            app.output_dict[obj["request_id"]] = obj["text"].upper()
-            logger.info(f"Upper message: {app.output_dict}")
+            obj = app.state.upper_queue.get_nowait()
+            app.state.upper_result[obj["request_id"]] = obj["text"].upper()
+            logger.info(f"Upper message: {app.state.upper_result}")
         except Empty:
             pass
+    await asyncio.sleep(0.0001)
 
 
 class Payload(BaseModel):
     text: str
 
 
-async def get_result(request_id):
+async def get_result(request_id: str) -> dict[str, str]:
     """Get data from output dict that global."""
     while True:
-        if request_id in app.output_dict:
-            result = app.output_dict[request_id]
-            del app.output_dict[request_id]
+        if request_id in app.state.upper_result:
+            result: str = app.state.upper_result[request_id]
+            del app.state.upper_result[request_id]
             return {"message": result}
         await asyncio.sleep(0.0025)
 
 
-@app.post("/upper", response_class=UJSONResponse)
+@app.get("/")
+@app.get("/api")
+async def health():
+    return {"message": "Workflow API already start up"}
+
+
+@app.post("/api")
 async def message_upper(payload: Payload):
     """Convert message from any case to the upper case."""
     request_id: str = str(uuid.uuid4())
-    app.queue.put(
+    app.state.upper_queue.put(
         {"text": payload.text, "request_id": request_id},
     )
     return await get_result(request_id)
@@ -85,5 +138,20 @@ if str2bool(os.getenv("WORKFLOW_API_ENABLE_ROUTE_WORKFLOW", "true")):
 
 if str2bool(os.getenv("WORKFLOW_API_ENABLE_ROUTE_SCHEDULE", "true")):
     from .route import schedule
+    from .scheduler import workflow_task
 
     app.include_router(schedule)
+
+    @schedule.on_event("startup")
+    @repeat_at(cron="* * * * *", delay=2)
+    def schedule_broker_up():
+        logger.debug(
+            f"[SCHEDULER]: Start listening schedule from queue "
+            f"{app.state.scheduler}"
+        )
+        if app.state.workflow_tasks:
+            workflow_task(
+                app.state.workflow_tasks,
+                stop=datetime.now() + timedelta(minutes=1),
+                threads=app.state.workflow_threads,
+            )

ddeutil/workflow/cli.py

@@ -29,21 +29,21 @@ cli.add_typer(
 
 @cli.command()
 def run(
-    pipeline: Annotated[
+    workflow: Annotated[
         str,
-        Argument(help="A pipeline name that want to run manually"),
+        Argument(help="A workflow name that want to run manually"),
     ],
     params: Annotated[
         str,
         Argument(
-            help="A json string for parameters of this pipeline execution."
+            help="A json string for parameters of this workflow execution."
         ),
     ],
 ):
-    """Run pipeline workflow manually with an input custom parameters that able
-    to receive with pipeline params config.
+    """Run workflow workflow manually with an input custom parameters that able
+    to receive with workflow params config.
     """
-    logger.info(f"Running pipeline name: {pipeline}")
+    logger.info(f"Running workflow name: {workflow}")
     logger.info(f"... with Parameters: {json.dumps(json.loads(params))}")
 
 
@@ -63,7 +63,7 @@ def schedule(
     externals: Annotated[
         Optional[str],
         Argument(
-            help="A json string for parameters of this pipeline execution."
+            help="A json string for parameters of this workflow execution."
         ),
     ] = None,
 ):
@@ -77,20 +77,20 @@ def schedule(
             tz=ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
         )
 
-    from .scheduler import workflow
+    from .scheduler import workflow_runner
 
     # NOTE: Start running workflow scheduler application.
-    workflow_rs: list[str] = workflow(
+    workflow_rs: list[str] = workflow_runner(
         stop=stop, excluded=excluded, externals=json.loads(externals)
     )
     logger.info(f"Application run success: {workflow_rs}")
 
 
-@cli_log.command("pipeline-get")
-def pipeline_log_get(
+@cli_log.command("workflow-get")
+def workflow_log_get(
     name: Annotated[
         str,
-        Argument(help="A pipeline name that want to getting log"),
+        Argument(help="A workflow name that want to getting log"),
     ],
     limit: Annotated[
         int,
@@ -113,8 +113,8 @@ class LogMode(str, Enum):
     delete = "delete"
 
 
-@cli_log.command("pipeline-delete")
-def pipeline_log_delete(
+@cli_log.command("workflow-delete")
+def workflow_log_delete(
     mode: Annotated[
         LogMode,
         Argument(case_sensitive=True),

ddeutil/workflow/exceptions.py

@@ -6,22 +6,22 @@
 from __future__ import annotations
 
 
-class WorkflowException(Exception): ...
+class BaseWorkflowException(Exception): ...
 
 
-class UtilException(WorkflowException): ...
+class UtilException(BaseWorkflowException): ...
 
 
-class StageException(WorkflowException): ...
+class StageException(BaseWorkflowException): ...
 
 
-class JobException(WorkflowException): ...
+class JobException(BaseWorkflowException): ...
 
 
-class PipelineException(WorkflowException): ...
+class WorkflowException(BaseWorkflowException): ...
 
 
-class PipelineFailException(WorkflowException): ...
+class WorkflowFailException(WorkflowException): ...
 
 
 class ParamValueException(WorkflowException): ...