ddeutil-workflow 0.0.8__py3-none-any.whl → 0.0.10__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.8"
+__version__: str = "0.0.10"

ddeutil/workflow/__init__.py

@@ -10,22 +10,11 @@ from .exceptions import (
     StageException,
     UtilException,
 )
-from .on import AwsOn, On
-from .pipeline import Job, Pipeline
-from .stage import (
-    BashStage,
-    EmptyStage,
-    HookStage,
-    PyStage,
-    Stage,
-    TriggerStage,
-)
+from .on import On, interval2crontab
+from .pipeline import Job, Pipeline, Strategy
+from .stage import Stage, handler_result
 from .utils import (
-    ChoiceParam,
-    DatetimeParam,
-    IntParam,
     Param,
-    StrParam,
     dash2underscore,
     param2template,
 )

ddeutil/workflow/api.py

@@ -6,115 +6,84 @@
 from __future__ import annotations
 
 import asyncio
-import queue
-import time
+import os
 import uuid
-from contextlib import asynccontextmanager
-from datetime import datetime
-
-from apscheduler.executors.pool import ProcessPoolExecutor
-from apscheduler.jobstores.memory import MemoryJobStore
-from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
-from apscheduler.schedulers.asyncio import AsyncIOScheduler
-from fastapi import BackgroundTasks, FastAPI
+from queue import Empty, Queue
+
+from ddeutil.core import str2bool
+from dotenv import load_dotenv
+from fastapi import FastAPI
 from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import UJSONResponse
 from pydantic import BaseModel
 
+from .__about__ import __version__
 from .log import get_logger
 from .repeat import repeat_every
-from .route import schedule_route, workflow_route
 
-logger = get_logger(__name__)
+load_dotenv()
+logger = get_logger("ddeutil.workflow")
 
 
+app = FastAPI(
+    titile="Workflow API",
+    description=(
+        "This is workflow FastAPI web application that use to manage manual "
+        "execute or schedule workflow via RestAPI."
+    ),
+    version=__version__,
+)
+app.add_middleware(GZipMiddleware, minimum_size=1000)
+app.queue = Queue()
+app.output_dict = {}
+app.queue_limit = 5
+
+
+@app.on_event("startup")
+@repeat_every(seconds=10)
 def broker_upper_messages():
+    """Broker for receive message from the `/upper` path and change it to upper
+    case. This broker use interval running in background every 10 seconds.
+    """
     for _ in range(app.queue_limit):
         try:
             obj = app.queue.get_nowait()
             app.output_dict[obj["request_id"]] = obj["text"].upper()
             logger.info(f"Upper message: {app.output_dict}")
-        except queue.Empty:
+        except Empty:
             pass
 
 
-jobstores = {
-    "default": MemoryJobStore(),
-    "sqlite": SQLAlchemyJobStore(url="sqlite:///jobs-store.sqlite"),
-}
-executors = {
-    "default": {"type": "threadpool", "max_workers": 5},
-    "processpool": ProcessPoolExecutor(max_workers=5),
-}
-scheduler = AsyncIOScheduler(
-    jobstores=jobstores,
-    executors=executors,
-    timezone="Asia/Bangkok",
-)
-
-
-@asynccontextmanager
-async def lifespan(_: FastAPI):
-    scheduler.start()
-    yield
-    scheduler.shutdown(wait=False)
-
-
-app = FastAPI(lifespan=lifespan)
-app.add_middleware(GZipMiddleware, minimum_size=1000)
-app.include_router(schedule_route)
-app.include_router(workflow_route)
-
-app.scheduler = scheduler
-app.scheduler.add_job(
-    broker_upper_messages,
-    "interval",
-    seconds=10,
-)
-app.queue = queue.Queue()
-app.output_dict = {}
-app.queue_limit = 2
-
-
-def write_pipeline(task_id: str, message=""):
-    logger.info(f"{task_id} : {message}")
-    time.sleep(5)
-    logger.info(f"{task_id} : run task successfully!!!")
-
-
-@app.post("/schedule/{name}", response_class=UJSONResponse)
-async def send_schedule(name: str, background_tasks: BackgroundTasks):
-    background_tasks.add_task(
-        write_pipeline,
-        name,
-        message=f"some message for {name}",
-    )
-    await fetch_current_time()
-    return {"message": f"Schedule sent {name!r} in the background"}
-
-
-@repeat_every(seconds=2, max_repetitions=3)
-async def fetch_current_time():
-    logger.info(f"Fetch: {datetime.now()}")
-
-
 class Payload(BaseModel):
     text: str
 
 
 async def get_result(request_id):
-    while 1:
+    """Get data from output dict that global."""
+    while True:
         if request_id in app.output_dict:
             result = app.output_dict[request_id]
             del app.output_dict[request_id]
             return {"message": result}
-        await asyncio.sleep(0.001)
+        await asyncio.sleep(0.0025)
 
 
 @app.post("/upper", response_class=UJSONResponse)
 async def message_upper(payload: Payload):
+    """Convert message from any case to the upper case."""
     request_id: str = str(uuid.uuid4())
     app.queue.put(
         {"text": payload.text, "request_id": request_id},
     )
     return await get_result(request_id)
+
+
+if str2bool(os.getenv("WORKFLOW_API_ENABLE_ROUTE_WORKFLOW", "true")):
+    from .route import workflow
+
+    app.include_router(workflow)
+
+if str2bool(os.getenv("WORKFLOW_API_ENABLE_ROUTE_SCHEDULE", "true")):
+    from .route import schedule
+
+    app.include_router(schedule)

ddeutil/workflow/cli.py

@@ -0,0 +1,134 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime
+from enum import Enum
+from typing import Annotated, Optional
+from zoneinfo import ZoneInfo
+
+from ddeutil.core import str2list
+from typer import Argument, Option, Typer
+
+from .log import get_logger
+
+logger = get_logger("ddeutil.workflow")
+cli: Typer = Typer()
+cli_log: Typer = Typer()
+cli.add_typer(
+    cli_log,
+    name="log",
+    help="Logging of workflow CLI",
+)
+
+
+@cli.command()
+def run(
+    pipeline: Annotated[
+        str,
+        Argument(help="A pipeline name that want to run manually"),
+    ],
+    params: Annotated[
+        str,
+        Argument(
+            help="A json string for parameters of this pipeline execution."
+        ),
+    ],
+):
+    """Run pipeline workflow manually with an input custom parameters that able
+    to receive with pipeline params config.
+    """
+    logger.info(f"Running pipeline name: {pipeline}")
+    logger.info(f"... with Parameters: {json.dumps(json.loads(params))}")
+
+
+@cli.command()
+def schedule(
+    stop: Annotated[
+        Optional[datetime],
+        Argument(
+            formats=["%Y-%m-%d", "%Y-%m-%d %H:%M:%S"],
+            help="A stopping datetime that want to stop on schedule app.",
+        ),
+    ] = None,
+    excluded: Annotated[
+        Optional[str],
+        Argument(help="A list of exclude workflow name in str."),
+    ] = None,
+    externals: Annotated[
+        Optional[str],
+        Argument(
+            help="A json string for parameters of this pipeline execution."
+        ),
+    ] = None,
+):
+    """Start workflow scheduler that will call workflow function from scheduler
+    module.
+    """
+    excluded: list[str] = str2list(excluded) if excluded else []
+    externals: str = externals or "{}"
+    if stop:
+        stop: datetime = stop.astimezone(
+            tz=ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
+        )
+
+    from .scheduler import workflow
+
+    # NOTE: Start running workflow scheduler application.
+    workflow_rs: list[str] = workflow(
+        stop=stop, excluded=excluded, externals=json.loads(externals)
+    )
+    logger.info(f"Application run success: {workflow_rs}")
+
+
+@cli_log.command("pipeline-get")
+def pipeline_log_get(
+    name: Annotated[
+        str,
+        Argument(help="A pipeline name that want to getting log"),
+    ],
+    limit: Annotated[
+        int,
+        Argument(help="A number of the limitation of logging"),
+    ] = 100,
+    desc: Annotated[
+        bool,
+        Option(
+            "--desc",
+            help="A descending flag that order by logging release datetime.",
+        ),
+    ] = True,
+):
+    logger.info(f"{name} : limit {limit} : desc: {desc}")
+    return [""]
+
+
+class LogMode(str, Enum):
+    get = "get"
+    delete = "delete"
+
+
+@cli_log.command("pipeline-delete")
+def pipeline_log_delete(
+    mode: Annotated[
+        LogMode,
+        Argument(case_sensitive=True),
+    ]
+):
+    logger.info(mode)
+
+
+@cli.callback()
+def main():
+    """
+    Manage workflow with CLI.
+    """
+
+
+if __name__ == "__main__":
+    cli()