ddeutil-workflow 0.0.6__py3-none-any.whl → 0.0.8__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.6"
+__version__: str = "0.0.8"

ddeutil/workflow/__init__.py

@@ -3,7 +3,29 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-from .exceptions import StageException
-from .on import On
-from .pipeline import Pipeline
-from .stage import Stage
+from .exceptions import (
+    JobException,
+    ParamValueException,
+    PipelineException,
+    StageException,
+    UtilException,
+)
+from .on import AwsOn, On
+from .pipeline import Job, Pipeline
+from .stage import (
+    BashStage,
+    EmptyStage,
+    HookStage,
+    PyStage,
+    Stage,
+    TriggerStage,
+)
+from .utils import (
+    ChoiceParam,
+    DatetimeParam,
+    IntParam,
+    Param,
+    StrParam,
+    dash2underscore,
+    param2template,
+)

ddeutil/workflow/__types.py

@@ -27,12 +27,21 @@ class Re:
     """Regular expression config."""
 
     # NOTE: Search caller
+    # \${{\s*(?P<caller>[a-zA-Z0-9_.\s'\"\[\]\(\)\-\{}]+?)\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
     __re_caller: str = r"""
         \$
         {{
-            \s*(?P<caller>
+            \s*
+            (?P<caller>
                 [a-zA-Z0-9_.\s'\"\[\]\(\)\-\{}]+?
             )\s*
+            (?P<post_filters>
+                (?:
+                    \|\s*
+                    (?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]*)
+                    \s*
+                )*
+            )
         }}
     """
     RE_CALLER: Pattern = re.compile(
@@ -40,6 +49,7 @@ class Re:
     )
 
     # NOTE: Search task
+    # ^(?P<path>[^/@]+)/(?P<func>[^@]+)@(?P<tag>.+)$
     __re_task_fmt: str = r"""
         ^
             (?P<path>[^/@]+)

ddeutil/workflow/api.py

@@ -0,0 +1,120 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import asyncio
+import queue
+import time
+import uuid
+from contextlib import asynccontextmanager
+from datetime import datetime
+
+from apscheduler.executors.pool import ProcessPoolExecutor
+from apscheduler.jobstores.memory import MemoryJobStore
+from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from fastapi import BackgroundTasks, FastAPI
+from fastapi.middleware.gzip import GZipMiddleware
+from fastapi.responses import UJSONResponse
+from pydantic import BaseModel
+
+from .log import get_logger
+from .repeat import repeat_every
+from .route import schedule_route, workflow_route
+
+logger = get_logger(__name__)
+
+
+def broker_upper_messages():
+    for _ in range(app.queue_limit):
+        try:
+            obj = app.queue.get_nowait()
+            app.output_dict[obj["request_id"]] = obj["text"].upper()
+            logger.info(f"Upper message: {app.output_dict}")
+        except queue.Empty:
+            pass
+
+
+jobstores = {
+    "default": MemoryJobStore(),
+    "sqlite": SQLAlchemyJobStore(url="sqlite:///jobs-store.sqlite"),
+}
+executors = {
+    "default": {"type": "threadpool", "max_workers": 5},
+    "processpool": ProcessPoolExecutor(max_workers=5),
+}
+scheduler = AsyncIOScheduler(
+    jobstores=jobstores,
+    executors=executors,
+    timezone="Asia/Bangkok",
+)
+
+
+@asynccontextmanager
+async def lifespan(_: FastAPI):
+    scheduler.start()
+    yield
+    scheduler.shutdown(wait=False)
+
+
+app = FastAPI(lifespan=lifespan)
+app.add_middleware(GZipMiddleware, minimum_size=1000)
+app.include_router(schedule_route)
+app.include_router(workflow_route)
+
+app.scheduler = scheduler
+app.scheduler.add_job(
+    broker_upper_messages,
+    "interval",
+    seconds=10,
+)
+app.queue = queue.Queue()
+app.output_dict = {}
+app.queue_limit = 2
+
+
+def write_pipeline(task_id: str, message=""):
+    logger.info(f"{task_id} : {message}")
+    time.sleep(5)
+    logger.info(f"{task_id} : run task successfully!!!")
+
+
+@app.post("/schedule/{name}", response_class=UJSONResponse)
+async def send_schedule(name: str, background_tasks: BackgroundTasks):
+    background_tasks.add_task(
+        write_pipeline,
+        name,
+        message=f"some message for {name}",
+    )
+    await fetch_current_time()
+    return {"message": f"Schedule sent {name!r} in the background"}
+
+
+@repeat_every(seconds=2, max_repetitions=3)
+async def fetch_current_time():
+    logger.info(f"Fetch: {datetime.now()}")
+
+
+class Payload(BaseModel):
+    text: str
+
+
+async def get_result(request_id):
+    while 1:
+        if request_id in app.output_dict:
+            result = app.output_dict[request_id]
+            del app.output_dict[request_id]
+            return {"message": result}
+        await asyncio.sleep(0.001)
+
+
+@app.post("/upper", response_class=UJSONResponse)
+async def message_upper(payload: Payload):
+    request_id: str = str(uuid.uuid4())
+    app.queue.put(
+        {"text": payload.text, "request_id": request_id},
+    )
+    return await get_result(request_id)

ddeutil/workflow/app.py

@@ -0,0 +1,45 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import functools
+import time
+
+import schedule
+
+
+def catch_exceptions(cancel_on_failure=False):
+    """Catch exception error from scheduler job."""
+
+    def catch_exceptions_decorator(job_func):
+        @functools.wraps(job_func)
+        def wrapper(*args, **kwargs):
+            try:
+                return job_func(*args, **kwargs)
+            except Exception as err:
+                print(err)
+
+                if cancel_on_failure:
+                    return schedule.CancelJob
+
+        return wrapper
+
+    return catch_exceptions_decorator
+
+
+@catch_exceptions(cancel_on_failure=True)
+def bad_task():
+    return 1 / 0
+
+
+schedule.every(5).seconds.do(bad_task)
+
+if __name__ == "__main__":
+    while True:
+        schedule.run_pending()
+        time.sleep(1)
+        if not schedule.get_jobs():
+            break

ddeutil/workflow/exceptions.py

@@ -3,9 +3,6 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""
-Define Errors Object for Node package
-"""
 from __future__ import annotations
 
 
@@ -22,3 +19,6 @@ class JobException(WorkflowException): ...
 
 
 class PipelineException(WorkflowException): ...
+
+
+class ParamValueException(WorkflowException): ...

ddeutil/workflow/log.py

@@ -0,0 +1,79 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+from functools import lru_cache
+from typing import Union
+
+from pydantic import BaseModel, Field
+from rich.console import Console
+from rich.logging import RichHandler
+
+from .__types import DictData
+
+console = Console(color_system="256", width=200, style="blue")
+
+
+@lru_cache
+def get_logger(module_name):
+    logger = logging.getLogger(module_name)
+    handler = RichHandler(
+        rich_tracebacks=True, console=console, tracebacks_show_locals=True
+    )
+    handler.setFormatter(
+        logging.Formatter(
+            "[ %(threadName)s:%(funcName)s:%(process)d ] - %(message)s"
+        )
+    )
+    logger.addHandler(handler)
+    logger.setLevel(logging.DEBUG)
+    return logger
+
+
+class BaseLog(BaseModel):
+    """Base logging model."""
+
+    parent_id: str
+    id: str
+    input: DictData
+    output: DictData
+    update_time: datetime = Field(default_factory=datetime.now)
+
+
+class StageLog(BaseLog): ...
+
+
+class JobLog(BaseLog): ...
+
+
+class PipelineLog(BaseLog): ...
+
+
+Log = Union[
+    StageLog,
+    JobLog,
+    PipelineLog,
+]
+
+
+def push_log_memory(log: DictData):
+    """Push message log to globals log queue."""
+    print(log)
+
+
+LOGS_REGISTRY = {
+    "memory": push_log_memory,
+}
+
+
+def push_log(log: DictData, mode: str = "memory"):
+    return LOGS_REGISTRY[mode](log)
+
+
+def save_log():
+    """Save log that push to queue to target saving"""