ddeutil-workflow 0.0.5__py3-none-any.whl → 0.0.7__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.5"
+__version__: str = "0.0.7"

ddeutil/workflow/__init__.py

@@ -0,0 +1,31 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from .exceptions import (
+    JobException,
+    ParamValueException,
+    PipelineException,
+    StageException,
+    UtilException,
+)
+from .on import AwsOn, On
+from .pipeline import Job, Pipeline
+from .stage import (
+    BashStage,
+    EmptyStage,
+    HookStage,
+    PyStage,
+    Stage,
+    TriggerStage,
+)
+from .utils import (
+    ChoiceParam,
+    DatetimeParam,
+    IntParam,
+    Param,
+    StrParam,
+    dash2underscore,
+    param2template,
+)

ddeutil/workflow/__types.py

@@ -5,8 +5,60 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
-from typing import Any
+import re
+from re import (
+    IGNORECASE,
+    MULTILINE,
+    UNICODE,
+    VERBOSE,
+    Pattern,
+)
+from typing import Any, Union
 
 TupleStr = tuple[str, ...]
 DictData = dict[str, Any]
 DictStr = dict[str, str]
+Matrix = dict[str, Union[list[str], list[int]]]
+MatrixInclude = list[dict[str, Union[str, int]]]
+MatrixExclude = list[dict[str, Union[str, int]]]
+
+
+class Re:
+    """Regular expression config."""
+
+    # NOTE: Search caller
+    # \${{\s*(?P<caller>[a-zA-Z0-9_.\s'\"\[\]\(\)\-\{}]+?)\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
+    __re_caller: str = r"""
+        \$
+        {{
+            \s*
+            (?P<caller>
+                [a-zA-Z0-9_.\s'\"\[\]\(\)\-\{}]+?
+            )\s*
+            (?P<post_filters>
+                (?:
+                    \|\s*
+                    (?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]*)
+                    \s*
+                )*
+            )
+        }}
+    """
+    RE_CALLER: Pattern = re.compile(
+        __re_caller, MULTILINE | IGNORECASE | UNICODE | VERBOSE
+    )
+
+    # NOTE: Search task
+    # ^(?P<path>[^/@]+)/(?P<func>[^@]+)@(?P<tag>.+)$
+    __re_task_fmt: str = r"""
+        ^
+            (?P<path>[^/@]+)
+            /
+            (?P<func>[^@]+)
+            @
+            (?P<tag>.+)
+        $
+    """
+    RE_TASK_FMT: Pattern = re.compile(
+        __re_task_fmt, MULTILINE | IGNORECASE | UNICODE | VERBOSE
+    )

ddeutil/workflow/api.py

@@ -0,0 +1,120 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import asyncio
+import queue
+import time
+import uuid
+from contextlib import asynccontextmanager
+from datetime import datetime
+
+from apscheduler.executors.pool import ProcessPoolExecutor
+from apscheduler.jobstores.memory import MemoryJobStore
+from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from fastapi import BackgroundTasks, FastAPI
+from fastapi.middleware.gzip import GZipMiddleware
+from fastapi.responses import UJSONResponse
+from pydantic import BaseModel
+
+from .log import get_logger
+from .repeat import repeat_every
+from .route import schedule_route, workflow_route
+
+logger = get_logger(__name__)
+
+
+def broker_upper_messages():
+    for _ in range(app.queue_limit):
+        try:
+            obj = app.queue.get_nowait()
+            app.output_dict[obj["request_id"]] = obj["text"].upper()
+            logger.info(f"Upper message: {app.output_dict}")
+        except queue.Empty:
+            pass
+
+
+jobstores = {
+    "default": MemoryJobStore(),
+    "sqlite": SQLAlchemyJobStore(url="sqlite:///jobs-store.sqlite"),
+}
+executors = {
+    "default": {"type": "threadpool", "max_workers": 5},
+    "processpool": ProcessPoolExecutor(max_workers=5),
+}
+scheduler = AsyncIOScheduler(
+    jobstores=jobstores,
+    executors=executors,
+    timezone="Asia/Bangkok",
+)
+
+
+@asynccontextmanager
+async def lifespan(_: FastAPI):
+    scheduler.start()
+    yield
+    scheduler.shutdown(wait=False)
+
+
+app = FastAPI(lifespan=lifespan)
+app.add_middleware(GZipMiddleware, minimum_size=1000)
+app.include_router(schedule_route)
+app.include_router(workflow_route)
+
+app.scheduler = scheduler
+app.scheduler.add_job(
+    broker_upper_messages,
+    "interval",
+    seconds=10,
+)
+app.queue = queue.Queue()
+app.output_dict = {}
+app.queue_limit = 2
+
+
+def write_pipeline(task_id: str, message=""):
+    logger.info(f"{task_id} : {message}")
+    time.sleep(5)
+    logger.info(f"{task_id} : run task successfully!!!")
+
+
+@app.post("/schedule/{name}", response_class=UJSONResponse)
+async def send_schedule(name: str, background_tasks: BackgroundTasks):
+    background_tasks.add_task(
+        write_pipeline,
+        name,
+        message=f"some message for {name}",
+    )
+    await fetch_current_time()
+    return {"message": f"Schedule sent {name!r} in the background"}
+
+
+@repeat_every(seconds=2, max_repetitions=3)
+async def fetch_current_time():
+    logger.info(f"Fetch: {datetime.now()}")
+
+
+class Payload(BaseModel):
+    text: str
+
+
+async def get_result(request_id):
+    while 1:
+        if request_id in app.output_dict:
+            result = app.output_dict[request_id]
+            del app.output_dict[request_id]
+            return {"message": result}
+        await asyncio.sleep(0.001)
+
+
+@app.post("/upper", response_class=UJSONResponse)
+async def message_upper(payload: Payload):
+    request_id: str = str(uuid.uuid4())
+    app.queue.put(
+        {"text": payload.text, "request_id": request_id},
+    )
+    return await get_result(request_id)

ddeutil/workflow/app.py

@@ -0,0 +1,41 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+import functools
+import time
+
+import schedule
+
+
+def catch_exceptions(cancel_on_failure=False):
+    def catch_exceptions_decorator(job_func):
+        @functools.wraps(job_func)
+        def wrapper(*args, **kwargs):
+            try:
+                return job_func(*args, **kwargs)
+            except Exception as err:
+                print(err)
+
+                if cancel_on_failure:
+                    return schedule.CancelJob
+
+        return wrapper
+
+    return catch_exceptions_decorator
+
+
+@catch_exceptions(cancel_on_failure=True)
+def bad_task():
+    return 1 / 0
+
+
+schedule.every(5).seconds.do(bad_task)
+
+if __name__ == "__main__":
+    while True:
+        schedule.run_pending()
+        time.sleep(1)
+        if not schedule.get_jobs():
+            break

ddeutil/workflow/exceptions.py

@@ -9,4 +9,19 @@ Define Errors Object for Node package
 from __future__ import annotations
 
 
-class TaskException(Exception): ...
+class WorkflowException(Exception): ...
+
+
+class UtilException(WorkflowException): ...
+
+
+class StageException(WorkflowException): ...
+
+
+class JobException(WorkflowException): ...
+
+
+class PipelineException(WorkflowException): ...
+
+
+class ParamValueException(ValueError): ...

ddeutil/workflow/loader.py

@@ -6,49 +6,17 @@
 from __future__ import annotations
 
 from functools import cached_property
-from typing import Any, ClassVar, TypeVar
+from typing import TypeVar
 
-from ddeutil.core import (
-    getdot,
-    hasdot,
-    import_string,
-)
-from ddeutil.io import (
-    PathData,
-    PathSearch,
-    YamlEnvFl,
-)
-from pydantic import BaseModel, Field
-from pydantic.functional_validators import model_validator
+from ddeutil.core import import_string
+from ddeutil.io import PathSearch, YamlFlResolve
+from pydantic import BaseModel
 
-from .__regex import RegexConf
 from .__types import DictData
+from .utils import ConfParams, config
 
-T = TypeVar("T")
-BaseModelType = type[BaseModel]
 AnyModel = TypeVar("AnyModel", bound=BaseModel)
-
-
-class Engine(BaseModel):
-    """Engine Model"""
-
-    paths: PathData = Field(default_factory=PathData)
-    registry: list[str] = Field(default_factory=lambda: ["ddeutil.workflow"])
-
-    @model_validator(mode="before")
-    def __prepare_registry(cls, values: DictData) -> DictData:
-        """Prepare registry value that passing with string type. It convert the
-        string type to list of string.
-        """
-        if (_regis := values.get("registry")) and isinstance(_regis, str):
-            values["registry"] = [_regis]
-        return values
-
-
-class Params(BaseModel):
-    """Params Model"""
-
-    engine: Engine = Field(default_factory=Engine)
+AnyModelType = type[AnyModel]
 
 
 class SimLoad:
@@ -66,26 +34,22 @@ class SimLoad:
     def __init__(
         self,
         name: str,
-        params: Params,
+        params: ConfParams,
         externals: DictData,
     ) -> None:
         self.data: DictData = {}
         for file in PathSearch(params.engine.paths.conf).files:
             if any(file.suffix.endswith(s) for s in ("yml", "yaml")) and (
-                data := YamlEnvFl(file).read().get(name, {})
+                data := YamlFlResolve(file).read().get(name, {})
             ):
                 self.data = data
         if not self.data:
             raise ValueError(f"Config {name!r} does not found on conf path")
-        self.__conf_params: Params = params
+        self.conf_params: ConfParams = params
         self.externals: DictData = externals
 
-    @property
-    def conf_params(self) -> Params:
-        return self.__conf_params
-
     @cached_property
-    def type(self) -> BaseModelType:
+    def type(self) -> AnyModelType:
         """Return object of string type which implement on any registry. The
         object type
         """
@@ -104,79 +68,13 @@ class SimLoad:
                     continue
             return import_string(f"{_typ}")
 
-    def load(self) -> AnyModel:
-        """Parsing config data to the object type for initialize with model
-        validate method.
-        """
-        return self.type.model_validate(self.data)
-
 
 class Loader(SimLoad):
-    """Main Loader Object that get the config `yaml` file from current path.
+    """Loader Object that get the config `yaml` file from current path.
 
     :param name: A name of config data that will read by Yaml Loader object.
     :param externals: An external parameters
     """
 
-    conf_name: ClassVar[str] = "workflows-conf"
-
-    def __init__(
-        self,
-        name: str,
-        externals: DictData,
-        *,
-        path: str | None = None,
-    ) -> None:
-        self.data: DictData = {}
-
-        # NOTE: import params object from specific config file
-        params: Params = self.config(path)
-
-        super().__init__(name, params, externals)
-
-    @classmethod
-    def config(cls, path: str | None = None) -> Params:
-        """Load Config data from ``workflows-conf.yaml`` file."""
-        return Params.model_validate(
-            YamlEnvFl(path or f"./{cls.conf_name}.yaml").read()
-        )
-
-
-def map_params(value: Any, params: dict[str, Any]) -> Any:
-    """Map caller value that found from ``RE_CALLER`` regular expression.
-
-    :param value: A value that want to mapped with an params
-    :param params: A parameter value that getting with matched regular
-        expression.
-
-    :rtype: Any
-    :returns: An any getter value from the params input.
-    """
-    if isinstance(value, dict):
-        return {k: map_params(value[k], params) for k in value}
-    elif isinstance(value, (list, tuple, set)):
-        return type(value)([map_params(i, params) for i in value])
-    elif not isinstance(value, str):
-        return value
-
-    if not (found := RegexConf.RE_CALLER.search(value)):
-        return value
-
-    # NOTE: get caller value that setting inside; ``${{ <caller-value> }}``
-    caller: str = found.group("caller")
-    if not hasdot(caller, params):
-        raise ValueError(f"params does not set caller: {caller!r}")
-    getter: Any = getdot(caller, params)
-
-    # NOTE: check type of vars
-    if isinstance(getter, (str, int)):
-        return value.replace(found.group(0), str(getter))
-
-    # NOTE:
-    #   If type of getter caller does not formatting, it will return origin
-    #   value.
-    if value.replace(found.group(0), "") != "":
-        raise ValueError(
-            "Callable variable should not pass other outside ${{ ... }}"
-        )
-    return getter
+    def __init__(self, name: str, externals: DictData) -> None:
+        super().__init__(name, config(), externals)

ddeutil/workflow/log.py

@@ -0,0 +1,30 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import logging
+from functools import lru_cache
+
+from rich.console import Console
+from rich.logging import RichHandler
+
+console = Console(color_system="256", width=200, style="blue")
+
+
+@lru_cache
+def get_logger(module_name):
+    logger = logging.getLogger(module_name)
+    handler = RichHandler(
+        rich_tracebacks=True, console=console, tracebacks_show_locals=True
+    )
+    handler.setFormatter(
+        logging.Formatter(
+            "[ %(threadName)s:%(funcName)s:%(process)d ] - %(message)s"
+        )
+    )
+    logger.addHandler(handler)
+    logger.setLevel(logging.DEBUG)
+    return logger

ddeutil/workflow/on.py

@@ -10,29 +10,35 @@ from typing import Annotated, Literal
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from pydantic import BaseModel, ConfigDict, Field
-from pydantic.functional_validators import field_validator
+from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
 try:
-    from .__schedule import WEEKDAYS
     from .__types import DictData, DictStr
-    from .loader import CronJob, CronRunner, Loader
+    from .loader import Loader
+    from .scheduler import WEEKDAYS, CronJob, CronJobYear, CronRunner
 except ImportError:
-    from ddeutil.workflow.__scheduler import WEEKDAYS, CronJob, CronRunner
     from ddeutil.workflow.__types import DictData, DictStr
     from ddeutil.workflow.loader import Loader
+    from ddeutil.workflow.scheduler import (
+        WEEKDAYS,
+        CronJob,
+        CronJobYear,
+        CronRunner,
+    )
 
 
 def interval2crontab(
     interval: Literal["daily", "weekly", "monthly"],
-    day: str = "monday",
+    day: str | None = None,
     time: str = "00:00",
 ) -> str:
     """Return the crontab string that was generated from specific values.
 
     :param interval: A interval value that is one of 'daily', 'weekly', or
         'monthly'.
-    :param day: A day value that will be day of week.
+    :param day: A day value that will be day of week. The default value is
+        monday if it be weekly interval.
     :param time: A time value that passing with format '%H:%M'.
 
     Examples:
@@ -42,18 +48,23 @@ def interval2crontab(
         '18 30 * * 5'
         >>> interval2crontab(interval='monthly', time='00:00')
         '0 0 1 * *'
+        >>> interval2crontab(interval='monthly', day='tuesday', time='12:00')
+        '12 0 1 * 2'
     """
+    d: str = "*"
+    if interval == "weekly":
+        d = WEEKDAYS[(day or "monday")[:3].title()]
+    elif interval == "monthly" and day:
+        d = WEEKDAYS[day[:3].title()]
+
     h, m = tuple(
         i.lstrip("0") if i != "00" else "0" for i in time.split(":", maxsplit=1)
     )
-    return (
-        f"{h} {m} {'1' if interval == 'monthly' else '*'} * "
-        f"{WEEKDAYS[day[:3].title()] if interval == 'weekly' else '*'}"
-    )
+    return f"{h} {m} {'1' if interval == 'monthly' else '*'} * {d}"
 
 
-class Schedule(BaseModel):
-    """Schedule Model
+class On(BaseModel):
+    """On Model (Schedule)
 
     See Also:
         * ``generate()`` is the main usecase of this schedule object.
@@ -62,8 +73,17 @@ class Schedule(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     # NOTE: This is fields of the base schedule.
-    cronjob: Annotated[CronJob, Field(description="Cron job of this schedule")]
-    tz: Annotated[str, Field(description="A timezone string value")] = "Etc/UTC"
+    cronjob: Annotated[
+        CronJob,
+        Field(description="Cron job of this schedule"),
+    ]
+    tz: Annotated[
+        str,
+        Field(
+            description="A timezone string value",
+            alias="timezone",
+        ),
+    ] = "Etc/UTC"
     extras: Annotated[
         DictData,
         Field(
@@ -105,17 +125,36 @@ class Schedule(BaseModel):
         if loader.type != cls:
             raise ValueError(f"Type {loader.type} does not match with {cls}")
 
-        if "interval" in loader.data:
-            return cls.from_value(loader.data, externals=externals)
-        if "cronjob" not in loader.data:
-            raise ValueError("Config does not set ``cronjob`` value")
-        if "timezone" in loader.data:
-            return cls(
-                cronjob=loader.data["cronjob"],
-                tz=loader.data["timezone"],
+        loader_data: DictData = loader.data
+        if "interval" in loader_data:
+            return cls.model_validate(
+                obj=dict(
+                    cronjob=interval2crontab(
+                        **{
+                            v: loader_data[v]
+                            for v in loader_data
+                            if v in ("interval", "day", "time")
+                        }
+                    ),
+                    extras=externals,
+                    **loader_data,
+                )
+            )
+        if "cronjob" not in loader_data:
+            raise ValueError("Config does not set ``cronjob`` key")
+        return cls.model_validate(
+            obj=dict(
+                cronjob=loader_data.pop("cronjob"),
                 extras=externals,
+                **loader_data,
             )
-        return cls(cronjob=loader.data["cronjob"], extras=externals)
+        )
+
+    @model_validator(mode="before")
+    def __prepare_values(cls, values):
+        if tz := values.pop("tz", None):
+            values["timezone"] = tz
+        return values
 
     @field_validator("tz")
     def __validate_tz(cls, value: str):
@@ -136,8 +175,21 @@ class Schedule(BaseModel):
         """Return Cron runner object."""
         if not isinstance(start, datetime):
             start: datetime = datetime.fromisoformat(start)
-        return self.cronjob.schedule(date=(start.astimezone(ZoneInfo(self.tz))))
+        return self.cronjob.schedule(date=start, tz=self.tz)
 
 
-class AwsSchedule(Schedule):
-    """Implement Schedule for AWS Service."""
+class AwsOn(On):
+    """Implement On AWS Schedule for AWS Service like AWS Glue."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    # NOTE: This is fields of the base schedule.
+    cronjob: Annotated[
+        CronJobYear,
+        Field(description="Cron job of this schedule"),
+    ]
+
+    @field_validator("cronjob", mode="before")
+    def __prepare_cronjob(cls, value: str | CronJobYear) -> CronJobYear:
+        """Prepare crontab value that able to receive with string type."""
+        return CronJobYear(value) if isinstance(value, str) else value