ddeutil-workflow 0.0.26.post1__py3-none-any.whl → 0.0.28__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.26.post1"
+__version__: str = "0.0.28"

ddeutil/workflow/__init__.py

@@ -8,6 +8,7 @@ from .conf import (
     Config,
     Loader,
     Log,
+    config,
     env,
     get_log,
     get_logger,
@@ -24,6 +25,13 @@ from .exceptions import (
     UtilException,
     WorkflowException,
 )
+from .hook import (
+    ReturnTagFunc,
+    TagFunc,
+    extract_hook,
+    make_registry,
+    tag,
+)
 from .job import (
     Job,
     Strategy,
@@ -48,33 +56,30 @@ from .stage import (
     PyStage,
     Stage,
     TriggerStage,
-    extract_hook,
 )
-from .utils import (
+from .templates import (
     FILTERS,
     FilterFunc,
     FilterRegistry,
-    ReturnTagFunc,
-    TagFunc,
+    custom_filter,
+    get_args_const,
+    has_template,
+    make_filter_registry,
+    map_post_filter,
+    not_in_template,
+    param2template,
+    str2template,
+)
+from .utils import (
     batch,
     cross_product,
-    custom_filter,
     dash2underscore,
     delay,
     filter_func,
     gen_id,
-    get_args_const,
     get_diff_sec,
     get_dt_now,
-    has_template,
     make_exec,
-    make_filter_registry,
-    make_registry,
-    map_post_filter,
-    not_in_template,
-    param2template,
-    str2template,
-    tag,
 )
 from .workflow import (
     Workflow,

ddeutil/workflow/api/api.py

@@ -5,25 +5,21 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
-import asyncio
 import contextlib
-import uuid
 from collections.abc import AsyncIterator
 from datetime import datetime, timedelta
-from queue import Empty, Queue
 from typing import TypedDict
 
 from dotenv import load_dotenv
 from fastapi import FastAPI
 from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import UJSONResponse
-from pydantic import BaseModel
 
 from ..__about__ import __version__
 from ..conf import config, get_logger
 from ..scheduler import ReleaseThread, ReleaseThreads
 from ..workflow import WorkflowQueue, WorkflowTask
-from .repeat import repeat_at, repeat_every
+from .repeat import repeat_at
 
 load_dotenv()
 logger = get_logger("ddeutil.workflow")
@@ -32,11 +28,6 @@ logger = get_logger("ddeutil.workflow")
 class State(TypedDict):
     """TypeDict for State of FastAPI application."""
 
-    # NOTE: For upper queue route.
-    upper_queue: Queue
-    upper_result: dict[str, str]
-
-    # NOTE: For schedule listener.
     scheduler: list[str]
     workflow_threads: ReleaseThreads
     workflow_tasks: list[WorkflowTask]
@@ -46,15 +37,11 @@ class State(TypedDict):
 @contextlib.asynccontextmanager
 async def lifespan(a: FastAPI) -> AsyncIterator[State]:
     """Lifespan function for the FastAPI application."""
-    a.state.upper_queue = Queue()
-    a.state.upper_result = {}
     a.state.scheduler = []
     a.state.workflow_threads = {}
     a.state.workflow_tasks = []
     a.state.workflow_queue = {}
 
-    await asyncio.create_task(broker_upper_messages())
-
     yield {
         "upper_queue": a.state.upper_queue,
         "upper_result": a.state.upper_result,
@@ -87,50 +74,11 @@ app = FastAPI(
 app.add_middleware(GZipMiddleware, minimum_size=1000)
 
 
-@repeat_every(seconds=10)
-async def broker_upper_messages():
-    """Broker for receive message from the `/upper` path and change it to upper
-    case. This broker use interval running in background every 10 seconds.
-    """
-    for _ in range(10):
-        try:
-            obj = app.state.upper_queue.get_nowait()
-            app.state.upper_result[obj["request_id"]] = obj["text"].upper()
-            logger.info(f"Upper message: {app.state.upper_result}")
-        except Empty:
-            pass
-    await asyncio.sleep(0.0001)
-
-
-class Payload(BaseModel):
-    text: str
-
-
-async def get_result(request_id: str) -> dict[str, str]:
-    """Get data from output dict that global."""
-    while True:
-        if request_id in app.state.upper_result:
-            result: str = app.state.upper_result[request_id]
-            del app.state.upper_result[request_id]
-            return {"message": result}
-        await asyncio.sleep(0.0025)
-
-
 @app.get("/")
 async def health():
     return {"message": "Workflow API already start up"}
 
 
-@app.post(f"{config.prefix_path}/upper")
-async def message_upper(payload: Payload):
-    """Convert message from any case to the upper case."""
-    request_id: str = str(uuid.uuid4())
-    app.state.upper_queue.put(
-        {"text": payload.text, "request_id": request_id},
-    )
-    return await get_result(request_id)
-
-
 # NOTE: Enable the workflow route.
 if config.enable_route_workflow:
     from .route import workflow_route

ddeutil/workflow/conf.py

@@ -24,9 +24,11 @@ from typing_extensions import Self
 
 from .__types import DictData, TupleStr
 
+PREFIX: str = "WORKFLOW"
+
 
 def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
-    return os.getenv(f"WORKFLOW_{var}", default)
+    return os.getenv(f"{PREFIX}_{var.upper().replace(' ', '_')}", default)
 
 
 def glob_files(path: Path) -> Iterator[Path]:  # pragma: no cov
@@ -80,14 +82,18 @@ def get_logger(name: str):
 
 
 class Config:  # pragma: no cov
-    """Config object for keeping application configuration on current session
+    """Config object for keeping core configurations on the current session
     without changing when if the application still running.
+
+        The config value can change when you call that config property again.
     """
 
+    __slots__ = ()
+
     # NOTE: Core
     @property
     def root_path(self) -> Path:
-        return Path(env("ROOT_PATH", "."))
+        return Path(env("CORE_ROOT_PATH", "."))
 
     @property
     def conf_path(self) -> Path:
@@ -95,7 +101,7 @@ class Config:  # pragma: no cov
 
         :rtype: Path
         """
-        return self.root_path / env("CORE_PATH_CONF", "conf")
+        return self.root_path / env("CORE_CONF_PATH", "conf")
 
     @property
     def tz(self) -> ZoneInfo:
@@ -108,15 +114,13 @@ class Config:  # pragma: no cov
     # NOTE: Register
     @property
     def regis_hook(self) -> list[str]:
-        regis_hook_str: str = env(
-            "CORE_REGISTRY", "src,src.ddeutil.workflow,tests,tests.utils"
-        )
+        regis_hook_str: str = env("CORE_REGISTRY", "src")
         return [r.strip() for r in regis_hook_str.split(",")]
 
     @property
     def regis_filter(self) -> list[str]:
         regis_filter_str: str = env(
-            "CORE_REGISTRY_FILTER", "ddeutil.workflow.utils"
+            "CORE_REGISTRY_FILTER", "ddeutil.workflow.templates"
         )
         return [r.strip() for r in regis_filter_str.split(",")]
 
@@ -312,6 +316,10 @@ class SimLoad:
         )
 
 
+config = Config()
+logger = get_logger("ddeutil.workflow")
+
+
 class Loader(SimLoad):
     """Loader Object that get the config `yaml` file from current path.
 
@@ -337,21 +345,17 @@ class Loader(SimLoad):
         :rtype: Iterator[tuple[str, DictData]]
         """
         return super().finds(
-            obj=obj, conf=Config(), included=included, excluded=excluded
+            obj=obj, conf=config, included=included, excluded=excluded
         )
 
     def __init__(self, name: str, externals: DictData) -> None:
-        super().__init__(name, conf=Config(), externals=externals)
-
-
-config = Config()
-logger = get_logger("ddeutil.workflow")
+        super().__init__(name, conf=config, externals=externals)
 
 
 class BaseLog(BaseModel, ABC):
     """Base Log Pydantic Model with abstraction class property that implement
     only model fields. This model should to use with inherit to logging
-    sub-class like file, sqlite, etc.
+    subclass like file, sqlite, etc.
     """
 
     name: str = Field(description="A workflow name.")
@@ -359,9 +363,7 @@ class BaseLog(BaseModel, ABC):
     type: str = Field(description="A running type before logging.")
     context: DictData = Field(
         default_factory=dict,
-        description=(
-            "A context data that receive from a workflow execution result.",
-        ),
+        description="A context that receive from a workflow execution result.",
     )
     parent_run_id: Optional[str] = Field(default=None)
     run_id: str
@@ -388,7 +390,7 @@ class BaseLog(BaseModel, ABC):
 
 class FileLog(BaseLog):
     """File Log Pydantic Model that use to saving log data from result of
-    workflow execution. It inherit from BaseLog model that implement the
+    workflow execution. It inherits from BaseLog model that implement the
     ``self.save`` method for file.
     """
 
@@ -427,8 +429,8 @@ class FileLog(BaseLog):
         workflow name and release values. If a release does not pass to an input
         argument, it will return the latest release from the current log path.
 
-        :param name:
-        :param release:
+        :param name: A workflow name that want to search log.
+        :param release: A release datetime that want to search log.
 
         :raise FileNotFoundError:
         :raise NotImplementedError:
@@ -492,8 +494,14 @@ class FileLog(BaseLog):
 
         :rtype: Self
         """
+        from .utils import cut_id
+
         # NOTE: Check environ variable was set for real writing.
         if not config.enable_write_log:
+            logger.debug(
+                f"({cut_id(self.run_id)}) [LOG]: Skip writing log cause "
+                f"config was set"
+            )
             return self
 
         log_file: Path = self.pointer() / f"{self.run_id}.log"
@@ -522,7 +530,20 @@ class SQLiteLog(BaseLog):  # pragma: no cov
         primary key ( run_id )
         """
 
-    def save(self, excluded: list[str] | None) -> None:
+    def save(self, excluded: list[str] | None) -> SQLiteLog:
+        """Save logging data that receive a context data from a workflow
+        execution result.
+        """
+        from .utils import cut_id
+
+        # NOTE: Check environ variable was set for real writing.
+        if not config.enable_write_log:
+            logger.debug(
+                f"({cut_id(self.run_id)}) [LOG]: Skip writing log cause "
+                f"config was set"
+            )
+            return self
+
         raise NotImplementedError("SQLiteLog does not implement yet.")
 
 
@@ -532,7 +553,7 @@ Log = Union[
 ]
 
 
-def get_log() -> Log:  # pragma: no cov
+def get_log() -> type[Log]:  # pragma: no cov
     if config.log_path.is_file():
         return SQLiteLog
     return FileLog

ddeutil/workflow/cron.py

@@ -32,10 +32,10 @@ def interval2crontab(
 ) -> str:
     """Return the crontab string that was generated from specific values.
 
-    :param interval: A interval value that is one of 'daily', 'weekly', or
+    :param interval: An interval value that is one of 'daily', 'weekly', or
         'monthly'.
     :param day: A day value that will be day of week. The default value is
-        monday if it be weekly interval.
+        monday if it is weekly interval.
     :param time: A time value that passing with format '%H:%M'.
 
     Examples:
@@ -50,9 +50,9 @@ def interval2crontab(
     """
     d: str = "*"
     if interval == "weekly":
-        d = WEEKDAYS[(day or "monday")[:3].title()]
+        d = str(WEEKDAYS[(day or "monday")[:3].title()])
     elif interval == "monthly" and day:
-        d = WEEKDAYS[day[:3].title()]
+        d = str(WEEKDAYS[day[:3].title()])
 
     h, m = tuple(
         i.lstrip("0") if i != "00" else "0" for i in time.split(":", maxsplit=1)
@@ -95,7 +95,7 @@ class On(BaseModel):
 
         :param value: A mapping value that will generate crontab before create
             schedule model.
-        :param externals: A extras external parameter that will keep in extras.
+        :param externals: An extras external parameter that will keep in extras.
         """
         passing: DictStr = {}
         if "timezone" in value:
@@ -114,8 +114,8 @@ class On(BaseModel):
         """Constructor from the name of config that will use loader object for
         getting the data.
 
-        :param name: A name of config that will getting from loader.
-        :param externals: A extras external parameter that will keep in extras.
+        :param name: A name of config that will get from loader.
+        :param externals: An extras external parameter that will keep in extras.
         """
         externals: DictData = externals or {}
         loader: Loader = Loader(name, externals=externals)

ddeutil/workflow/exceptions.py

@@ -4,7 +4,7 @@
 # license information.
 # ------------------------------------------------------------------------------
 """Exception objects for this package do not do anything because I want to
-create the lightweight workflow package. So, this module do just a exception
+create the lightweight workflow package. So, this module do just an exception
 annotate for handle error only.
 """
 from __future__ import annotations
@@ -29,6 +29,3 @@ class WorkflowFailException(WorkflowException): ...
 
 
 class ParamValueException(WorkflowException): ...
-
-
-class CliException(BaseWorkflowException): ...

ddeutil/workflow/hook.py

@@ -0,0 +1,168 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import inspect
+import logging
+from dataclasses import dataclass
+from functools import wraps
+from importlib import import_module
+from typing import Any, Callable, Protocol, TypeVar
+
+try:
+    from typing import ParamSpec
+except ImportError:
+    from typing_extensions import ParamSpec
+
+from ddeutil.core import lazy
+
+from .__types import Re
+from .conf import config
+
+T = TypeVar("T")
+P = ParamSpec("P")
+
+logger = logging.getLogger("ddeutil.workflow")
+
+
+class TagFunc(Protocol):
+    """Tag Function Protocol"""
+
+    name: str
+    tag: str
+
+    def __call__(self, *args, **kwargs): ...  # pragma: no cov
+
+
+ReturnTagFunc = Callable[P, TagFunc]
+DecoratorTagFunc = Callable[[Callable[[...], Any]], ReturnTagFunc]
+
+
+def tag(
+    name: str, alias: str | None = None
+) -> DecoratorTagFunc:  # pragma: no cov
+    """Tag decorator function that set function attributes, ``tag`` and ``name``
+    for making registries variable.
+
+    :param: name: A tag name for make different use-case of a function.
+    :param: alias: A alias function name that keeping in registries. If this
+        value does not supply, it will use original function name from __name__.
+    :rtype: Callable[P, TagFunc]
+    """
+
+    def func_internal(func: Callable[[...], Any]) -> ReturnTagFunc:
+        func.tag = name
+        func.name = alias or func.__name__.replace("_", "-")
+
+        @wraps(func)
+        def wrapped(*args, **kwargs):
+            # NOTE: Able to do anything before calling hook function.
+            return func(*args, **kwargs)
+
+        return wrapped
+
+    return func_internal
+
+
+Registry = dict[str, Callable[[], TagFunc]]
+
+
+def make_registry(submodule: str) -> dict[str, Registry]:
+    """Return registries of all functions that able to called with task.
+
+    :param submodule: A module prefix that want to import registry.
+    :rtype: dict[str, Registry]
+    """
+    rs: dict[str, Registry] = {}
+    for module in config.regis_hook:
+        # NOTE: try to sequential import task functions
+        try:
+            importer = import_module(f"{module}.{submodule}")
+        except ModuleNotFoundError:
+            continue
+
+        for fstr, func in inspect.getmembers(importer, inspect.isfunction):
+            # NOTE: check function attribute that already set tag by
+            #   ``utils.tag`` decorator.
+            if not (hasattr(func, "tag") and hasattr(func, "name")):
+                continue
+
+            # NOTE: Define type of the func value.
+            func: TagFunc
+
+            # NOTE: Create new register name if it not exists
+            if func.name not in rs:
+                rs[func.name] = {func.tag: lazy(f"{module}.{submodule}.{fstr}")}
+                continue
+
+            if func.tag in rs[func.name]:
+                raise ValueError(
+                    f"The tag {func.tag!r} already exists on "
+                    f"{module}.{submodule}, you should change this tag name or "
+                    f"change it func name."
+                )
+            rs[func.name][func.tag] = lazy(f"{module}.{submodule}.{fstr}")
+
+    return rs
+
+
+@dataclass(frozen=True)
+class HookSearchData:
+    """Hook Search dataclass that use for receive regular expression grouping
+    dict from searching hook string value.
+    """
+
+    path: str
+    func: str
+    tag: str
+
+
+def extract_hook(hook: str) -> Callable[[], TagFunc]:
+    """Extract Hook function from string value to hook partial function that
+    does run it at runtime.
+
+    :raise NotImplementedError: When the searching hook's function result does
+        not exist in the registry.
+    :raise NotImplementedError: When the searching hook's tag result does not
+        exist in the registry with its function key.
+
+    :param hook: A hook value that able to match with Task regex.
+
+        The format of hook value should contain 3 regular expression groups
+    which match with the below config format:
+
+        >>> "^(?P<path>[^/@]+)/(?P<func>[^@]+)@(?P<tag>.+)$"
+
+    Examples:
+        >>> extract_hook("tasks/el-postgres-to-delta@polars")
+        ...
+        >>> extract_hook("tasks/return-type-not-valid@raise")
+        ...
+
+    :rtype: Callable[[], TagFunc]
+    """
+    if not (found := Re.RE_TASK_FMT.search(hook)):
+        raise ValueError(
+            f"Hook {hook!r} does not match with hook format regex."
+        )
+
+    # NOTE: Pass the searching hook string to `path`, `func`, and `tag`.
+    hook: HookSearchData = HookSearchData(**found.groupdict())
+
+    # NOTE: Registry object should implement on this package only.
+    rgt: dict[str, Registry] = make_registry(f"{hook.path}")
+    if hook.func not in rgt:
+        raise NotImplementedError(
+            f"``REGISTER-MODULES.{hook.path}.registries`` does not "
+            f"implement registry: {hook.func!r}."
+        )
+
+    if hook.tag not in rgt[hook.func]:
+        raise NotImplementedError(
+            f"tag: {hook.tag!r} does not found on registry func: "
+            f"``REGISTER-MODULES.{hook.path}.registries.{hook.func}``"
+        )
+    return rgt[hook.func][hook.tag]