ddeutil-workflow 0.0.26.post0__py3-none-any.whl → 0.0.27__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.26.post0"
+__version__: str = "0.0.27"

ddeutil/workflow/__init__.py

@@ -3,11 +3,15 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-from .__cron import CronRunner
+from .__cron import CronJob, CronRunner
 from .conf import (
     Config,
-    FileLog,
     Loader,
+    Log,
+    config,
+    env,
+    get_log,
+    get_logger,
 )
 from .cron import (
     On,
@@ -21,6 +25,13 @@ from .exceptions import (
     UtilException,
     WorkflowException,
 )
+from .hook import (
+    ReturnTagFunc,
+    TagFunc,
+    extract_hook,
+    make_registry,
+    tag,
+)
 from .job import (
     Job,
     Strategy,
@@ -45,33 +56,30 @@ from .stage import (
     PyStage,
     Stage,
     TriggerStage,
-    extract_hook,
 )
-from .utils import (
+from .templates import (
     FILTERS,
     FilterFunc,
     FilterRegistry,
-    ReturnTagFunc,
-    TagFunc,
+    custom_filter,
+    get_args_const,
+    has_template,
+    make_filter_registry,
+    map_post_filter,
+    not_in_template,
+    param2template,
+    str2template,
+)
+from .utils import (
     batch,
     cross_product,
-    custom_filter,
     dash2underscore,
     delay,
     filter_func,
     gen_id,
-    get_args_const,
     get_diff_sec,
     get_dt_now,
-    has_template,
     make_exec,
-    make_filter_registry,
-    make_registry,
-    map_post_filter,
-    not_in_template,
-    param2template,
-    str2template,
-    tag,
 )
 from .workflow import (
     Workflow,

ddeutil/workflow/conf.py

@@ -13,27 +13,30 @@ from collections.abc import Iterator
 from datetime import datetime, timedelta
 from functools import cached_property, lru_cache
 from pathlib import Path
-from typing import ClassVar, Optional, TypeVar, Union
+from typing import ClassVar, Optional, Union
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
 from ddeutil.io import YamlFlResolve
-from dotenv import load_dotenv
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, TupleStr
 
-AnyModel = TypeVar("AnyModel", bound=BaseModel)
-AnyModelType = type[AnyModel]
 
-load_dotenv()
+def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
+    return os.getenv(f"WORKFLOW_{var}", default)
+
+
+def glob_files(path: Path) -> Iterator[Path]:  # pragma: no cov
+    yield from (file for file in path.rglob("*") if file.is_file())
 
-env = os.getenv
 
 __all__: TupleStr = (
+    "env",
     "get_logger",
+    "get_log",
     "Config",
     "SimLoad",
     "Loader",
@@ -52,6 +55,14 @@ def get_logger(name: str):
     :param name: A module name that want to log.
     """
     lg = logging.getLogger(name)
+
+    # NOTE: Developers using this package can then disable all logging just for
+    #   this package by;
+    #
+    #   `logging.getLogger('ddeutil.workflow').propagate = False`
+    #
+    lg.addHandler(logging.NullHandler())
+
     formatter = logging.Formatter(
         fmt=(
             "%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d, "
@@ -68,115 +79,139 @@ def get_logger(name: str):
     return lg
 
 
-class Config:
+class Config:  # pragma: no cov
     """Config object for keeping application configuration on current session
     without changing when if the application still running.
     """
 
     # NOTE: Core
-    root_path: Path = Path(os.getenv("WORKFLOW_ROOT_PATH", "."))
-    tz: ZoneInfo = ZoneInfo(env("WORKFLOW_CORE_TIMEZONE", "UTC"))
-    gen_id_simple_mode: bool = str2bool(
-        os.getenv("WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE", "true")
-    )
+    @property
+    def root_path(self) -> Path:
+        return Path(env("ROOT_PATH", "."))
+
+    @property
+    def conf_path(self) -> Path:
+        """Config path that use root_path class argument for this construction.
+
+        :rtype: Path
+        """
+        return self.root_path / env("CORE_PATH_CONF", "conf")
+
+    @property
+    def tz(self) -> ZoneInfo:
+        return ZoneInfo(env("CORE_TIMEZONE", "UTC"))
+
+    @property
+    def gen_id_simple_mode(self) -> bool:
+        return str2bool(env("CORE_GENERATE_ID_SIMPLE_MODE", "true"))
 
     # NOTE: Register
-    regis_hook_str: str = os.getenv(
-        "WORKFLOW_CORE_REGISTRY", "src,src.ddeutil.workflow,tests,tests.utils"
-    )
-    regis_filter_str: str = os.getenv(
-        "WORKFLOW_CORE_REGISTRY_FILTER", "ddeutil.workflow.utils"
-    )
+    @property
+    def regis_hook(self) -> list[str]:
+        regis_hook_str: str = env("CORE_REGISTRY", "src")
+        return [r.strip() for r in regis_hook_str.split(",")]
+
+    @property
+    def regis_filter(self) -> list[str]:
+        regis_filter_str: str = env(
+            "CORE_REGISTRY_FILTER", "ddeutil.workflow.templates"
+        )
+        return [r.strip() for r in regis_filter_str.split(",")]
 
     # NOTE: Logging
-    debug: bool = str2bool(os.getenv("WORKFLOW_LOG_DEBUG_MODE", "true"))
-    enable_write_log: bool = str2bool(
-        os.getenv("WORKFLOW_LOG_ENABLE_WRITE", "false")
-    )
-    log_path: Path = Path(os.getenv("WORKFLOW_LOG_PATH", "./logs"))
+    @property
+    def log_path(self) -> Path:
+        return Path(env("LOG_PATH", "./logs"))
+
+    @property
+    def debug(self) -> bool:
+        return str2bool(env("LOG_DEBUG_MODE", "true"))
+
+    @property
+    def enable_write_log(self) -> bool:
+        return str2bool(env("LOG_ENABLE_WRITE", "false"))
 
     # NOTE: Stage
-    stage_raise_error: bool = str2bool(
-        env("WORKFLOW_CORE_STAGE_RAISE_ERROR", "false")
-    )
-    stage_default_id: bool = str2bool(
-        env("WORKFLOW_CORE_STAGE_DEFAULT_ID", "false")
-    )
+    @property
+    def stage_raise_error(self) -> bool:
+        return str2bool(env("CORE_STAGE_RAISE_ERROR", "false"))
 
-    # NOTE: Job
-    job_raise_error: bool = str2bool(
-        env("WORKFLOW_CORE_JOB_RAISE_ERROR", "true")
-    )
-    job_default_id: bool = str2bool(
-        env("WORKFLOW_CORE_JOB_DEFAULT_ID", "false")
-    )
+    @property
+    def stage_default_id(self) -> bool:
+        return str2bool(env("CORE_STAGE_DEFAULT_ID", "false"))
 
-    # NOTE: Workflow
-    max_job_parallel: int = int(env("WORKFLOW_CORE_MAX_JOB_PARALLEL", "2"))
-    max_job_exec_timeout: int = int(
-        env("WORKFLOW_CORE_MAX_JOB_EXEC_TIMEOUT", "600")
-    )
-    max_poking_pool_worker: int = int(
-        os.getenv("WORKFLOW_CORE_MAX_NUM_POKING", "4")
-    )
-    max_on_per_workflow: int = int(
-        env("WORKFLOW_CORE_MAX_CRON_PER_WORKFLOW", "5")
-    )
-    max_queue_complete_hist: int = int(
-        os.getenv("WORKFLOW_CORE_MAX_QUEUE_COMPLETE_HIST", "16")
-    )
+    # NOTE: Job
+    @property
+    def job_raise_error(self) -> bool:
+        return str2bool(env("CORE_JOB_RAISE_ERROR", "true"))
 
-    # NOTE: Schedule App
-    max_schedule_process: int = int(env("WORKFLOW_APP_MAX_PROCESS", "2"))
-    max_schedule_per_process: int = int(
-        env("WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS", "100")
-    )
-    stop_boundary_delta_str: str = env(
-        "WORKFLOW_APP_STOP_BOUNDARY_DELTA", '{"minutes": 5, "seconds": 20}'
-    )
+    @property
+    def job_default_id(self) -> bool:
+        return str2bool(env("CORE_JOB_DEFAULT_ID", "false"))
 
-    # NOTE: API
-    prefix_path: str = env("WORKFLOW_API_PREFIX_PATH", "/api/v1")
-    enable_route_workflow: bool = str2bool(
-        env("WORKFLOW_API_ENABLE_ROUTE_WORKFLOW", "true")
-    )
-    enable_route_schedule: bool = str2bool(
-        env("WORKFLOW_API_ENABLE_ROUTE_SCHEDULE", "true")
-    )
+    # NOTE: Workflow
+    @property
+    def max_job_parallel(self) -> int:
+        max_job_parallel = int(env("CORE_MAX_JOB_PARALLEL", "2"))
 
-    def __init__(self) -> None:
         # VALIDATE: the MAX_JOB_PARALLEL value should not less than 0.
-        if self.max_job_parallel < 0:
+        if max_job_parallel < 0:
             raise ValueError(
-                f"``MAX_JOB_PARALLEL`` should more than 0 but got "
-                f"{self.max_job_parallel}."
+                f"``WORKFLOW_MAX_JOB_PARALLEL`` should more than 0 but got "
+                f"{max_job_parallel}."
             )
+        return max_job_parallel
+
+    @property
+    def max_job_exec_timeout(self) -> int:
+        return int(env("CORE_MAX_JOB_EXEC_TIMEOUT", "600"))
+
+    @property
+    def max_poking_pool_worker(self) -> int:
+        return int(env("CORE_MAX_NUM_POKING", "4"))
+
+    @property
+    def max_on_per_workflow(self) -> int:
+        return int(env("CORE_MAX_CRON_PER_WORKFLOW", "5"))
+
+    @property
+    def max_queue_complete_hist(self) -> int:
+        return int(env("CORE_MAX_QUEUE_COMPLETE_HIST", "16"))
+
+    # NOTE: Schedule App
+    @property
+    def max_schedule_process(self) -> int:
+        return int(env("APP_MAX_PROCESS", "2"))
 
+    @property
+    def max_schedule_per_process(self) -> int:
+        return int(env("APP_MAX_SCHEDULE_PER_PROCESS", "100"))
+
+    @property
+    def stop_boundary_delta(self) -> timedelta:
+        stop_boundary_delta_str: str = env(
+            "APP_STOP_BOUNDARY_DELTA", '{"minutes": 5, "seconds": 20}'
+        )
         try:
-            self.stop_boundary_delta: timedelta = timedelta(
-                **json.loads(self.stop_boundary_delta_str)
-            )
+            return timedelta(**json.loads(stop_boundary_delta_str))
         except Exception as err:
             raise ValueError(
                 "Config ``WORKFLOW_APP_STOP_BOUNDARY_DELTA`` can not parsing to"
-                f"timedelta with {self.stop_boundary_delta_str}."
+                f"timedelta with {stop_boundary_delta_str}."
             ) from err
 
+    # NOTE: API
     @property
-    def conf_path(self) -> Path:
-        """Config path that use root_path class argument for this construction.
-
-        :rtype: Path
-        """
-        return self.root_path / os.getenv("WORKFLOW_CORE_PATH_CONF", "conf")
+    def prefix_path(self) -> str:
+        return env("API_PREFIX_PATH", "/api/v1")
 
     @property
-    def regis_hook(self) -> list[str]:
-        return [r.strip() for r in self.regis_hook_str.split(",")]
+    def enable_route_workflow(self) -> bool:
+        return str2bool(env("API_ENABLE_ROUTE_WORKFLOW", "true"))
 
     @property
-    def regis_filter(self) -> list[str]:
-        return [r.strip() for r in self.regis_filter_str.split(",")]
+    def enable_route_schedule(self) -> bool:
+        return str2bool(env("API_ENABLE_ROUTE_SCHEDULE", "true"))
 
 
 class SimLoad:
@@ -206,14 +241,9 @@ class SimLoad:
         externals: DictData | None = None,
     ) -> None:
         self.data: DictData = {}
-        for file in conf.conf_path.rglob("*"):
-            if not file.is_file():
-                continue
-
-            if data := self.filter_suffix(
-                file,
-                name,
-            ):
+        for file in glob_files(conf.conf_path):
+
+            if data := self.filter_suffix(file, name):
                 self.data = data
 
         # VALIDATE: check the data that reading should not empty.
@@ -245,10 +275,7 @@ class SimLoad:
         :rtype: Iterator[tuple[str, DictData]]
         """
         exclude: list[str] = excluded or []
-        for file in conf.conf_path.rglob("*"):
-
-            if not file.is_file():
-                continue
+        for file in glob_files(conf.conf_path):
 
             for key, data in cls.filter_suffix(file).items():
 
@@ -274,7 +301,7 @@ class SimLoad:
         """Return object of string type which implement on any registry. The
         object type.
 
-        :rtype: AnyModelType
+        :rtype: str
         """
         if _typ := self.data.get("type"):
             return _typ
@@ -283,6 +310,10 @@ class SimLoad:
         )
 
 
+config = Config()
+logger = get_logger("ddeutil.workflow")
+
+
 class Loader(SimLoad):
     """Loader Object that get the config `yaml` file from current path.
 
@@ -308,15 +339,11 @@ class Loader(SimLoad):
         :rtype: Iterator[tuple[str, DictData]]
         """
         return super().finds(
-            obj=obj, conf=Config(), included=included, excluded=excluded
+            obj=obj, conf=config, included=included, excluded=excluded
         )
 
     def __init__(self, name: str, externals: DictData) -> None:
-        super().__init__(name, conf=Config(), externals=externals)
-
-
-config = Config()
-logger = get_logger("ddeutil.workflow")
+        super().__init__(name, conf=config, externals=externals)
 
 
 class BaseLog(BaseModel, ABC):
@@ -398,8 +425,8 @@ class FileLog(BaseLog):
         workflow name and release values. If a release does not pass to an input
         argument, it will return the latest release from the current log path.
 
-        :param name:
-        :param release:
+        :param name: A workflow name that want to search log.
+        :param release: A release datetime that want to search log.
 
         :raise FileNotFoundError:
         :raise NotImplementedError:
@@ -463,8 +490,14 @@ class FileLog(BaseLog):
 
         :rtype: Self
         """
+        from .utils import cut_id
+
         # NOTE: Check environ variable was set for real writing.
         if not config.enable_write_log:
+            logger.debug(
+                f"({cut_id(self.run_id)}) [LOG]: Skip writing log cause "
+                f"config was set"
+            )
             return self
 
         log_file: Path = self.pointer() / f"{self.run_id}.log"
@@ -481,7 +514,32 @@ class FileLog(BaseLog):
 
 class SQLiteLog(BaseLog):  # pragma: no cov
 
+    table: str = "workflow_log"
+    ddl: str = """
+        workflow        str,
+        release         int,
+        type            str,
+        context         json,
+        parent_run_id   int,
+        run_id          int,
+        update          datetime
+        primary key ( run_id )
+        """
+
     def save(self, excluded: list[str] | None) -> None:
+        """Save logging data that receive a context data from a workflow
+        execution result.
+        """
+        from .utils import cut_id
+
+        # NOTE: Check environ variable was set for real writing.
+        if not config.enable_write_log:
+            logger.debug(
+                f"({cut_id(self.run_id)}) [LOG]: Skip writing log cause "
+                f"config was set"
+            )
+            return self
+
         raise NotImplementedError("SQLiteLog does not implement yet.")
 
 
@@ -489,3 +547,9 @@ Log = Union[
     FileLog,
     SQLiteLog,
 ]
+
+
+def get_log() -> Log:  # pragma: no cov
+    if config.log_path.is_file():
+        return SQLiteLog
+    return FileLog

ddeutil/workflow/exceptions.py

@@ -29,6 +29,3 @@ class WorkflowFailException(WorkflowException): ...
 
 
 class ParamValueException(WorkflowException): ...
-
-
-class CliException(BaseWorkflowException): ...

ddeutil/workflow/hook.py

@@ -0,0 +1,153 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import inspect
+import logging
+from dataclasses import dataclass
+from functools import wraps
+from importlib import import_module
+from typing import Any, Callable, Protocol, TypeVar
+
+try:
+    from typing import ParamSpec
+except ImportError:
+    from typing_extensions import ParamSpec
+
+from ddeutil.core import lazy
+
+from .__types import Re
+from .conf import config
+
+T = TypeVar("T")
+P = ParamSpec("P")
+
+logger = logging.getLogger("ddeutil.workflow")
+
+
+class TagFunc(Protocol):
+    """Tag Function Protocol"""
+
+    name: str
+    tag: str
+
+    def __call__(self, *args, **kwargs): ...  # pragma: no cov
+
+
+ReturnTagFunc = Callable[P, TagFunc]
+DecoratorTagFunc = Callable[[Callable[[...], Any]], ReturnTagFunc]
+
+
+def tag(
+    name: str, alias: str | None = None
+) -> DecoratorTagFunc:  # pragma: no cov
+    """Tag decorator function that set function attributes, ``tag`` and ``name``
+    for making registries variable.
+
+    :param: name: A tag name for make different use-case of a function.
+    :param: alias: A alias function name that keeping in registries. If this
+        value does not supply, it will use original function name from __name__.
+    :rtype: Callable[P, TagFunc]
+    """
+
+    def func_internal(func: Callable[[...], Any]) -> ReturnTagFunc:
+        func.tag = name
+        func.name = alias or func.__name__.replace("_", "-")
+
+        @wraps(func)
+        def wrapped(*args, **kwargs):
+            # NOTE: Able to do anything before calling hook function.
+            return func(*args, **kwargs)
+
+        return wrapped
+
+    return func_internal
+
+
+Registry = dict[str, Callable[[], TagFunc]]
+
+
+def make_registry(submodule: str) -> dict[str, Registry]:
+    """Return registries of all functions that able to called with task.
+
+    :param submodule: A module prefix that want to import registry.
+    :rtype: dict[str, Registry]
+    """
+    rs: dict[str, Registry] = {}
+    for module in config.regis_hook:
+        # NOTE: try to sequential import task functions
+        try:
+            importer = import_module(f"{module}.{submodule}")
+        except ModuleNotFoundError:
+            continue
+
+        for fstr, func in inspect.getmembers(importer, inspect.isfunction):
+            # NOTE: check function attribute that already set tag by
+            #   ``utils.tag`` decorator.
+            if not hasattr(func, "tag"):
+                continue
+
+            # NOTE: Create new register name if it not exists
+            if func.name not in rs:
+                rs[func.name] = {func.tag: lazy(f"{module}.{submodule}.{fstr}")}
+                continue
+
+            if func.tag in rs[func.name]:
+                raise ValueError(
+                    f"The tag {func.tag!r} already exists on "
+                    f"{module}.{submodule}, you should change this tag name or "
+                    f"change it func name."
+                )
+            rs[func.name][func.tag] = lazy(f"{module}.{submodule}.{fstr}")
+
+    return rs
+
+
+@dataclass(frozen=True)
+class HookSearchData:
+    """Hook Search dataclass that use for receive regular expression grouping
+    dict from searching hook string value.
+    """
+
+    path: str
+    func: str
+    tag: str
+
+
+def extract_hook(hook: str) -> Callable[[], TagFunc]:
+    """Extract Hook function from string value to hook partial function that
+    does run it at runtime.
+
+    :raise NotImplementedError: When the searching hook's function result does
+        not exist in the registry.
+    :raise NotImplementedError: When the searching hook's tag result does not
+        exists in the registry with its function key.
+
+    :param hook: A hook value that able to match with Task regex.
+    :rtype: Callable[[], TagFunc]
+    """
+    if not (found := Re.RE_TASK_FMT.search(hook)):
+        raise ValueError(
+            f"Hook {hook!r} does not match with hook format regex."
+        )
+
+    # NOTE: Pass the searching hook string to `path`, `func`, and `tag`.
+    hook: HookSearchData = HookSearchData(**found.groupdict())
+
+    # NOTE: Registry object should implement on this package only.
+    rgt: dict[str, Registry] = make_registry(f"{hook.path}")
+    if hook.func not in rgt:
+        raise NotImplementedError(
+            f"``REGISTER-MODULES.{hook.path}.registries`` does not "
+            f"implement registry: {hook.func!r}."
+        )
+
+    if hook.tag not in rgt[hook.func]:
+        raise NotImplementedError(
+            f"tag: {hook.tag!r} does not found on registry func: "
+            f"``REGISTER-MODULES.{hook.path}.registries.{hook.func}``"
+        )
+    return rgt[hook.func][hook.tag]

ddeutil/workflow/job.py

@@ -38,13 +38,13 @@ from .exceptions import (
 )
 from .result import Result
 from .stage import Stage
+from .templates import has_template
 from .utils import (
     cross_product,
     cut_id,
     dash2underscore,
     filter_func,
     gen_id,
-    has_template,
 )
 
 logger = get_logger("ddeutil.workflow")

ddeutil/workflow/scheduler.py

@@ -51,7 +51,7 @@ except ImportError:  # pragma: no cov
 
 from .__cron import CronRunner
 from .__types import DictData, TupleStr
-from .conf import FileLog, Loader, Log, config, get_logger
+from .conf import Loader, Log, config, get_log, get_logger
 from .cron import On
 from .exceptions import WorkflowException
 from .utils import (
@@ -493,7 +493,7 @@ def schedule_control(
             "Should install schedule package before use this module."
         ) from None
 
-    log: type[Log] = log or FileLog
+    log: type[Log] = log or get_log()
     scheduler: Scheduler = Scheduler()
     start_date: datetime = datetime.now(tz=config.tz)
     stop_date: datetime = stop or (start_date + config.stop_boundary_delta)