ddeutil-workflow 0.0.1__py3-none-any.whl

ddeutil/workflow/loader.py

@@ -0,0 +1,310 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import copy
+import logging
+import urllib.parse
+from functools import cached_property
+from typing import Any, Callable, TypeVar
+
+from ddeutil.core import (
+    clear_cache,
+    getdot,
+    hasdot,
+    import_string,
+    setdot,
+)
+from ddeutil.io import (
+    ConfigNotFound,
+    Params,
+    PathSearch,
+    Register,
+    YamlEnvFl,
+    map_func,
+)
+from ddeutil.io.__conf import UPDATE_KEY, VERSION_KEY
+from fmtutil import Datetime
+from pydantic import BaseModel
+from typing_extensions import Self
+
+from .__regex import RegexConf
+from .__types import DictData, TupleStr
+from .exceptions import ConfigArgumentError
+
+AnyModel = TypeVar("AnyModel", bound=BaseModel)
+
+
+class YamlEnvQuote(YamlEnvFl):
+
+    @staticmethod
+    def prepare(x: str) -> str:
+        return urllib.parse.quote_plus(str(x))
+
+
+class BaseLoad:
+    """Base configuration data loading object for load config data from
+    `cls.load_stage` stage. The base loading object contain necessary
+    properties and method for type object.
+
+    :param data: dict : A configuration data content with fix keys, `name`,
+        `fullname`, and `data`.
+    :param params: Optional[dict] : A parameters mapping for some
+        subclass of loading use.
+    """
+
+    # NOTE: Set loading config for inherit
+    load_prefixes: TupleStr = ("conn",)
+    load_datetime_name: str = "audit_date"
+    load_datetime_fmt: str = "%Y-%m-%d %H:%M:%S"
+
+    # NOTE: Set preparing config for inherit
+    data_excluded: TupleStr = (UPDATE_KEY, VERSION_KEY)
+    option_key: TupleStr = ("parameters",)
+    datetime_key: TupleStr = ("endpoint",)
+
+    @classmethod
+    def from_register(
+        cls,
+        name: str,
+        params: Params,
+        externals: DictData | None = None,
+    ) -> Self:
+        """Loading config data from register object.
+
+        :param name: A name of config data catalog that can register.
+        :type name: str
+        :param params: A params object.
+        :type params: Params
+        :param externals: A external parameters
+        :type externals: DictData | None(=None)
+        """
+        try:
+            rs: Register = Register(
+                name=name,
+                stage=params.stage_final,
+                params=params,
+                loader=YamlEnvQuote,
+            )
+        except ConfigNotFound:
+            rs: Register = Register(
+                name=name,
+                params=params,
+                loader=YamlEnvQuote,
+            ).deploy(stop=params.stage_final)
+        return cls(
+            name=rs.name,
+            data=rs.data().copy(),
+            params=params,
+            externals=externals,
+        )
+
+    def __init__(
+        self,
+        name: str,
+        data: DictData,
+        params: Params,
+        externals: DictData | None = None,
+    ) -> None:
+        """Main initialize base config object which get a name of configuration
+        and load data by the register object.
+        """
+        self.name: str = name
+        self.__data: DictData = data
+        self.params: Params = params
+        self.externals: DictData = externals or {}
+
+        # NOTE: Validate step of base loading object.
+        if not any(
+            self.name.startswith(prefix) for prefix in self.load_prefixes
+        ):
+            raise ConfigArgumentError(
+                "prefix",
+                (
+                    f"{self.name!r} does not starts with the "
+                    f"{self.__class__.__name__} prefixes: "
+                    f"{self.load_prefixes!r}."
+                ),
+            )
+
+    @property
+    def updt(self):
+        return self.data.get(UPDATE_KEY)
+
+    @cached_property
+    def _map_data(self) -> DictData:
+        """Return configuration data without key in the excluded key set."""
+        data: DictData = self.__data.copy()
+        rs: DictData = {k: data[k] for k in data if k not in self.data_excluded}
+
+        # Mapping datetime format to string value.
+        for _ in self.datetime_key:
+            if hasdot(_, rs):
+                # Fill format datetime object to any type value.
+                rs: DictData = setdot(
+                    _,
+                    rs,
+                    map_func(
+                        getdot(_, rs),
+                        Datetime.parse(
+                            value=self.externals[self.load_datetime_name],
+                            fmt=self.load_datetime_fmt,
+                        ).format,
+                    ),
+                )
+        return rs
+
+    @property
+    def data(self) -> DictData:
+        """Return deep copy of the input data.
+
+        :rtype: DictData
+        """
+        return copy.deepcopy(self._map_data)
+
+    @clear_cache(attrs=("type", "_map_data"))
+    def refresh(self) -> Self:
+        """Refresh configuration data. This process will use `deploy` method
+        of the register object.
+
+        :rtype: Self
+        """
+        return self.from_register(
+            name=self.name,
+            params=self.params,
+            externals=self.externals,
+        )
+
+    @cached_property
+    def type(self) -> Any:
+        """Return object type which implement in `config_object` key."""
+        if not (_typ := self.data.get("type")):
+            raise ValueError(
+                f"the 'type' value: {_typ} does not exists in config data."
+            )
+        return import_string(f"ddeutil.pipe.{_typ}")
+
+
+class SimLoad:
+    """Simple Load Object that will search config data by name.
+
+    :param name: A name of config data that will read by Yaml Loader object.
+    :param params: A Params model object.
+    :param externals: An external parameters
+
+    Note:
+        The config data should have ``type`` key for engine can know what is
+    config should to do next.
+    """
+
+    import_prefix: str = "ddeutil.workflow"
+
+    def __init__(
+        self,
+        name: str,
+        params: Params,
+        externals: DictData,
+    ) -> None:
+        self.data: DictData = {}
+        for file in PathSearch(params.engine.paths.conf).files:
+            if any(file.suffix.endswith(s) for s in ("yml", "yaml")) and (
+                data := YamlEnvFl(file).read().get(name, {})
+            ):
+                self.data = data
+        if not self.data:
+            raise ConfigNotFound(f"Config {name!r} does not found on conf path")
+        self.__conf_params: Params = params
+        self.externals: DictData = externals
+
+    @property
+    def conf_params(self) -> Params:
+        return self.__conf_params
+
+    @cached_property
+    def type(self) -> AnyModel:
+        """Return object type which implement in `config_object` key."""
+        if not (_typ := self.data.get("type")):
+            raise ValueError(
+                f"the 'type' value: {_typ} does not exists in config data."
+            )
+        try:
+            # NOTE: Auto adding module prefix if it does not set
+            return import_string(f"ddeutil.workflow.{_typ}")
+        except ModuleNotFoundError:
+            return import_string(f"{_typ}")
+
+    def params(self) -> dict[str, Callable[[Any], Any]]:
+        """Return a mapping of key from params and imported value on params."""
+        if not (p := self.data.get("params", {})):
+            return p
+
+        try:
+            return {i: import_string(f"{self.import_prefix}.{p[i]}") for i in p}
+        except ModuleNotFoundError as err:
+            logging.error(err)
+            raise err
+
+    def validate_params(self, param: dict[str, Any]) -> dict[str, Any]:
+        """Return parameter that want to catch before workflow running."""
+        try:
+            return {i: caller(param[i]) for i, caller in self.params().items()}
+        except KeyError as err:
+            logging.error(f"Parameter: {err} does not exists from passing")
+            raise err
+        except ValueError as err:
+            logging.error("Value that passing to params does not valid")
+            raise err
+
+
+class Loader(SimLoad):
+    """Main Loader Object."""
+
+    def __init__(
+        self,
+        name: str,
+        externals: DictData,
+        *,
+        path: str | None = None,
+    ) -> None:
+        self.data: DictData = {}
+
+        # NOTE: import params object from specific config file
+        params: Params = self.config(path)
+
+        super().__init__(name, params, externals)
+
+    @classmethod
+    def config(cls, path: str | None = None) -> Params:
+        return Params.model_validate(
+            YamlEnvFl(path or "./workflows-conf.yaml").read()
+        )
+
+
+def map_caller(value: str, params: dict[str, Any]) -> Any:
+    """Map caller value that found from ``RE_CALLER`` regex.
+
+    :returns: Any value that getter of caller receive from the params.
+    """
+    if not (found := RegexConf.RE_CALLER.search(value)):
+        return value
+    # NOTE: get caller value that setting inside; ``${{ <caller-value> }}``
+    caller = found.group("caller")
+    if not hasdot(caller, params):
+        raise ValueError(f"params does not set caller: {caller!r}")
+    getter = getdot(caller, params)
+
+    # NOTE: check type of vars
+    if isinstance(getter, (str, int)):
+        return value.replace(found.group(0), str(getter))
+
+    # NOTE:
+    #   If type of getter caller does not formatting, it will return origin
+    #   value.
+    if value.replace(found.group(0), "") != "":
+        raise ValueError(
+            "Callable variable should not pass other outside ${{ ... }}"
+        )
+    return getter

ddeutil/workflow/pipeline.py

@@ -0,0 +1,338 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import inspect
+import subprocess
+from inspect import Parameter
+from subprocess import CompletedProcess
+from typing import Any, Callable, Optional, Union
+
+from pydantic import BaseModel, Field
+from typing_extensions import Self
+
+from .__regex import RegexConf
+from .__types import DictData
+from .exceptions import PipeArgumentError, PyException, TaskException
+from .loader import Loader, map_caller
+
+
+class StageResult(BaseModel): ...
+
+
+class JobResult(BaseModel): ...
+
+
+class PipeResult(BaseModel): ...
+
+
+class EmptyStage(BaseModel):
+    """Empty stage that is doing nothing and logging the name of stage only."""
+
+    id: Optional[str] = None
+    name: str
+
+    def execute(self, params: dict[str, Any]) -> dict[str, Any]:
+        return params
+
+
+class ShellStage(EmptyStage):
+    """Shell statement stage."""
+
+    shell: str
+    env: dict[str, str] = Field(default_factory=dict)
+
+    @staticmethod
+    def __prepare_shell(shell: str):
+        """Prepare shell statement string that include newline"""
+        return shell.replace("\n", ";")
+
+    def set_outputs(
+        self, rs: CompletedProcess, params: dict[str, Any]
+    ) -> dict[str, Any]:
+        """Set outputs to params"""
+        # NOTE: skipping set outputs of stage execution when id does not set.
+        if self.id is None:
+            return params
+
+        if "stages" not in params:
+            params["stages"] = {}
+
+        params["stages"][self.id] = {
+            # NOTE: The output will fileter unnecessary keys from ``_locals``.
+            "outputs": {
+                "return_code": rs.returncode,
+                "stdout": rs.stdout,
+                "stderr": rs.stderr,
+            },
+        }
+        return params
+
+    def execute(self, params: dict[str, Any]) -> dict[str, Any]:
+        """Execute the Shell & Powershell statement with the Python build-in
+        ``subprocess`` package.
+        """
+        rs: CompletedProcess = subprocess.run(
+            self.__prepare_shell(self.shell),
+            capture_output=True,
+            text=True,
+            shell=True,
+        )
+        if rs.returncode > 0:
+            print(f"{rs.stderr}\nRunning Statement:\n---\n{self.shell}")
+            # FIXME: raise err for this execution.
+            # raise ShellException(
+            #     f"{rs.stderr}\nRunning Statement:\n---\n"
+            #     f"{self.shell}"
+            # )
+        self.set_outputs(rs, params)
+        return params
+
+
+class PyStage(EmptyStage):
+    """Python executor stage that running the Python statement that receive
+    globals nad additional variables.
+    """
+
+    run: str
+    vars: dict[str, Any] = Field(default_factory=dict)
+
+    def get_var(self, params: dict[str, Any]) -> dict[str, Any]:
+        """Return variables"""
+        rs = self.vars.copy()
+        for p, v in self.vars.items():
+            rs[p] = map_caller(v, params)
+        return rs
+
+    def set_outputs(
+        self, lc: dict[str, Any], params: dict[str, Any]
+    ) -> dict[str, Any]:
+        """Set outputs to params"""
+        # NOTE: skipping set outputs of stage execution when id does not set.
+        if self.id is None:
+            return params
+
+        if "stages" not in params:
+            params["stages"] = {}
+
+        params["stages"][self.id] = {
+            # NOTE: The output will fileter unnecessary keys from ``_locals``.
+            "outputs": {k: lc[k] for k in lc if k != "__annotations__"},
+        }
+        return params
+
+    def execute(self, params: dict[str, Any]) -> dict[str, Any]:
+        """Execute the Python statement that pass all globals and input params
+        to globals argument on ``exec`` build-in function.
+
+        :param params: A parameter that want to pass before run any statement.
+        :type params: dict[str, Any]
+
+        :rtype: dict[str, Any]
+        :returns: A parameters from an input that was mapped output if the stage
+            ID was set.
+        """
+        _globals: dict[str, Any] = globals() | params | self.get_var(params)
+        _locals: dict[str, Any] = {}
+        try:
+            exec(map_caller(self.run, params), _globals, _locals)
+        except Exception as err:
+            raise PyException(
+                f"{err.__class__.__name__}: {err}\nRunning Statement:\n---\n"
+                f"{self.run}"
+            ) from None
+
+        # NOTE: set outputs from ``_locals`` value from ``exec``.
+        self.set_outputs(_locals, params)
+        return params | {k: _globals[k] for k in params if k in _globals}
+
+
+class TaskSearch(BaseModel):
+    path: str
+    func: str
+    tag: str
+
+
+class TaskStage(EmptyStage):
+    task: str
+    args: dict[str, Any]
+
+    @staticmethod
+    def extract_task(task: str) -> Callable[[], Callable[[Any], Any]]:
+        """Extract Task string value to task function."""
+        if not (found := RegexConf.RE_TASK_FMT.search(task)):
+            raise ValueError("Task does not match with task format regex.")
+        tasks = TaskSearch(**found.groupdict())
+
+        from ddeutil.core import import_string
+
+        try:
+            rgt = import_string(f"ddeutil.workflow.{tasks.path}.registries")
+            if tasks.func not in rgt:
+                raise NotImplementedError(
+                    f"ddeutil.workflow.{tasks.path}.registries does not "
+                    f"implement registry: {tasks.func}."
+                )
+        except ImportError:
+
+            # NOTE: Try to import this task function fom target module.
+            try:
+                return import_string(
+                    f"ddeutil.workflow.{tasks.path}.{tasks.func}"
+                )
+            except ImportError:
+                raise NotImplementedError(
+                    f"ddeutil.workflow.{tasks.path} does not implement "
+                    f"registries or {tasks.func}."
+                ) from None
+
+        if tasks.tag not in rgt[tasks.func]:
+            raise NotImplementedError(
+                f"tag: {tasks.tag} does not found on registry func: "
+                f"ddeutil.workflow.{tasks.path}.registries."
+                f"{tasks.func}"
+            )
+        return rgt[tasks.func][tasks.tag]
+
+    def execute(self, params: dict[str, Any]) -> dict[str, Any]:
+        """Execute the Task function."""
+        task_caller = self.extract_task(self.task)()
+        if not callable(task_caller):
+            raise ImportError("Task caller function does not callable.")
+
+        # NOTE: check task caller parameters
+        ips = inspect.signature(task_caller)
+        if any(
+            k not in self.args
+            for k in ips.parameters
+            if ips.parameters[k].default == Parameter.empty
+        ):
+            raise ValueError(
+                f"necessary parameters, ({', '.join(ips.parameters.keys())}), "
+                f"does not set to args"
+            )
+        try:
+            rs = task_caller(**self.args)
+        except Exception as err:
+            raise TaskException(f"{err.__class__.__name__}: {err}") from err
+        return {"output": rs}
+
+
+class HookStage(EmptyStage):
+    hook: str
+    args: dict[str, Any]
+
+    def execute(self, params: dict[str, Any]) -> dict[str, Any]: ...
+
+
+# NOTE: Order of parsing stage data
+Stage = Union[
+    PyStage,
+    ShellStage,
+    TaskStage,
+    HookStage,
+    EmptyStage,
+]
+
+
+class Job(BaseModel):
+    stages: list[Stage] = Field(default_factory=list)
+    needs: list[str] = Field(default_factory=list)
+
+    def stage(self, stage_id: str) -> Stage:
+        for stage in self.stages:
+            if stage_id == (stage.id or ""):
+                return stage
+        raise ValueError(f"Stage ID {stage_id} does not exists")
+
+    def execute(self, params: dict[str, Any] | None = None) -> dict[str, Any]:
+        for stage in self.stages:
+            # NOTE:
+            #       I do not use below syntax because `params` dict be the
+            #   reference memory pointer and it was changed when I action
+            #   anything like update or re-construct this.
+            #       ... params |= stage.execute(params=params)
+            stage.execute(params=params)
+        return params
+
+
+class Strategy(BaseModel):
+    matrix: list[str]
+    include: list[str]
+    exclude: list[str]
+
+
+class JobStrategy(Job):
+    """Strategy job"""
+
+    strategy: Strategy
+
+
+class Pipeline(BaseModel):
+    """Pipeline Model"""
+
+    params: dict[str, Any] = Field(default_factory=dict)
+    jobs: dict[str, Job]
+
+    @classmethod
+    def from_loader(
+        cls,
+        name: str,
+        externals: DictData,
+    ) -> Self:
+        loader: Loader = Loader(name, externals=externals)
+        if "jobs" not in loader.data:
+            raise PipeArgumentError("jobs", "Config does not set ``jobs``")
+        return cls(
+            jobs=loader.data["jobs"],
+            params=loader.params(),
+        )
+
+    def job(self, name: str) -> Job:
+        """Return Job model that exists on this pipeline."""
+        if name not in self.jobs:
+            raise ValueError(f"Job {name} does not exists")
+        return self.jobs[name]
+
+    def execute(self, params: dict[str, Any] | None = None):
+        """Execute pipeline with passing dynamic parameters.
+
+        See Also:
+
+            The result of execution process for each jobs and stages on this
+        pipeline will keeping in dict which able to catch out with all jobs and
+        stages by dot annotation.
+
+            For example, when I want to use the output from previous stage, I
+        can access it with syntax:
+
+            ... "<job-name>.stages.<stage-id>.outputs.<key>"
+
+        """
+        params: dict[str, Any] = params or {}
+        check_key = tuple(f"{k!r}" for k in self.params if k not in params)
+        if check_key:
+            raise ValueError(
+                f"Parameters that needed on pipeline does not pass: "
+                f"{', '.join(check_key)}."
+            )
+        params: dict[str, Any] = {
+            "params": (
+                params
+                | {
+                    k: self.params[k](params[k])
+                    for k in params
+                    if k in self.params
+                }
+            )
+        }
+        for job_id in self.jobs:
+            print(f"[PIPELINE]: Start execute the job: {job_id!r}")
+            job = self.jobs[job_id]
+            # TODO: Condition on ``needs`` of this job was set. It should create
+            #   multithreading process on this step.
+            job.execute(params=params)
+        return params