ddeutil-workflow 0.0.2__py3-none-any.whl → 0.0.3__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.2"
+__version__: str = "0.0.3"

ddeutil/workflow/__types.py

@@ -9,3 +9,4 @@ from typing import Any
 
 TupleStr = tuple[str, ...]
 DictData = dict[str, Any]
+DictStr = dict[str, str]

ddeutil/workflow/conn.py

@@ -43,8 +43,14 @@ class BaseConn(BaseModel):
     ]
 
     @classmethod
-    def from_dict(cls, values: DictData):
-        """Construct Connection with dict of data"""
+    def from_dict(cls, values: DictData) -> Self:
+        """Construct Connection Model from dict data. This construct is
+        different with ``.model_validate()`` because it will prepare the values
+        before using it if the data dose not have 'url'.
+
+        :param values: A dict data that use to construct this model.
+        """
+        # NOTE: filter out the fields of this model.
         filter_data: DictData = {
             k: values.pop(k)
             for k in values.copy()
@@ -73,15 +79,11 @@ class BaseConn(BaseModel):
         )
 
     @classmethod
-    def from_loader(
-        cls,
-        name: str,
-        externals: DictData,
-    ) -> Self:
+    def from_loader(cls, name: str, externals: DictData) -> Self:
         """Construct Connection with Loader object with specific config name.
 
-        :param name:
-        :param externals:
+        :param name: A config name.
+        :param externals: A external data that want to adding to extras.
         """
         loader: Loader = Loader(name, externals=externals)
         # NOTE: Validate the config type match with current connection model
@@ -96,6 +98,7 @@ class BaseConn(BaseModel):
 
     @field_validator("endpoint")
     def __prepare_slash(cls, value: str) -> str:
+        """Prepare slash character that map double form URL model loading."""
         if value.startswith("//"):
             return value[1:]
         return value
@@ -148,7 +151,7 @@ class SFTP(Conn):
     dialect: Literal["sftp"] = "sftp"
 
     def __client(self):
-        from .vendors.sftp_wrapped import WrapSFTP
+        from .vendors.sftp import WrapSFTP
 
         return WrapSFTP(
             host=self.host,

ddeutil/workflow/exceptions.py

@@ -9,24 +9,4 @@ Define Errors Object for Node package
 from __future__ import annotations
 
 
-class BaseError(Exception):
-    """Base Error Object that use for catch any errors statement of
-    all step in this src
-    """
-
-
-class WorkflowBaseError(BaseError):
-    """Core Base Error object"""
-
-
-class ConfigNotFound(WorkflowBaseError):
-    """Error raise for a method not found the config file or data."""
-
-
-class PyException(Exception): ...
-
-
-class ShellException(Exception): ...
-
-
 class TaskException(Exception): ...

ddeutil/workflow/loader.py

@@ -6,7 +6,7 @@
 from __future__ import annotations
 
 from functools import cached_property
-from typing import Any, TypeVar
+from typing import Any, ClassVar, TypeVar
 
 from ddeutil.core import (
     getdot,
@@ -14,12 +14,12 @@ from ddeutil.core import (
     import_string,
 )
 from ddeutil.io import (
-    ConfigNotFound,
-    Params,
+    PathData,
     PathSearch,
     YamlEnvFl,
 )
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
+from pydantic.functional_validators import model_validator
 
 from .__regex import RegexConf
 from .__types import DictData
@@ -29,6 +29,25 @@ BaseModelType = type[BaseModel]
 AnyModel = TypeVar("AnyModel", bound=BaseModel)
 
 
+class Engine(BaseModel):
+    """Engine Model"""
+
+    paths: PathData = Field(default_factory=PathData)
+    registry: list[str] = Field(default_factory=lambda: ["ddeutil.workflow"])
+
+    @model_validator(mode="before")
+    def __prepare_registry(cls, values: DictData) -> DictData:
+        if (_regis := values.get("registry")) and isinstance(_regis, str):
+            values["registry"] = [_regis]
+        return values
+
+
+class Params(BaseModel):
+    """Params Model"""
+
+    engine: Engine = Field(default_factory=Engine)
+
+
 class SimLoad:
     """Simple Load Object that will search config data by name.
 
@@ -36,13 +55,11 @@ class SimLoad:
     :param params: A Params model object.
     :param externals: An external parameters
 
-    Note:
+    Noted:
         The config data should have ``type`` key for engine can know what is
     config should to do next.
     """
 
-    import_prefix: str = "ddeutil.workflow"
-
     def __init__(
         self,
         name: str,
@@ -56,7 +73,7 @@ class SimLoad:
             ):
                 self.data = data
         if not self.data:
-            raise ConfigNotFound(f"Config {name!r} does not found on conf path")
+            raise ValueError(f"Config {name!r} does not found on conf path")
         self.__conf_params: Params = params
         self.externals: DictData = externals
 
@@ -75,6 +92,11 @@ class SimLoad:
             # NOTE: Auto adding module prefix if it does not set
             return import_string(f"ddeutil.workflow.{_typ}")
         except ModuleNotFoundError:
+            for registry in self.conf_params.engine.registry:
+                try:
+                    return import_string(f"{registry}.{_typ}")
+                except ModuleNotFoundError:
+                    continue
             return import_string(f"{_typ}")
 
     def load(self) -> AnyModel:
@@ -82,12 +104,14 @@ class SimLoad:
 
 
 class Loader(SimLoad):
-    """Main Loader Object.
+    """Main Loader Object that get the config `yaml` file from current path.
 
     :param name: A name of config data that will read by Yaml Loader object.
     :param externals: An external parameters
     """
 
+    conf_name: ClassVar[str] = "workflows-conf"
+
     def __init__(
         self,
         name: str,
@@ -106,12 +130,16 @@ class Loader(SimLoad):
     def config(cls, path: str | None = None) -> Params:
         """Load Config data from ``workflows-conf.yaml`` file."""
         return Params.model_validate(
-            YamlEnvFl(path or "./workflows-conf.yaml").read()
+            YamlEnvFl(path or f"./{cls.conf_name}.yaml").read()
         )
 
 
 def map_params(value: Any, params: dict[str, Any]) -> Any:
-    """Map caller value that found from ``RE_CALLER`` regex.
+    """Map caller value that found from ``RE_CALLER`` regular expression.
+
+    :param value: A value that want to mapped with an params
+    :param params: A parameter value that getting with matched regular
+        expression.
 
     :rtype: Any
     :returns: An any getter value from the params input.

ddeutil/workflow/pipeline.py

@@ -6,38 +6,52 @@
 from __future__ import annotations
 
 import inspect
+import itertools
 import logging
 import subprocess
+import time
 from abc import ABC, abstractmethod
-from datetime import date, datetime
 from inspect import Parameter
+from queue import Queue
 from subprocess import CompletedProcess
-from typing import Any, Callable, Literal, Optional, Union
+from typing import Any, Callable, Optional, Union
 
-from ddeutil.io.models.lineage import dt_now
+import msgspec as spec
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__regex import RegexConf
-from .__types import DictData
-from .exceptions import PyException, TaskException
+from .__types import DictData, DictStr
+from .exceptions import TaskException
 from .loader import Loader, map_params
-from .utils import make_registry
+from .utils import Params, make_registry
 
 
 class BaseStage(BaseModel, ABC):
-    """Base Stage Model."""
-
-    id: Optional[str] = None
-    name: str
+    """Base Stage Model that keep only id and name fields."""
+
+    id: Optional[str] = Field(
+        default=None,
+        description=(
+            "The stage ID that use to keep execution output or getting by job "
+            "owner."
+        ),
+    )
+    name: str = Field(
+        description="The stage name that want to logging when start execution."
+    )
 
     @abstractmethod
     def execute(self, params: DictData) -> DictData:
+        """Execute abstraction method that action something by sub-model class.
+
+        :param params: A parameter data that want to use in this execution.
+        """
         raise NotImplementedError("Stage should implement ``execute`` method.")
 
     def set_outputs(self, rs: DictData, params: DictData) -> DictData:
-        """Set outputs to params"""
+        """Set an outputs from execution process to an input params."""
         if self.id is None:
             return params
 
@@ -61,7 +75,7 @@ class ShellStage(BaseStage):
     """Shell statement stage."""
 
     shell: str
-    env: dict[str, str] = Field(default_factory=dict)
+    env: DictStr = Field(default_factory=dict)
 
     @staticmethod
     def __prepare_shell(shell: str):
@@ -100,7 +114,7 @@ class ShellStage(BaseStage):
         if rs.returncode > 0:
             print(f"{rs.stderr}\nRunning Statement:\n---\n{self.shell}")
             # FIXME: raise err for this execution.
-            # raise ShellException(
+            # raise TaskException(
             #     f"{rs.stderr}\nRunning Statement:\n---\n"
             #     f"{self.shell}"
             # )
@@ -116,7 +130,7 @@ class PyStage(BaseStage):
     run: str
     vars: DictData = Field(default_factory=dict)
 
-    def get_var(self, params: DictData) -> DictData:
+    def get_vars(self, params: DictData) -> DictData:
         """Return variables"""
         rs = self.vars.copy()
         for p, v in self.vars.items():
@@ -149,12 +163,12 @@ class PyStage(BaseStage):
         :returns: A parameters from an input that was mapped output if the stage
             ID was set.
         """
-        _globals: DictData = globals() | params | self.get_var(params)
+        _globals: DictData = globals() | params | self.get_vars(params)
         _locals: DictData = {}
         try:
             exec(map_params(self.run, params), _globals, _locals)
         except Exception as err:
-            raise PyException(
+            raise TaskException(
                 f"{err.__class__.__name__}: {err}\nRunning Statement:\n---\n"
                 f"{self.run}"
             ) from None
@@ -164,13 +178,17 @@ class PyStage(BaseStage):
         return params | {k: _globals[k] for k in params if k in _globals}
 
 
-class TaskSearch(BaseModel):
-    """Task Search Model"""
+class TaskSearch(spec.Struct, kw_only=True, tag="task"):
+    """Task Search Struct that use the `msgspec` for the best performance."""
 
     path: str
     func: str
     tag: str
 
+    def to_dict(self) -> DictData:
+        """Return dict data from struct fields."""
+        return {f: getattr(self, f) for f in self.__struct_fields__}
+
 
 class TaskStage(BaseStage):
     """Task executor stage that running the Python function."""
@@ -183,7 +201,7 @@ class TaskStage(BaseStage):
         """Extract Task string value to task function."""
         if not (found := RegexConf.RE_TASK_FMT.search(task)):
             raise ValueError("Task does not match with task format regex.")
-        tasks = TaskSearch(**found.groupdict())
+        tasks: TaskSearch = TaskSearch(**found.groupdict())
 
         # NOTE: Registry object should implement on this package only.
         # TODO: This prefix value to search registry should dynamic with
@@ -238,153 +256,131 @@ Stage = Union[
 
 
 class Strategy(BaseModel):
-    """Strategy Model"""
+    """Strategy Model that will combine a matrix together for running the
+    special job.
+
+    Examples:
+        >>> strategy = {
+        ...     'matrix': {
+        ...         'first': [1, 2, 3],
+        ...         'second': ['foo', 'bar']
+        ...     },
+        ...     'include': [{'first': 4, 'second': 'foo'}],
+        ...     'exclude': [{'first': 1, 'second': 'bar'}],
+        ... }
+    """
 
-    matrix: list[str] = Field(default_factory=list)
-    include: list[str] = Field(default_factory=list)
-    exclude: list[str] = Field(default_factory=list)
+    fail_fast: bool = Field(default=False)
+    max_parallel: int = Field(default=-1)
+    matrix: dict[str, Union[list[str], list[int]]] = Field(default_factory=dict)
+    include: list[dict[str, Union[str, int]]] = Field(default_factory=list)
+    exclude: list[dict[str, Union[str, int]]] = Field(default_factory=list)
+
+    @model_validator(mode="before")
+    def __prepare_keys(cls, values: DictData) -> DictData:
+        if "max-parallel" in values:
+            values["max_parallel"] = values.pop("max-parallel")
+        if "fail-fast" in values:
+            values["fail_fast"] = values.pop("fail-fast")
+        return values
 
 
 class Job(BaseModel):
     """Job Model"""
 
+    runs_on: Optional[str] = Field(default=None)
     stages: list[Stage] = Field(default_factory=list)
     needs: list[str] = Field(default_factory=list)
     strategy: Strategy = Field(default_factory=Strategy)
 
+    @model_validator(mode="before")
+    def __prepare_keys(cls, values: DictData) -> DictData:
+        if "runs-on" in values:
+            values["runs_on"] = values.pop("runs-on")
+        return values
+
     def stage(self, stage_id: str) -> Stage:
+        """Return stage model that match with an input stage ID."""
         for stage in self.stages:
             if stage_id == (stage.id or ""):
                 return stage
         raise ValueError(f"Stage ID {stage_id} does not exists")
 
+    def make_strategy(self) -> list[DictStr]:
+        """Return List of combination of matrix values that already filter with
+        exclude and add include values.
+        """
+        if not (mt := self.strategy.matrix):
+            return [{}]
+        final: list[DictStr] = []
+        for r in [
+            {_k: _v for e in mapped for _k, _v in e.items()}
+            for mapped in itertools.product(
+                *[[{k: v} for v in vs] for k, vs in mt.items()]
+            )
+        ]:
+            if any(
+                all(r[k] == v for k, v in exclude.items())
+                for exclude in self.strategy.exclude
+            ):
+                continue
+            final.append(r)
+
+        if not final:
+            return [{}]
+
+        for include in self.strategy.include:
+            if include.keys() != final[0].keys():
+                raise ValueError("Include should have the keys equal to matrix")
+            if any(all(include[k] == v for k, v in f.items()) for f in final):
+                continue
+            final.append(include)
+        return final
+
     def execute(self, params: DictData | None = None) -> DictData:
         """Execute job with passing dynamic parameters from the pipeline."""
-        for stage in self.stages:
-            # NOTE:
-            #       I do not use below syntax because `params` dict be the
-            #   reference memory pointer and it was changed when I action
-            #   anything like update or re-construct this.
-            #       ... params |= stage.execute(params=params)
-            stage.execute(params=params)
+        for strategy in self.make_strategy():
+            params.update({"matrix": strategy})
+
+            # IMPORTANT: The stage execution only run sequentially one-by-one.
+            for stage in self.stages:
+                logging.info(
+                    f"[JOB]: Start execute the stage: "
+                    f"{(stage.id if stage.id else stage.name)!r}"
+                )
+
+                # NOTE:
+                #       I do not use below syntax because `params` dict be the
+                #   reference memory pointer and it was changed when I action
+                #   anything like update or re-construct this.
+                #       ... params |= stage.execute(params=params)
+                stage.execute(params=params)
+        # TODO: We should not return matrix key to outside
         return params
 
 
-class BaseParams(BaseModel, ABC):
-    """Base Parameter that use to make Params Model."""
-
-    desc: Optional[str] = None
-    required: bool = True
-    type: str
-
-    @abstractmethod
-    def receive(self, value: Optional[Any] = None) -> Any:
-        raise ValueError(
-            "Receive value and validate typing before return valid value."
-        )
-
-
-class DefaultParams(BaseParams):
-    """Default Parameter that will check default if it required"""
-
-    default: Optional[str] = None
-
-    @abstractmethod
-    def receive(self, value: Optional[Any] = None) -> Any:
-        raise ValueError(
-            "Receive value and validate typing before return valid value."
-        )
-
-    @model_validator(mode="after")
-    def check_default(self) -> Self:
-        if not self.required and self.default is None:
-            raise ValueError(
-                "Default should set when this parameter does not required."
-            )
-        return self
-
-
-class DatetimeParams(DefaultParams):
-    """Datetime parameter."""
-
-    type: Literal["datetime"] = "datetime"
-    required: bool = False
-    default: datetime = Field(default_factory=dt_now)
-
-    def receive(self, value: str | datetime | date | None = None) -> datetime:
-        if value is None:
-            return self.default
-
-        if isinstance(value, datetime):
-            return value
-        elif isinstance(value, date):
-            return datetime(value.year, value.month, value.day)
-        elif not isinstance(value, str):
-            raise ValueError(
-                f"Value that want to convert to datetime does not support for "
-                f"type: {type(value)}"
-            )
-        return datetime.fromisoformat(value)
-
-
-class StrParams(DefaultParams):
-    """String parameter."""
-
-    type: Literal["str"] = "str"
-
-    def receive(self, value: Optional[str] = None) -> str | None:
-        if value is None:
-            return self.default
-        return str(value)
-
-
-class IntParams(DefaultParams):
-    """Integer parameter."""
-
-    type: Literal["int"] = "int"
-
-    def receive(self, value: Optional[int] = None) -> int | None:
-        if value is None:
-            return self.default
-        if not isinstance(value, int):
-            try:
-                return int(str(value))
-            except TypeError as err:
-                raise ValueError(
-                    f"Value that want to convert to integer does not support "
-                    f"for type: {type(value)}"
-                ) from err
-        return value
-
-
-class ChoiceParams(BaseParams):
-    type: Literal["choice"] = "choice"
-    options: list[str]
-
-    def receive(self, value: Optional[str] = None) -> str:
-        """Receive value that match with options."""
-        # NOTE:
-        #   Return the first value in options if does not pass any input value
-        if value is None:
-            return self.options[0]
-        if any(value not in self.options):
-            raise ValueError(f"{value} does not match any value in options")
-        return value
-
-
-Params = Union[
-    ChoiceParams,
-    DatetimeParams,
-    StrParams,
-]
-
-
 class Pipeline(BaseModel):
-    """Pipeline Model"""
+    """Pipeline Model this is the main feature of this project because it use to
+    be workflow data for running everywhere that you want. It use lightweight
+    coding line to execute it.
+    """
 
     params: dict[str, Params] = Field(default_factory=dict)
     jobs: dict[str, Job]
 
+    @model_validator(mode="before")
+    def __prepare_params(cls, values: DictData) -> DictData:
+        if params := values.pop("params", {}):
+            values["params"] = {
+                p: (
+                    {"type": params[p]}
+                    if isinstance(params[p], str)
+                    else params[p]
+                )
+                for p in params
+            }
+        return values
+
     @classmethod
     def from_loader(
         cls,
@@ -399,6 +395,10 @@ class Pipeline(BaseModel):
             params=loader.data["params"],
         )
 
+    @model_validator(mode="after")
+    def job_checking_needs(self):
+        return self
+
     def job(self, name: str) -> Job:
         """Return Job model that exists on this pipeline.
 
@@ -406,13 +406,23 @@ class Pipeline(BaseModel):
         :type name: str
 
         :rtype: Job
+        :returns: A job model that exists on this pipeline by input name.
         """
         if name not in self.jobs:
-            raise ValueError(f"Job {name} does not exists")
+            raise ValueError(f"Job {name!r} does not exists")
         return self.jobs[name]
 
-    def execute(self, params: DictData | None = None) -> DictData:
-        """Execute pipeline with passing dynamic parameters.
+    def execute(
+        self,
+        params: DictData | None = None,
+        time_out: int = 60,
+    ) -> DictData:
+        """Execute pipeline with passing dynamic parameters to any jobs that
+        included in the pipeline.
+
+        :param params: An input parameters that use on pipeline execution.
+        :param time_out: A time out second value for limit time of this
+            execution.
 
         See Also:
 
@@ -427,8 +437,7 @@ class Pipeline(BaseModel):
 
         """
         params: DictData = params or {}
-        check_key = tuple(f"{k!r}" for k in self.params if k not in params)
-        if check_key:
+        if check_key := tuple(f"{k!r}" for k in self.params if k not in params):
             raise ValueError(
                 f"Parameters that needed on pipeline does not pass: "
                 f"{', '.join(check_key)}."
@@ -445,12 +454,39 @@ class Pipeline(BaseModel):
                     for k in params
                     if k in self.params
                 }
-            )
+            ),
+            "jobs": {},
         }
+
+        jq = Queue()
         for job_id in self.jobs:
-            print(f"[PIPELINE]: Start execute the job: {job_id!r}")
+            jq.put(job_id)
+
+        ts: float = time.monotonic()
+        not_time_out_flag = True
+
+        # IMPORTANT: The job execution can run parallel and waiting by needed.
+        while not jq.empty() and (
+            not_time_out_flag := ((time.monotonic() - ts) < time_out)
+        ):
+            job_id: str = jq.get()
+            logging.info(f"[PIPELINE]: Start execute the job: {job_id!r}")
             job: Job = self.jobs[job_id]
             # TODO: Condition on ``needs`` of this job was set. It should create
             #   multithreading process on this step.
+            #   But, I don't know how to handle changes params between each job
+            #   execution while its use them together.
+            #   ---
+            #   >>> import multiprocessing
+            #   >>> with multiprocessing.Pool(processes=3) as pool:
+            #   ...     results = pool.starmap(merge_names, ('', '', ...))
+            if any(params["jobs"].get(need) for need in job.needs):
+                jq.put(job_id)
             job.execute(params=params)
+            params["jobs"][job_id] = {
+                "stages": params.pop("stages", {}),
+                "matrix": params.pop("matrix", {}),
+            }
+        if not not_time_out_flag:
+            raise RuntimeError("Execution of pipeline was time out")
         return params