ddeutil-workflow 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

ddeutil/workflow/pipeline.py

@@ -6,53 +6,83 @@
 from __future__ import annotations
 
 import inspect
+import itertools
+import logging
 import subprocess
+import time
+from abc import ABC, abstractmethod
 from inspect import Parameter
+from queue import Queue
 from subprocess import CompletedProcess
 from typing import Any, Callable, Optional, Union
 
+import msgspec as spec
 from pydantic import BaseModel, Field
+from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__regex import RegexConf
-from .__types import DictData
-from .exceptions import PipeArgumentError, PyException, TaskException
-from .loader import Loader, map_caller
-
-
-class StageResult(BaseModel): ...
-
+from .__types import DictData, DictStr
+from .exceptions import TaskException
+from .loader import Loader, map_params
+from .utils import Params, make_registry
+
+
+class BaseStage(BaseModel, ABC):
+    """Base Stage Model that keep only id and name fields."""
+
+    id: Optional[str] = Field(
+        default=None,
+        description=(
+            "The stage ID that use to keep execution output or getting by job "
+            "owner."
+        ),
+    )
+    name: str = Field(
+        description="The stage name that want to logging when start execution."
+    )
+
+    @abstractmethod
+    def execute(self, params: DictData) -> DictData:
+        """Execute abstraction method that action something by sub-model class.
+
+        :param params: A parameter data that want to use in this execution.
+        """
+        raise NotImplementedError("Stage should implement ``execute`` method.")
 
-class JobResult(BaseModel): ...
+    def set_outputs(self, rs: DictData, params: DictData) -> DictData:
+        """Set an outputs from execution process to an input params."""
+        if self.id is None:
+            return params
 
+        if "stages" not in params:
+            params["stages"] = {}
 
-class PipeResult(BaseModel): ...
+        params["stages"][self.id] = {"outputs": rs}
+        return params
 
 
-class EmptyStage(BaseModel):
+class EmptyStage(BaseStage):
     """Empty stage that is doing nothing and logging the name of stage only."""
 
-    id: Optional[str] = None
-    name: str
-
-    def execute(self, params: dict[str, Any]) -> dict[str, Any]:
+    def execute(self, params: DictData) -> DictData:
+        """Execute for the Empty stage that do only logging out."""
+        logging.info(f"Execute: {self.name!r}")
         return params
 
 
-class ShellStage(EmptyStage):
+class ShellStage(BaseStage):
     """Shell statement stage."""
 
     shell: str
-    env: dict[str, str] = Field(default_factory=dict)
+    env: DictStr = Field(default_factory=dict)
 
     @staticmethod
     def __prepare_shell(shell: str):
         """Prepare shell statement string that include newline"""
         return shell.replace("\n", ";")
 
-    def set_outputs(
-        self, rs: CompletedProcess, params: dict[str, Any]
-    ) -> dict[str, Any]:
+    def set_outputs(self, rs: CompletedProcess, params: DictData) -> DictData:
         """Set outputs to params"""
         # NOTE: skipping set outputs of stage execution when id does not set.
         if self.id is None:
@@ -71,7 +101,7 @@ class ShellStage(EmptyStage):
         }
         return params
 
-    def execute(self, params: dict[str, Any]) -> dict[str, Any]:
+    def execute(self, params: DictData) -> DictData:
         """Execute the Shell & Powershell statement with the Python build-in
         ``subprocess`` package.
         """
@@ -84,7 +114,7 @@ class ShellStage(EmptyStage):
         if rs.returncode > 0:
             print(f"{rs.stderr}\nRunning Statement:\n---\n{self.shell}")
             # FIXME: raise err for this execution.
-            # raise ShellException(
+            # raise TaskException(
             #     f"{rs.stderr}\nRunning Statement:\n---\n"
             #     f"{self.shell}"
             # )
@@ -92,24 +122,22 @@ class ShellStage(EmptyStage):
         return params
 
 
-class PyStage(EmptyStage):
+class PyStage(BaseStage):
     """Python executor stage that running the Python statement that receive
     globals nad additional variables.
     """
 
     run: str
-    vars: dict[str, Any] = Field(default_factory=dict)
+    vars: DictData = Field(default_factory=dict)
 
-    def get_var(self, params: dict[str, Any]) -> dict[str, Any]:
+    def get_vars(self, params: DictData) -> DictData:
         """Return variables"""
         rs = self.vars.copy()
         for p, v in self.vars.items():
-            rs[p] = map_caller(v, params)
+            rs[p] = map_params(v, params)
         return rs
 
-    def set_outputs(
-        self, lc: dict[str, Any], params: dict[str, Any]
-    ) -> dict[str, Any]:
+    def set_outputs(self, rs: DictData, params: DictData) -> DictData:
         """Set outputs to params"""
         # NOTE: skipping set outputs of stage execution when id does not set.
         if self.id is None:
@@ -120,27 +148,27 @@ class PyStage(EmptyStage):
 
         params["stages"][self.id] = {
             # NOTE: The output will fileter unnecessary keys from ``_locals``.
-            "outputs": {k: lc[k] for k in lc if k != "__annotations__"},
+            "outputs": {k: rs[k] for k in rs if k != "__annotations__"},
         }
         return params
 
-    def execute(self, params: dict[str, Any]) -> dict[str, Any]:
+    def execute(self, params: DictData) -> DictData:
         """Execute the Python statement that pass all globals and input params
         to globals argument on ``exec`` build-in function.
 
         :param params: A parameter that want to pass before run any statement.
-        :type params: dict[str, Any]
+        :type params: DictData
 
-        :rtype: dict[str, Any]
+        :rtype: DictData
         :returns: A parameters from an input that was mapped output if the stage
             ID was set.
         """
-        _globals: dict[str, Any] = globals() | params | self.get_var(params)
-        _locals: dict[str, Any] = {}
+        _globals: DictData = globals() | params | self.get_vars(params)
+        _locals: DictData = {}
         try:
-            exec(map_caller(self.run, params), _globals, _locals)
+            exec(map_params(self.run, params), _globals, _locals)
         except Exception as err:
-            raise PyException(
+            raise TaskException(
                 f"{err.__class__.__name__}: {err}\nRunning Statement:\n---\n"
                 f"{self.run}"
             ) from None
@@ -150,44 +178,40 @@ class PyStage(EmptyStage):
         return params | {k: _globals[k] for k in params if k in _globals}
 
 
-class TaskSearch(BaseModel):
+class TaskSearch(spec.Struct, kw_only=True, tag="task"):
+    """Task Search Struct that use the `msgspec` for the best performance."""
+
     path: str
     func: str
     tag: str
 
+    def to_dict(self) -> DictData:
+        """Return dict data from struct fields."""
+        return {f: getattr(self, f) for f in self.__struct_fields__}
+
+
+class TaskStage(BaseStage):
+    """Task executor stage that running the Python function."""
 
-class TaskStage(EmptyStage):
     task: str
-    args: dict[str, Any]
+    args: DictData
 
     @staticmethod
     def extract_task(task: str) -> Callable[[], Callable[[Any], Any]]:
         """Extract Task string value to task function."""
         if not (found := RegexConf.RE_TASK_FMT.search(task)):
             raise ValueError("Task does not match with task format regex.")
-        tasks = TaskSearch(**found.groupdict())
-
-        from ddeutil.core import import_string
-
-        try:
-            rgt = import_string(f"ddeutil.workflow.{tasks.path}.registries")
-            if tasks.func not in rgt:
-                raise NotImplementedError(
-                    f"ddeutil.workflow.{tasks.path}.registries does not "
-                    f"implement registry: {tasks.func}."
-                )
-        except ImportError:
+        tasks: TaskSearch = TaskSearch(**found.groupdict())
 
-            # NOTE: Try to import this task function fom target module.
-            try:
-                return import_string(
-                    f"ddeutil.workflow.{tasks.path}.{tasks.func}"
-                )
-            except ImportError:
-                raise NotImplementedError(
-                    f"ddeutil.workflow.{tasks.path} does not implement "
-                    f"registries or {tasks.func}."
-                ) from None
+        # NOTE: Registry object should implement on this package only.
+        # TODO: This prefix value to search registry should dynamic with
+        #   config file.
+        rgt = make_registry(f"ddeutil.workflow.{tasks.path}")
+        if tasks.func not in rgt:
+            raise NotImplementedError(
+                f"ddeutil.workflow.{tasks.path}.registries does not "
+                f"implement registry: {tasks.func}."
+            )
 
         if tasks.tag not in rgt[tasks.func]:
             raise NotImplementedError(
@@ -197,7 +221,7 @@ class TaskStage(EmptyStage):
             )
         return rgt[tasks.func][tasks.tag]
 
-    def execute(self, params: dict[str, Any]) -> dict[str, Any]:
+    def execute(self, params: DictData) -> DictData:
         """Execute the Task function."""
         task_caller = self.extract_task(self.task)()
         if not callable(task_caller):
@@ -215,17 +239,11 @@ class TaskStage(EmptyStage):
                 f"does not set to args"
             )
         try:
-            rs = task_caller(**self.args)
+            rs = task_caller(**map_params(self.args, params))
         except Exception as err:
             raise TaskException(f"{err.__class__.__name__}: {err}") from err
-        return {"output": rs}
-
-
-class HookStage(EmptyStage):
-    hook: str
-    args: dict[str, Any]
-
-    def execute(self, params: dict[str, Any]) -> dict[str, Any]: ...
+        self.set_outputs(rs, params)
+        return params
 
 
 # NOTE: Order of parsing stage data
@@ -233,72 +251,178 @@ Stage = Union[
     PyStage,
     ShellStage,
     TaskStage,
-    HookStage,
     EmptyStage,
 ]
 
 
+class Strategy(BaseModel):
+    """Strategy Model that will combine a matrix together for running the
+    special job.
+
+    Examples:
+        >>> strategy = {
+        ...     'matrix': {
+        ...         'first': [1, 2, 3],
+        ...         'second': ['foo', 'bar']
+        ...     },
+        ...     'include': [{'first': 4, 'second': 'foo'}],
+        ...     'exclude': [{'first': 1, 'second': 'bar'}],
+        ... }
+    """
+
+    fail_fast: bool = Field(default=False)
+    max_parallel: int = Field(default=-1)
+    matrix: dict[str, Union[list[str], list[int]]] = Field(default_factory=dict)
+    include: list[dict[str, Union[str, int]]] = Field(default_factory=list)
+    exclude: list[dict[str, Union[str, int]]] = Field(default_factory=list)
+
+    @model_validator(mode="before")
+    def __prepare_keys(cls, values: DictData) -> DictData:
+        if "max-parallel" in values:
+            values["max_parallel"] = values.pop("max-parallel")
+        if "fail-fast" in values:
+            values["fail_fast"] = values.pop("fail-fast")
+        return values
+
+
 class Job(BaseModel):
+    """Job Model"""
+
+    runs_on: Optional[str] = Field(default=None)
     stages: list[Stage] = Field(default_factory=list)
     needs: list[str] = Field(default_factory=list)
+    strategy: Strategy = Field(default_factory=Strategy)
+
+    @model_validator(mode="before")
+    def __prepare_keys(cls, values: DictData) -> DictData:
+        if "runs-on" in values:
+            values["runs_on"] = values.pop("runs-on")
+        return values
 
     def stage(self, stage_id: str) -> Stage:
+        """Return stage model that match with an input stage ID."""
         for stage in self.stages:
             if stage_id == (stage.id or ""):
                 return stage
         raise ValueError(f"Stage ID {stage_id} does not exists")
 
-    def execute(self, params: dict[str, Any] | None = None) -> dict[str, Any]:
-        for stage in self.stages:
-            # NOTE:
-            #       I do not use below syntax because `params` dict be the
-            #   reference memory pointer and it was changed when I action
-            #   anything like update or re-construct this.
-            #       ... params |= stage.execute(params=params)
-            stage.execute(params=params)
-        return params
-
-
-class Strategy(BaseModel):
-    matrix: list[str]
-    include: list[str]
-    exclude: list[str]
-
-
-class JobStrategy(Job):
-    """Strategy job"""
+    def make_strategy(self) -> list[DictStr]:
+        """Return List of combination of matrix values that already filter with
+        exclude and add include values.
+        """
+        if not (mt := self.strategy.matrix):
+            return [{}]
+        final: list[DictStr] = []
+        for r in [
+            {_k: _v for e in mapped for _k, _v in e.items()}
+            for mapped in itertools.product(
+                *[[{k: v} for v in vs] for k, vs in mt.items()]
+            )
+        ]:
+            if any(
+                all(r[k] == v for k, v in exclude.items())
+                for exclude in self.strategy.exclude
+            ):
+                continue
+            final.append(r)
+
+        if not final:
+            return [{}]
+
+        for include in self.strategy.include:
+            if include.keys() != final[0].keys():
+                raise ValueError("Include should have the keys equal to matrix")
+            if any(all(include[k] == v for k, v in f.items()) for f in final):
+                continue
+            final.append(include)
+        return final
+
+    def execute(self, params: DictData | None = None) -> DictData:
+        """Execute job with passing dynamic parameters from the pipeline."""
+        for strategy in self.make_strategy():
+            params.update({"matrix": strategy})
+
+            # IMPORTANT: The stage execution only run sequentially one-by-one.
+            for stage in self.stages:
+                logging.info(
+                    f"[JOB]: Start execute the stage: "
+                    f"{(stage.id if stage.id else stage.name)!r}"
+                )
 
-    strategy: Strategy
+                # NOTE:
+                #       I do not use below syntax because `params` dict be the
+                #   reference memory pointer and it was changed when I action
+                #   anything like update or re-construct this.
+                #       ... params |= stage.execute(params=params)
+                stage.execute(params=params)
+        # TODO: We should not return matrix key to outside
+        return params
 
 
 class Pipeline(BaseModel):
-    """Pipeline Model"""
+    """Pipeline Model this is the main feature of this project because it use to
+    be workflow data for running everywhere that you want. It use lightweight
+    coding line to execute it.
+    """
 
-    params: dict[str, Any] = Field(default_factory=dict)
+    params: dict[str, Params] = Field(default_factory=dict)
     jobs: dict[str, Job]
 
+    @model_validator(mode="before")
+    def __prepare_params(cls, values: DictData) -> DictData:
+        if params := values.pop("params", {}):
+            values["params"] = {
+                p: (
+                    {"type": params[p]}
+                    if isinstance(params[p], str)
+                    else params[p]
+                )
+                for p in params
+            }
+        return values
+
     @classmethod
     def from_loader(
         cls,
         name: str,
-        externals: DictData,
+        externals: Optional[DictData] = None,
     ) -> Self:
-        loader: Loader = Loader(name, externals=externals)
+        loader: Loader = Loader(name, externals=(externals or {}))
         if "jobs" not in loader.data:
-            raise PipeArgumentError("jobs", "Config does not set ``jobs``")
+            raise ValueError("Config does not set ``jobs`` value")
         return cls(
             jobs=loader.data["jobs"],
-            params=loader.params(),
+            params=loader.data["params"],
         )
 
+    @model_validator(mode="after")
+    def job_checking_needs(self):
+        return self
+
     def job(self, name: str) -> Job:
-        """Return Job model that exists on this pipeline."""
+        """Return Job model that exists on this pipeline.
+
+        :param name: A job name that want to get from a mapping of job models.
+        :type name: str
+
+        :rtype: Job
+        :returns: A job model that exists on this pipeline by input name.
+        """
         if name not in self.jobs:
-            raise ValueError(f"Job {name} does not exists")
+            raise ValueError(f"Job {name!r} does not exists")
         return self.jobs[name]
 
-    def execute(self, params: dict[str, Any] | None = None):
-        """Execute pipeline with passing dynamic parameters.
+    def execute(
+        self,
+        params: DictData | None = None,
+        time_out: int = 60,
+    ) -> DictData:
+        """Execute pipeline with passing dynamic parameters to any jobs that
+        included in the pipeline.
+
+        :param params: An input parameters that use on pipeline execution.
+        :param time_out: A time out second value for limit time of this
+            execution.
 
         See Also:
 
@@ -312,27 +436,57 @@ class Pipeline(BaseModel):
             ... "<job-name>.stages.<stage-id>.outputs.<key>"
 
         """
-        params: dict[str, Any] = params or {}
-        check_key = tuple(f"{k!r}" for k in self.params if k not in params)
-        if check_key:
+        params: DictData = params or {}
+        if check_key := tuple(f"{k!r}" for k in self.params if k not in params):
             raise ValueError(
                 f"Parameters that needed on pipeline does not pass: "
                 f"{', '.join(check_key)}."
             )
-        params: dict[str, Any] = {
+
+        if any(p not in params for p in self.params if self.params[p].required):
+            raise ValueError("Required parameter does not pass")
+
+        params: DictData = {
             "params": (
                 params
                 | {
-                    k: self.params[k](params[k])
+                    k: self.params[k].receive(params[k])
                     for k in params
                     if k in self.params
                 }
-            )
+            ),
+            "jobs": {},
         }
+
+        jq = Queue()
         for job_id in self.jobs:
-            print(f"[PIPELINE]: Start execute the job: {job_id!r}")
-            job = self.jobs[job_id]
+            jq.put(job_id)
+
+        ts: float = time.monotonic()
+        not_time_out_flag = True
+
+        # IMPORTANT: The job execution can run parallel and waiting by needed.
+        while not jq.empty() and (
+            not_time_out_flag := ((time.monotonic() - ts) < time_out)
+        ):
+            job_id: str = jq.get()
+            logging.info(f"[PIPELINE]: Start execute the job: {job_id!r}")
+            job: Job = self.jobs[job_id]
             # TODO: Condition on ``needs`` of this job was set. It should create
             #   multithreading process on this step.
+            #   But, I don't know how to handle changes params between each job
+            #   execution while its use them together.
+            #   ---
+            #   >>> import multiprocessing
+            #   >>> with multiprocessing.Pool(processes=3) as pool:
+            #   ...     results = pool.starmap(merge_names, ('', '', ...))
+            if any(params["jobs"].get(need) for need in job.needs):
+                jq.put(job_id)
             job.execute(params=params)
+            params["jobs"][job_id] = {
+                "stages": params.pop("stages", {}),
+                "matrix": params.pop("matrix", {}),
+            }
+        if not not_time_out_flag:
+            raise RuntimeError("Execution of pipeline was time out")
         return params

ddeutil/workflow/schedule.py

@@ -9,19 +9,17 @@ from datetime import datetime
 from typing import Annotated
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
-from ddeutil.io import Params
 from ddeutil.workflow.vendors.__schedule import CronJob, CronRunner
 from pydantic import BaseModel, ConfigDict, Field
 from pydantic.functional_validators import field_validator
 from typing_extensions import Self
 
 from .__types import DictData
-from .exceptions import ScdlArgumentError
-from .loader import SimLoad
+from .loader import Loader
 
 
-class BaseScdl(BaseModel):
-    """Base Scdl (Schedule) Model"""
+class BaseSchedule(BaseModel):
+    """Base Schedule (Schedule) Model"""
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
@@ -37,14 +35,11 @@ class BaseScdl(BaseModel):
     def from_loader(
         cls,
         name: str,
-        params: Params,
         externals: DictData,
     ) -> Self:
-        loader: SimLoad = SimLoad(name, params=params, externals=externals)
+        loader: Loader = Loader(name, externals=externals)
         if "cronjob" not in loader.data:
-            raise ScdlArgumentError(
-                "cronjob", "Config does not set ``cronjob``"
-            )
+            raise ValueError("Config does not set ``cronjob`` value")
         return cls(cronjob=loader.data["cronjob"], extras=externals)
 
     @field_validator("tz")
@@ -66,16 +61,16 @@ class BaseScdl(BaseModel):
         return self.cronjob.schedule(date=(start.astimezone(ZoneInfo(self.tz))))
 
 
-class Scdl(BaseScdl):
-    """Scdl (Schedule) Model.
+class Schedule(BaseSchedule):
+    """Schedule (Schedule) Model.
 
     See Also:
         * ``generate()`` is the main usecase of this schedule object.
     """
 
 
-class ScdlBkk(Scdl):
-    """Asia Bangkok Scdl (Schedule) timezone Model.
+class ScheduleBkk(Schedule):
+    """Asia Bangkok Schedule (Schedule) timezone Model.
 
     This model use for change timezone from utc to Asia/Bangkok
     """
@@ -83,5 +78,5 @@ class ScdlBkk(Scdl):
     tz: Annotated[str, Field(description="Timezone")] = "Asia/Bangkok"
 
 
-class AwsScdl(BaseScdl):
+class AwsSchedule(BaseSchedule):
     """Implement Schedule for AWS Service."""

ddeutil/workflow/tasks/__init__.py

@@ -1,10 +1,6 @@
-from typing import Any
-
-from ddeutil.core import lazy
-
-registries: dict[str, Any] = {
-    "el-csv-to-parquet": {
-        "polars": lazy("ddeutil.workflow.tasks._polars.csv_to_parquet"),
-        "polars-dir": lazy("ddeutil.workflow.tasks._polars.csv_to_parquet_dir"),
-    },
-}
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from ._polars import *