ddeutil-workflow 0.0.1__py3-none-any.whl → 0.0.2__py3-none-any.whl

ddeutil/workflow/pipeline.py

@@ -6,40 +6,58 @@
 from __future__ import annotations
 
 import inspect
+import logging
 import subprocess
+from abc import ABC, abstractmethod
+from datetime import date, datetime
 from inspect import Parameter
 from subprocess import CompletedProcess
-from typing import Any, Callable, Optional, Union
+from typing import Any, Callable, Literal, Optional, Union
 
+from ddeutil.io.models.lineage import dt_now
 from pydantic import BaseModel, Field
+from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__regex import RegexConf
 from .__types import DictData
-from .exceptions import PipeArgumentError, PyException, TaskException
-from .loader import Loader, map_caller
+from .exceptions import PyException, TaskException
+from .loader import Loader, map_params
+from .utils import make_registry
 
 
-class StageResult(BaseModel): ...
+class BaseStage(BaseModel, ABC):
+    """Base Stage Model."""
 
+    id: Optional[str] = None
+    name: str
 
-class JobResult(BaseModel): ...
+    @abstractmethod
+    def execute(self, params: DictData) -> DictData:
+        raise NotImplementedError("Stage should implement ``execute`` method.")
 
+    def set_outputs(self, rs: DictData, params: DictData) -> DictData:
+        """Set outputs to params"""
+        if self.id is None:
+            return params
 
-class PipeResult(BaseModel): ...
+        if "stages" not in params:
+            params["stages"] = {}
 
+        params["stages"][self.id] = {"outputs": rs}
+        return params
 
-class EmptyStage(BaseModel):
-    """Empty stage that is doing nothing and logging the name of stage only."""
 
-    id: Optional[str] = None
-    name: str
+class EmptyStage(BaseStage):
+    """Empty stage that is doing nothing and logging the name of stage only."""
 
-    def execute(self, params: dict[str, Any]) -> dict[str, Any]:
+    def execute(self, params: DictData) -> DictData:
+        """Execute for the Empty stage that do only logging out."""
+        logging.info(f"Execute: {self.name!r}")
         return params
 
 
-class ShellStage(EmptyStage):
+class ShellStage(BaseStage):
     """Shell statement stage."""
 
     shell: str
@@ -50,9 +68,7 @@ class ShellStage(EmptyStage):
         """Prepare shell statement string that include newline"""
         return shell.replace("\n", ";")
 
-    def set_outputs(
-        self, rs: CompletedProcess, params: dict[str, Any]
-    ) -> dict[str, Any]:
+    def set_outputs(self, rs: CompletedProcess, params: DictData) -> DictData:
         """Set outputs to params"""
         # NOTE: skipping set outputs of stage execution when id does not set.
         if self.id is None:
@@ -71,7 +87,7 @@ class ShellStage(EmptyStage):
         }
         return params
 
-    def execute(self, params: dict[str, Any]) -> dict[str, Any]:
+    def execute(self, params: DictData) -> DictData:
         """Execute the Shell & Powershell statement with the Python build-in
         ``subprocess`` package.
         """
@@ -92,24 +108,22 @@ class ShellStage(EmptyStage):
         return params
 
 
-class PyStage(EmptyStage):
+class PyStage(BaseStage):
     """Python executor stage that running the Python statement that receive
     globals nad additional variables.
     """
 
     run: str
-    vars: dict[str, Any] = Field(default_factory=dict)
+    vars: DictData = Field(default_factory=dict)
 
-    def get_var(self, params: dict[str, Any]) -> dict[str, Any]:
+    def get_var(self, params: DictData) -> DictData:
         """Return variables"""
         rs = self.vars.copy()
         for p, v in self.vars.items():
-            rs[p] = map_caller(v, params)
+            rs[p] = map_params(v, params)
         return rs
 
-    def set_outputs(
-        self, lc: dict[str, Any], params: dict[str, Any]
-    ) -> dict[str, Any]:
+    def set_outputs(self, rs: DictData, params: DictData) -> DictData:
         """Set outputs to params"""
         # NOTE: skipping set outputs of stage execution when id does not set.
         if self.id is None:
@@ -120,25 +134,25 @@ class PyStage(EmptyStage):
 
         params["stages"][self.id] = {
             # NOTE: The output will fileter unnecessary keys from ``_locals``.
-            "outputs": {k: lc[k] for k in lc if k != "__annotations__"},
+            "outputs": {k: rs[k] for k in rs if k != "__annotations__"},
         }
         return params
 
-    def execute(self, params: dict[str, Any]) -> dict[str, Any]:
+    def execute(self, params: DictData) -> DictData:
         """Execute the Python statement that pass all globals and input params
         to globals argument on ``exec`` build-in function.
 
         :param params: A parameter that want to pass before run any statement.
-        :type params: dict[str, Any]
+        :type params: DictData
 
-        :rtype: dict[str, Any]
+        :rtype: DictData
         :returns: A parameters from an input that was mapped output if the stage
             ID was set.
         """
-        _globals: dict[str, Any] = globals() | params | self.get_var(params)
-        _locals: dict[str, Any] = {}
+        _globals: DictData = globals() | params | self.get_var(params)
+        _locals: DictData = {}
         try:
-            exec(map_caller(self.run, params), _globals, _locals)
+            exec(map_params(self.run, params), _globals, _locals)
         except Exception as err:
             raise PyException(
                 f"{err.__class__.__name__}: {err}\nRunning Statement:\n---\n"
@@ -151,14 +165,18 @@ class PyStage(EmptyStage):
 
 
 class TaskSearch(BaseModel):
+    """Task Search Model"""
+
     path: str
     func: str
     tag: str
 
 
-class TaskStage(EmptyStage):
+class TaskStage(BaseStage):
+    """Task executor stage that running the Python function."""
+
     task: str
-    args: dict[str, Any]
+    args: DictData
 
     @staticmethod
     def extract_task(task: str) -> Callable[[], Callable[[Any], Any]]:
@@ -167,27 +185,15 @@ class TaskStage(EmptyStage):
             raise ValueError("Task does not match with task format regex.")
         tasks = TaskSearch(**found.groupdict())
 
-        from ddeutil.core import import_string
-
-        try:
-            rgt = import_string(f"ddeutil.workflow.{tasks.path}.registries")
-            if tasks.func not in rgt:
-                raise NotImplementedError(
-                    f"ddeutil.workflow.{tasks.path}.registries does not "
-                    f"implement registry: {tasks.func}."
-                )
-        except ImportError:
-
-            # NOTE: Try to import this task function fom target module.
-            try:
-                return import_string(
-                    f"ddeutil.workflow.{tasks.path}.{tasks.func}"
-                )
-            except ImportError:
-                raise NotImplementedError(
-                    f"ddeutil.workflow.{tasks.path} does not implement "
-                    f"registries or {tasks.func}."
-                ) from None
+        # NOTE: Registry object should implement on this package only.
+        # TODO: This prefix value to search registry should dynamic with
+        #   config file.
+        rgt = make_registry(f"ddeutil.workflow.{tasks.path}")
+        if tasks.func not in rgt:
+            raise NotImplementedError(
+                f"ddeutil.workflow.{tasks.path}.registries does not "
+                f"implement registry: {tasks.func}."
+            )
 
         if tasks.tag not in rgt[tasks.func]:
             raise NotImplementedError(
@@ -197,7 +203,7 @@ class TaskStage(EmptyStage):
             )
         return rgt[tasks.func][tasks.tag]
 
-    def execute(self, params: dict[str, Any]) -> dict[str, Any]:
+    def execute(self, params: DictData) -> DictData:
         """Execute the Task function."""
         task_caller = self.extract_task(self.task)()
         if not callable(task_caller):
@@ -215,17 +221,11 @@ class TaskStage(EmptyStage):
                 f"does not set to args"
             )
         try:
-            rs = task_caller(**self.args)
+            rs = task_caller(**map_params(self.args, params))
         except Exception as err:
             raise TaskException(f"{err.__class__.__name__}: {err}") from err
-        return {"output": rs}
-
-
-class HookStage(EmptyStage):
-    hook: str
-    args: dict[str, Any]
-
-    def execute(self, params: dict[str, Any]) -> dict[str, Any]: ...
+        self.set_outputs(rs, params)
+        return params
 
 
 # NOTE: Order of parsing stage data
@@ -233,14 +233,24 @@ Stage = Union[
     PyStage,
     ShellStage,
     TaskStage,
-    HookStage,
     EmptyStage,
 ]
 
 
+class Strategy(BaseModel):
+    """Strategy Model"""
+
+    matrix: list[str] = Field(default_factory=list)
+    include: list[str] = Field(default_factory=list)
+    exclude: list[str] = Field(default_factory=list)
+
+
 class Job(BaseModel):
+    """Job Model"""
+
     stages: list[Stage] = Field(default_factory=list)
     needs: list[str] = Field(default_factory=list)
+    strategy: Strategy = Field(default_factory=Strategy)
 
     def stage(self, stage_id: str) -> Stage:
         for stage in self.stages:
@@ -248,7 +258,8 @@ class Job(BaseModel):
                 return stage
         raise ValueError(f"Stage ID {stage_id} does not exists")
 
-    def execute(self, params: dict[str, Any] | None = None) -> dict[str, Any]:
+    def execute(self, params: DictData | None = None) -> DictData:
+        """Execute job with passing dynamic parameters from the pipeline."""
         for stage in self.stages:
             # NOTE:
             #       I do not use below syntax because `params` dict be the
@@ -259,45 +270,148 @@ class Job(BaseModel):
         return params
 
 
-class Strategy(BaseModel):
-    matrix: list[str]
-    include: list[str]
-    exclude: list[str]
+class BaseParams(BaseModel, ABC):
+    """Base Parameter that use to make Params Model."""
 
+    desc: Optional[str] = None
+    required: bool = True
+    type: str
 
-class JobStrategy(Job):
-    """Strategy job"""
+    @abstractmethod
+    def receive(self, value: Optional[Any] = None) -> Any:
+        raise ValueError(
+            "Receive value and validate typing before return valid value."
+        )
+
+
+class DefaultParams(BaseParams):
+    """Default Parameter that will check default if it required"""
+
+    default: Optional[str] = None
+
+    @abstractmethod
+    def receive(self, value: Optional[Any] = None) -> Any:
+        raise ValueError(
+            "Receive value and validate typing before return valid value."
+        )
+
+    @model_validator(mode="after")
+    def check_default(self) -> Self:
+        if not self.required and self.default is None:
+            raise ValueError(
+                "Default should set when this parameter does not required."
+            )
+        return self
+
+
+class DatetimeParams(DefaultParams):
+    """Datetime parameter."""
+
+    type: Literal["datetime"] = "datetime"
+    required: bool = False
+    default: datetime = Field(default_factory=dt_now)
+
+    def receive(self, value: str | datetime | date | None = None) -> datetime:
+        if value is None:
+            return self.default
+
+        if isinstance(value, datetime):
+            return value
+        elif isinstance(value, date):
+            return datetime(value.year, value.month, value.day)
+        elif not isinstance(value, str):
+            raise ValueError(
+                f"Value that want to convert to datetime does not support for "
+                f"type: {type(value)}"
+            )
+        return datetime.fromisoformat(value)
+
+
+class StrParams(DefaultParams):
+    """String parameter."""
+
+    type: Literal["str"] = "str"
 
-    strategy: Strategy
+    def receive(self, value: Optional[str] = None) -> str | None:
+        if value is None:
+            return self.default
+        return str(value)
+
+
+class IntParams(DefaultParams):
+    """Integer parameter."""
+
+    type: Literal["int"] = "int"
+
+    def receive(self, value: Optional[int] = None) -> int | None:
+        if value is None:
+            return self.default
+        if not isinstance(value, int):
+            try:
+                return int(str(value))
+            except TypeError as err:
+                raise ValueError(
+                    f"Value that want to convert to integer does not support "
+                    f"for type: {type(value)}"
+                ) from err
+        return value
+
+
+class ChoiceParams(BaseParams):
+    type: Literal["choice"] = "choice"
+    options: list[str]
+
+    def receive(self, value: Optional[str] = None) -> str:
+        """Receive value that match with options."""
+        # NOTE:
+        #   Return the first value in options if does not pass any input value
+        if value is None:
+            return self.options[0]
+        if any(value not in self.options):
+            raise ValueError(f"{value} does not match any value in options")
+        return value
+
+
+Params = Union[
+    ChoiceParams,
+    DatetimeParams,
+    StrParams,
+]
 
 
 class Pipeline(BaseModel):
     """Pipeline Model"""
 
-    params: dict[str, Any] = Field(default_factory=dict)
+    params: dict[str, Params] = Field(default_factory=dict)
     jobs: dict[str, Job]
 
     @classmethod
     def from_loader(
         cls,
         name: str,
-        externals: DictData,
+        externals: Optional[DictData] = None,
     ) -> Self:
-        loader: Loader = Loader(name, externals=externals)
+        loader: Loader = Loader(name, externals=(externals or {}))
         if "jobs" not in loader.data:
-            raise PipeArgumentError("jobs", "Config does not set ``jobs``")
+            raise ValueError("Config does not set ``jobs`` value")
         return cls(
             jobs=loader.data["jobs"],
-            params=loader.params(),
+            params=loader.data["params"],
         )
 
     def job(self, name: str) -> Job:
-        """Return Job model that exists on this pipeline."""
+        """Return Job model that exists on this pipeline.
+
+        :param name: A job name that want to get from a mapping of job models.
+        :type name: str
+
+        :rtype: Job
+        """
         if name not in self.jobs:
             raise ValueError(f"Job {name} does not exists")
         return self.jobs[name]
 
-    def execute(self, params: dict[str, Any] | None = None):
+    def execute(self, params: DictData | None = None) -> DictData:
         """Execute pipeline with passing dynamic parameters.
 
         See Also:
@@ -312,18 +426,22 @@ class Pipeline(BaseModel):
             ... "<job-name>.stages.<stage-id>.outputs.<key>"
 
         """
-        params: dict[str, Any] = params or {}
+        params: DictData = params or {}
         check_key = tuple(f"{k!r}" for k in self.params if k not in params)
         if check_key:
             raise ValueError(
                 f"Parameters that needed on pipeline does not pass: "
                 f"{', '.join(check_key)}."
             )
-        params: dict[str, Any] = {
+
+        if any(p not in params for p in self.params if self.params[p].required):
+            raise ValueError("Required parameter does not pass")
+
+        params: DictData = {
             "params": (
                 params
                 | {
-                    k: self.params[k](params[k])
+                    k: self.params[k].receive(params[k])
                     for k in params
                     if k in self.params
                 }
@@ -331,7 +449,7 @@ class Pipeline(BaseModel):
         }
         for job_id in self.jobs:
             print(f"[PIPELINE]: Start execute the job: {job_id!r}")
-            job = self.jobs[job_id]
+            job: Job = self.jobs[job_id]
             # TODO: Condition on ``needs`` of this job was set. It should create
             #   multithreading process on this step.
             job.execute(params=params)

ddeutil/workflow/schedule.py

@@ -9,15 +9,13 @@ from datetime import datetime
 from typing import Annotated
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
-from ddeutil.io import Params
 from ddeutil.workflow.vendors.__schedule import CronJob, CronRunner
 from pydantic import BaseModel, ConfigDict, Field
 from pydantic.functional_validators import field_validator
 from typing_extensions import Self
 
 from .__types import DictData
-from .exceptions import ScdlArgumentError
-from .loader import SimLoad
+from .loader import Loader
 
 
 class BaseScdl(BaseModel):
@@ -37,14 +35,11 @@ class BaseScdl(BaseModel):
     def from_loader(
         cls,
         name: str,
-        params: Params,
         externals: DictData,
     ) -> Self:
-        loader: SimLoad = SimLoad(name, params=params, externals=externals)
+        loader: Loader = Loader(name, externals=externals)
         if "cronjob" not in loader.data:
-            raise ScdlArgumentError(
-                "cronjob", "Config does not set ``cronjob``"
-            )
+            raise ValueError("Config does not set ``cronjob`` value")
         return cls(cronjob=loader.data["cronjob"], extras=externals)
 
     @field_validator("tz")

ddeutil/workflow/tasks/__init__.py

@@ -1,10 +1,6 @@
-from typing import Any
-
-from ddeutil.core import lazy
-
-registries: dict[str, Any] = {
-    "el-csv-to-parquet": {
-        "polars": lazy("ddeutil.workflow.tasks._polars.csv_to_parquet"),
-        "polars-dir": lazy("ddeutil.workflow.tasks._polars.csv_to_parquet_dir"),
-    },
-}
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from ._polars import *

ddeutil/workflow/tasks/_pandas.py

@@ -0,0 +1,54 @@
+import logging
+import math
+
+try:
+    import pandas as pd
+
+    logging.debug(f"Polars version: {pd.__version__}")
+except ImportError as err:
+    raise ImportError(
+        "``split_iterable`` function want to use pandas package that does"
+        "not install on your interpreter."
+    ) from err
+
+
+def split_iterable(iterable, chunk_size=None, generator_flag: bool = True):
+    """
+    Split an iterable into mini batch with batch length of batch_number
+    supports batch of a pandas dataframe
+    usage:
+        >> for i in split_iterable([1,2,3,4,5], chunk_size=2):
+        >>    print(i)
+        [1, 2]
+        [3, 4]
+        [5]
+
+        for idx, mini_data in split_iterable(batch(df, chunk_size=10)):
+            print(idx)
+            print(mini_data)
+    """
+
+    chunk_size: int = chunk_size or 25000
+    num_chunks = math.ceil(len(iterable) / chunk_size)
+    if generator_flag:
+        for _ in range(num_chunks):
+            if isinstance(iterable, pd.DataFrame):
+                yield iterable.iloc[_ * chunk_size : (_ + 1) * chunk_size]
+            else:
+                yield iterable[_ * chunk_size : (_ + 1) * chunk_size]
+    else:
+        _chunks: list = []
+        for _ in range(num_chunks):
+            if isinstance(iterable, pd.DataFrame):
+                _chunks.append(
+                    iterable.iloc[_ * chunk_size : (_ + 1) * chunk_size]
+                )
+            else:
+                _chunks.append(iterable[_ * chunk_size : (_ + 1) * chunk_size])
+        return _chunks
+
+
+def chunks(dataframe: pd.DataFrame, n: int):
+    """Yield successive n-sized chunks from dataframe."""
+    for i in range(0, len(dataframe), n):
+        yield dataframe.iloc[i : i + n]

ddeutil/workflow/tasks/_polars.py

@@ -11,23 +11,41 @@ from uuid import uuid4
 import polars as pl
 import pyarrow.parquet as pq
 from ddeutil.workflow.dataset import PolarsCsv, PolarsParq
+from ddeutil.workflow.utils import tag
 
 
+def polars_dtype():
+    return {
+        "str": pl.Utf8,
+        "int": pl.Int32,
+    }
+
+
+@tag("polars-dir", name="el-csv-to-parquet")
 def csv_to_parquet_dir(
     source: str,
     sink: str,
     conversion: dict[str, Any] | None = None,
-):
+) -> dict[str, int]:
+    """Extract Load data from CSV to Parquet file.
+
+    :param source:
+    :param sink:
+    :param conversion:
+    """
     print("Start EL for CSV to Parquet with Polars Engine")
     print("---")
     # STEP 01: Read the source data to Polars.
     src_dataset: PolarsCsv = PolarsCsv.from_loader(name=source, externals={})
-    src_df = src_dataset.load()
+    src_df: pl.DataFrame = src_dataset.load()
     print(src_df)
 
     # STEP 02: Schema conversion on Polars DataFrame.
     conversion: dict[str, Any] = conversion or {}
     if conversion:
+        src_df = src_df.with_columns(
+            *[pl.col(c).cast(col.type).alias(col.name) for c, col in conversion]
+        )
         print("Start Schema Conversion ...")
 
     # STEP 03: Write data to parquet file format.
@@ -39,3 +57,28 @@ def csv_to_parquet_dir(
         basename_template=f"{sink.object}-{uuid4().hex}-{{i}}.snappy.parquet",
     )
     return {"records": src_df.select(pl.len()).item()}
+
+
+@tag("polars-dir-scan", name="el-csv-to-parquet")
+def csv_to_parquet_dir_scan(
+    source: str,
+    sink: str,
+    conversion: dict[str, Any] | None = None,
+) -> dict[str, int]:
+    print("Start EL for CSV to Parquet with Polars Engine")
+    print("---")
+    # STEP 01: Read the source data to Polars.
+    src_dataset: PolarsCsv = PolarsCsv.from_loader(name=source, externals={})
+    src_df: pl.LazyFrame = src_dataset.scan()
+
+    if conversion:
+        ...
+
+    sink = PolarsParq.from_loader(name=sink, externals={})
+    pq.write_to_dataset(
+        table=src_df.collect().to_arrow(),
+        root_path=f"{sink.conn.endpoint}/{sink.object}",
+        compression="snappy",
+        basename_template=f"{sink.object}-{uuid4().hex}-{{i}}.snappy.parquet",
+    )
+    return {"records": src_df.select(pl.len()).collect().item()}