ddeutil-workflow 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

ddeutil/workflow/tasks/_pandas.py

@@ -0,0 +1,54 @@
+import logging
+import math
+
+try:
+    import pandas as pd
+
+    logging.debug(f"Pandas version: {pd.__version__}")
+except ImportError as err:
+    raise ImportError(
+        "``split_iterable`` function want to use pandas package that does"
+        "not install on your interpreter."
+    ) from err
+
+
+def split_iterable(iterable, chunk_size=None, generator_flag: bool = True):
+    """
+    Split an iterable into mini batch with batch length of batch_number
+    supports batch of a pandas dataframe
+    usage:
+        >> for i in split_iterable([1,2,3,4,5], chunk_size=2):
+        >>    print(i)
+        [1, 2]
+        [3, 4]
+        [5]
+
+        for idx, mini_data in split_iterable(batch(df, chunk_size=10)):
+            print(idx)
+            print(mini_data)
+    """
+
+    chunk_size: int = chunk_size or 25000
+    num_chunks = math.ceil(len(iterable) / chunk_size)
+    if generator_flag:
+        for _ in range(num_chunks):
+            if isinstance(iterable, pd.DataFrame):
+                yield iterable.iloc[_ * chunk_size : (_ + 1) * chunk_size]
+            else:
+                yield iterable[_ * chunk_size : (_ + 1) * chunk_size]
+    else:
+        _chunks: list = []
+        for _ in range(num_chunks):
+            if isinstance(iterable, pd.DataFrame):
+                _chunks.append(
+                    iterable.iloc[_ * chunk_size : (_ + 1) * chunk_size]
+                )
+            else:
+                _chunks.append(iterable[_ * chunk_size : (_ + 1) * chunk_size])
+        return _chunks
+
+
+def chunks(dataframe: pd.DataFrame, n: int):
+    """Yield successive n-sized chunks from dataframe."""
+    for i in range(0, len(dataframe), n):
+        yield dataframe.iloc[i : i + n]

ddeutil/workflow/tasks/_polars.py

@@ -5,29 +5,55 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
+import logging
 from typing import Any
 from uuid import uuid4
 
-import polars as pl
+try:
+    import polars as pl
+
+    logging.debug(f"Polars version: {pl.__version__}")
+except ImportError:
+    raise ImportError(
+        "Please install polars if you want to use any relate task"
+    ) from None
 import pyarrow.parquet as pq
-from ddeutil.workflow.dataset import PolarsCsv, PolarsParq
+from ddeutil.workflow.utils import tag
+from ddeutil.workflow.vendors.pl import PolarsCsv, PolarsParq
+
+
+def polars_dtype():
+    return {
+        "str": pl.Utf8,
+        "int": pl.Int32,
+    }
 
 
+@tag("polars-dir", name="el-csv-to-parquet")
 def csv_to_parquet_dir(
     source: str,
     sink: str,
     conversion: dict[str, Any] | None = None,
-):
+) -> dict[str, int]:
+    """Extract Load data from CSV to Parquet file.
+
+    :param source:
+    :param sink:
+    :param conversion:
+    """
     print("Start EL for CSV to Parquet with Polars Engine")
     print("---")
     # STEP 01: Read the source data to Polars.
     src_dataset: PolarsCsv = PolarsCsv.from_loader(name=source, externals={})
-    src_df = src_dataset.load()
+    src_df: pl.DataFrame = src_dataset.load()
     print(src_df)
 
     # STEP 02: Schema conversion on Polars DataFrame.
     conversion: dict[str, Any] = conversion or {}
     if conversion:
+        src_df = src_df.with_columns(
+            *[pl.col(c).cast(col.type).alias(col.name) for c, col in conversion]
+        )
         print("Start Schema Conversion ...")
 
     # STEP 03: Write data to parquet file format.
@@ -39,3 +65,28 @@ def csv_to_parquet_dir(
         basename_template=f"{sink.object}-{uuid4().hex}-{{i}}.snappy.parquet",
     )
     return {"records": src_df.select(pl.len()).item()}
+
+
+@tag("polars-dir-scan", name="el-csv-to-parquet")
+def csv_to_parquet_dir_scan(
+    source: str,
+    sink: str,
+    conversion: dict[str, Any] | None = None,
+) -> dict[str, int]:
+    print("Start EL for CSV to Parquet with Polars Engine")
+    print("---")
+    # STEP 01: Read the source data to Polars.
+    src_dataset: PolarsCsv = PolarsCsv.from_loader(name=source, externals={})
+    src_df: pl.LazyFrame = src_dataset.scan()
+
+    if conversion:
+        ...
+
+    sink = PolarsParq.from_loader(name=sink, externals={})
+    pq.write_to_dataset(
+        table=src_df.collect().to_arrow(),
+        root_path=f"{sink.conn.endpoint}/{sink.object}",
+        compression="snappy",
+        basename_template=f"{sink.object}-{uuid4().hex}-{{i}}.snappy.parquet",
+    )
+    return {"records": src_df.select(pl.len()).collect().item()}

ddeutil/workflow/utils.py

@@ -0,0 +1,180 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import inspect
+from abc import ABC, abstractmethod
+from datetime import date, datetime
+from functools import wraps
+from importlib import import_module
+from typing import Any, Callable, Literal, Optional, Protocol, Union
+
+from ddeutil.core import lazy
+from ddeutil.io.models.lineage import dt_now
+from pydantic import BaseModel, Field
+from pydantic.functional_validators import model_validator
+from typing_extensions import Self
+
+
+class TagFunc(Protocol):
+    """Tag Function Protocol"""
+
+    name: str
+    tag: str
+
+    def __call__(self, *args, **kwargs): ...
+
+
+def tag(tag_value: str, name: str | None = None):
+    """Tag decorator function that set function attributes, ``tag`` and ``name``
+    for making registries variable.
+
+    :param: tag_value: A tag value for make different use-case of a function.
+    :param: name: A name that keeping in registries.
+    """
+
+    def func_internal(func: TagFunc):
+        func.tag = tag_value
+        func.name = name or func.__name__.replace("_", "-")
+
+        @wraps(func)
+        def wrapped(*args, **kwargs):
+            return func(*args, **kwargs)
+
+        return wrapped
+
+    return func_internal
+
+
+def make_registry(module: str) -> dict[str, dict[str, Callable[[], TagFunc]]]:
+    """Return registries of all functions that able to called with task."""
+    rs: dict[str, dict[str, Callable[[], Callable]]] = {}
+    for fstr, func in inspect.getmembers(
+        import_module(module), inspect.isfunction
+    ):
+        if not hasattr(func, "tag"):
+            continue
+
+        if func.name in rs:
+            if func.tag in rs[func.name]:
+                raise ValueError(
+                    f"The tag {func.tag!r} already exists on module {module}"
+                )
+            rs[func.name][func.tag] = lazy(f"{module}.{fstr}")
+            continue
+
+        # NOTE: Create new register name if it not exists
+        rs[func.name] = {func.tag: lazy(f"{module}.{fstr}")}
+    return rs
+
+
+class BaseParams(BaseModel, ABC):
+    """Base Parameter that use to make Params Model."""
+
+    desc: Optional[str] = None
+    required: bool = True
+    type: str
+
+    @abstractmethod
+    def receive(self, value: Optional[Any] = None) -> Any:
+        raise ValueError(
+            "Receive value and validate typing before return valid value."
+        )
+
+
+class DefaultParams(BaseParams):
+    """Default Parameter that will check default if it required"""
+
+    default: Optional[str] = None
+
+    @abstractmethod
+    def receive(self, value: Optional[Any] = None) -> Any:
+        raise ValueError(
+            "Receive value and validate typing before return valid value."
+        )
+
+    @model_validator(mode="after")
+    def check_default(self) -> Self:
+        if not self.required and self.default is None:
+            raise ValueError(
+                "Default should set when this parameter does not required."
+            )
+        return self
+
+
+class DatetimeParams(DefaultParams):
+    """Datetime parameter."""
+
+    type: Literal["datetime"] = "datetime"
+    required: bool = False
+    default: datetime = Field(default_factory=dt_now)
+
+    def receive(self, value: str | datetime | date | None = None) -> datetime:
+        if value is None:
+            return self.default
+
+        if isinstance(value, datetime):
+            return value
+        elif isinstance(value, date):
+            return datetime(value.year, value.month, value.day)
+        elif not isinstance(value, str):
+            raise ValueError(
+                f"Value that want to convert to datetime does not support for "
+                f"type: {type(value)}"
+            )
+        return datetime.fromisoformat(value)
+
+
+class StrParams(DefaultParams):
+    """String parameter."""
+
+    type: Literal["str"] = "str"
+
+    def receive(self, value: Optional[str] = None) -> str | None:
+        if value is None:
+            return self.default
+        return str(value)
+
+
+class IntParams(DefaultParams):
+    """Integer parameter."""
+
+    type: Literal["int"] = "int"
+
+    def receive(self, value: Optional[int] = None) -> int | None:
+        if value is None:
+            return self.default
+        if not isinstance(value, int):
+            try:
+                return int(str(value))
+            except TypeError as err:
+                raise ValueError(
+                    f"Value that want to convert to integer does not support "
+                    f"for type: {type(value)}"
+                ) from err
+        return value
+
+
+class ChoiceParams(BaseParams):
+    type: Literal["choice"] = "choice"
+    options: list[str]
+
+    def receive(self, value: Optional[str] = None) -> str:
+        """Receive value that match with options."""
+        # NOTE:
+        #   Return the first value in options if does not pass any input value
+        if value is None:
+            return self.options[0]
+        if any(value not in self.options):
+            raise ValueError(f"{value} does not match any value in options")
+        return value
+
+
+Params = Union[
+    ChoiceParams,
+    DatetimeParams,
+    StrParams,
+]

ddeutil/workflow/vendors/__dataset.py

@@ -0,0 +1,127 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Annotated, Any, Optional
+
+from fmtutil import Datetime, FormatterGroupType, make_group
+from fmtutil.utils import escape_fmt_group
+from pydantic import BaseModel, Field
+from typing_extensions import Self
+
+from ..__types import DictData, TupleStr
+from ..conn import SubclassConn
+from ..loader import Loader
+
+EXCLUDED_EXTRAS: TupleStr = ("type",)
+OBJ_FMTS: FormatterGroupType = make_group({"datetime": Datetime})
+
+
+class BaseDataset(BaseModel):
+    """Base Dataset Model. This model implement only loading construction."""
+
+    conn: Annotated[SubclassConn, Field(description="Connection Model")]
+    endpoint: Annotated[
+        Optional[str],
+        Field(description="Endpoint of connection"),
+    ] = None
+    object: str = Field(description="Dataset object that want to contract")
+    features: list = Field(default_factory=list)
+    extras: dict[str, Any] = Field(default_factory=dict)
+
+    @classmethod
+    def from_loader(
+        cls,
+        name: str,
+        externals: DictData,
+    ) -> Self:
+        """Construct Connection with Loader object with specific config name.
+
+        :param name: A name of dataset that want to load from config file.
+        :param externals: An external parameters.
+        """
+        loader: Loader = Loader(name, externals=externals)
+
+        # NOTE: Validate the config type match with current dataset model
+        if loader.type != cls:
+            raise ValueError(f"Type {loader.type} does not match with {cls}")
+
+        filter_data: DictData = {
+            k: loader.data.pop(k)
+            for k in loader.data.copy()
+            if k not in cls.model_fields and k not in EXCLUDED_EXTRAS
+        }
+
+        if "conn" not in loader.data:
+            raise ValueError("Dataset config does not set ``conn`` value")
+
+        # NOTE: Start loading connection config
+        conn_name: str = loader.data.pop("conn")
+        conn_loader: Loader = Loader(conn_name, externals=externals)
+        conn_model: SubclassConn = conn_loader.type.from_loader(
+            name=conn_name, externals=externals
+        )
+
+        # NOTE: Override ``endpoint`` value to getter connection data.
+        if "endpoint" in loader.data:
+            # NOTE: Update endpoint path without Pydantic validator.
+            conn_model.__dict__["endpoint"] = loader.data["endpoint"]
+        else:
+            loader.data.update({"endpoint": conn_model.endpoint})
+        return cls.model_validate(
+            obj={
+                "extras": (
+                    loader.data.pop("extras", {}) | filter_data | externals
+                ),
+                "conn": conn_model,
+                **loader.data,
+            }
+        )
+
+
+class Dataset(BaseDataset):
+
+    def exists(self) -> bool:
+        raise NotImplementedError("Object exists does not implement")
+
+    def format_object(
+        self,
+        _object: str | None = None,
+        dt: str | datetime | None = None,
+    ) -> str:
+        """Format the object value that implement datetime"""
+        if dt is None:
+            dt = datetime.now()
+        dt: datetime = (
+            dt if isinstance(dt, datetime) else datetime.fromisoformat(dt)
+        )
+        return (
+            OBJ_FMTS({"datetime": dt})
+            .format(escape_fmt_group(_object or self.object))
+            .replace("\\", "")
+        )
+
+
+class FlDataset(Dataset):
+
+    def exists(self) -> bool:
+        return self.conn.find_object(self.object)
+
+
+class TblDataset(Dataset):
+
+    def exists(self) -> bool:
+        return self.conn.find_object(self.object)
+
+
+class FlDataFrame(Dataset):
+
+    def exists(self) -> bool:
+        return self.conn.find_object(self.object)
+
+
+class TblDataFrame(Dataset): ...

ddeutil/workflow/vendors/pd.py

@@ -0,0 +1,13 @@
+class PandasCSV: ...
+
+
+class PandasJson: ...
+
+
+class PandasParq: ...
+
+
+class PandasDb: ...
+
+
+class PandasExcel: ...

ddeutil/workflow/vendors/pg.py

@@ -0,0 +1,11 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+from .__dataset import TblDataset
+
+
+class PostgresTbl(TblDataset): ...

ddeutil/workflow/{dataset.py → vendors/pl.py}

@@ -6,12 +6,10 @@
 from __future__ import annotations
 
 from datetime import datetime
-from typing import Annotated, Any, Optional
+from typing import Any, Optional
 
 from fmtutil import Datetime, FormatterGroupType, make_group
-from fmtutil.utils import escape_fmt_group
 from pydantic import BaseModel, Field
-from typing_extensions import Self
 
 try:
     import polars as pl
@@ -20,137 +18,11 @@ except ImportError:
         "Please install polars package\n\t\t$ pip install polars"
     ) from None
 
-from .__types import DictData, TupleStr
-from .conn import SubclassConn
-from .loader import Loader
+from ..__types import TupleStr
+from .__dataset import FlDataFrame, TblDataFrame
 
 EXCLUDED_EXTRAS: TupleStr = ("type",)
-OBJ_FMTS: FormatterGroupType = make_group(
-    {
-        "datetime": Datetime,
-    }
-)
-
-
-class BaseDataset(BaseModel):
-    """Base Dataset Model. This model implement only loading constructor."""
-
-    conn: Annotated[SubclassConn, Field(description="Connection Model")]
-    endpoint: Annotated[
-        Optional[str],
-        Field(description="Endpoint of connection"),
-    ] = None
-    object: str
-    features: list = Field(default_factory=list)
-    extras: dict[str, Any] = Field(default_factory=dict)
-
-    @classmethod
-    def from_loader(
-        cls,
-        name: str,
-        externals: DictData,
-    ) -> Self:
-        """Construct Connection with Loader object with specific config name.
-
-        :param name: A name of dataset that want to load from config file.
-        :param externals: An external parameters.
-        """
-        loader: Loader = Loader(name, externals=externals)
-
-        # NOTE: Validate the config type match with current dataset model
-        if loader.type != cls:
-            raise ValueError(f"Type {loader.type} does not match with {cls}")
-
-        filter_data: DictData = {
-            k: loader.data.pop(k)
-            for k in loader.data.copy()
-            if k not in cls.model_fields and k not in EXCLUDED_EXTRAS
-        }
-
-        if "conn" not in loader.data:
-            raise ValueError("Dataset config does not set ``conn`` value")
-
-        # NOTE: Start loading connection config
-        conn_name: str = loader.data.pop("conn")
-        conn_loader: Loader = Loader(conn_name, externals=externals)
-        conn_model: SubclassConn = conn_loader.type.from_loader(
-            name=conn_name, externals=externals
-        )
-
-        # NOTE: Override ``endpoint`` value to getter connection data.
-        if "endpoint" in loader.data:
-            # NOTE: Update endpoint path without Pydantic validator.
-            conn_model.__dict__["endpoint"] = loader.data["endpoint"]
-        else:
-            loader.data.update({"endpoint": conn_model.endpoint})
-        return cls.model_validate(
-            obj={
-                "extras": (
-                    loader.data.pop("extras", {}) | filter_data | externals
-                ),
-                "conn": conn_model,
-                **loader.data,
-            }
-        )
-
-
-class Dataset(BaseDataset):
-
-    def exists(self) -> bool:
-        raise NotImplementedError("Object exists does not implement")
-
-    def format_object(
-        self,
-        _object: str | None = None,
-        dt: str | datetime | None = None,
-    ) -> str:
-        """Format the object value that implement datetime"""
-        if dt is None:
-            dt = datetime.now()
-        dt: datetime = (
-            dt if isinstance(dt, datetime) else datetime.fromisoformat(dt)
-        )
-        return (
-            OBJ_FMTS({"datetime": dt})
-            .format(escape_fmt_group(_object or self.object))
-            .replace("\\", "")
-        )
-
-
-class FlDataset(Dataset):
-
-    def exists(self) -> bool:
-        return self.conn.find_object(self.object)
-
-
-class TblDataset(Dataset):
-
-    def exists(self) -> bool:
-        return self.conn.find_object(self.object)
-
-
-class FlDataFrame(Dataset):
-
-    def exists(self) -> bool:
-        return self.conn.find_object(self.object)
-
-
-class TblDataFrame(Dataset): ...
-
-
-class PandasCSV: ...
-
-
-class PandasJson: ...
-
-
-class PandasParq: ...
-
-
-class PandasDb: ...
-
-
-class PandasExcel: ...
+OBJ_FMTS: FormatterGroupType = make_group({"datetime": Datetime})
 
 
 class PolarsCsvArgs(BaseModel):
@@ -297,10 +169,4 @@ class PolarsParq(FlDataFrame):
         )
 
 
-class PostgresTbl(TblDataset): ...
-
-
-class SqliteTbl(TblDataset): ...
-
-
 class PolarsPostgres(TblDataFrame): ...