ddeutil-workflow 0.0.4__py3-none-any.whl → 0.0.6__py3-none-any.whl

ddeutil_workflow-0.0.6.dist-info/RECORD

@@ -0,0 +1,15 @@
+ddeutil/workflow/__about__.py,sha256=VEYa91VchyTUnF57lvvquHvitTViBxxLXuhcEnr4TAY,27
+ddeutil/workflow/__init__.py,sha256=Y5wLiJ0zS1CfoSOZ0oo7OL3LNMKvPmpUO4fVHuAOv8E,429
+ddeutil/workflow/__types.py,sha256=PfwDZBnTwe2JImD7UFS0J6Nq-1TcjBGebOVzJZoSuTQ,1354
+ddeutil/workflow/exceptions.py,sha256=9O12c4aNLi0dyjVBgCLveV3HN9PXcZfwFfLXdgm3Ffs,626
+ddeutil/workflow/loader.py,sha256=_ZD-XP5P7VbUeqItrUVPaKIZu6dMUZ2aywbCbReW1hQ,2778
+ddeutil/workflow/on.py,sha256=YoEqDbzJUwqOA3JRltbvlYr0rNTtxdmb7cWMxl8U19k,6717
+ddeutil/workflow/pipeline.py,sha256=8mIvY34_fsiqscBa9JB94MgN3Km5fkuD2iaNZSAQVuM,17843
+ddeutil/workflow/scheduler.py,sha256=FqmkvWCqwJ4eRf8aDn5Ce4FcNWqmcvu2aTTfL34lfgs,22184
+ddeutil/workflow/stage.py,sha256=bDJiGS21gYlYbFDnLTKH9aIbXfej9fT-V1ADoPX7w4s,13829
+ddeutil/workflow/utils.py,sha256=xapKxxnqIzlbKA45GaRcWn-VL30AhE7M8f46ynr-vbI,11173
+ddeutil_workflow-0.0.6.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.6.dist-info/METADATA,sha256=5X6ewXGn96MR9rDhVmmaoTuIwuRmfi72t7ezM8wxbvw,8612
+ddeutil_workflow-0.0.6.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+ddeutil_workflow-0.0.6.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.6.dist-info/RECORD,,

{ddeutil_workflow-0.0.4.dist-info → ddeutil_workflow-0.0.6.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: setuptools (72.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

ddeutil/workflow/__regex.py

@@ -1,44 +0,0 @@
-# -------------------------------------------------------------------------
-# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
-# Licensed under the MIT License. See LICENSE in the project root for
-# license information.
-# --------------------------------------------------------------------------
-import re
-from re import (
-    IGNORECASE,
-    MULTILINE,
-    UNICODE,
-    VERBOSE,
-    Pattern,
-)
-
-
-class RegexConf:
-    """Regular expression config."""
-
-    # NOTE: Search caller
-    __re_caller: str = r"""
-        \$
-        {{
-            \s*(?P<caller>
-                [a-zA-Z0-9_.\s'\"\[\]\(\)\-\{}]+?
-            )\s*
-        }}
-    """
-    RE_CALLER: Pattern = re.compile(
-        __re_caller, MULTILINE | IGNORECASE | UNICODE | VERBOSE
-    )
-
-    # NOTE: Search task
-    __re_task_fmt: str = r"""
-        ^
-            (?P<path>[^/@]+)
-            /
-            (?P<func>[^@]+)
-            @
-            (?P<tag>.+)
-        $
-    """
-    RE_TASK_FMT: Pattern = re.compile(
-        __re_task_fmt, MULTILINE | IGNORECASE | UNICODE | VERBOSE
-    )

ddeutil/workflow/conn.py

@@ -1,240 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
-# Licensed under the MIT License. See LICENSE in the project root for
-# license information.
-# ------------------------------------------------------------------------------
-from __future__ import annotations
-
-import logging
-from collections.abc import Iterator
-from pathlib import Path
-from typing import Annotated, Any, Literal, Optional, TypeVar
-
-from ddeutil.io.models.conn import Conn as ConnModel
-from pydantic import BaseModel, ConfigDict, Field
-from pydantic.functional_validators import field_validator
-from pydantic.types import SecretStr
-from typing_extensions import Self
-
-from .__types import DictData, TupleStr
-from .loader import Loader
-
-EXCLUDED_EXTRAS: TupleStr = (
-    "type",
-    "url",
-)
-
-
-class BaseConn(BaseModel):
-    """Base Conn (Connection) Model"""
-
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-
-    # NOTE: This is fields
-    dialect: str
-    host: Optional[str] = None
-    port: Optional[int] = None
-    user: Optional[str] = None
-    pwd: Optional[SecretStr] = None
-    endpoint: str
-    extras: Annotated[
-        DictData,
-        Field(default_factory=dict, description="Extras mapping of parameters"),
-    ]
-
-    @classmethod
-    def from_dict(cls, values: DictData) -> Self:
-        """Construct Connection Model from dict data. This construct is
-        different with ``.model_validate()`` because it will prepare the values
-        before using it if the data dose not have 'url'.
-
-        :param values: A dict data that use to construct this model.
-        """
-        # NOTE: filter out the fields of this model.
-        filter_data: DictData = {
-            k: values.pop(k)
-            for k in values.copy()
-            if k not in cls.model_fields and k not in EXCLUDED_EXTRAS
-        }
-        if "url" in values:
-            url: ConnModel = ConnModel.from_url(values.pop("url"))
-            return cls(
-                dialect=url.dialect,
-                host=url.host,
-                port=url.port,
-                user=url.user,
-                pwd=url.pwd,
-                # NOTE:
-                #   I will replace None endpoint with memory value for SQLite
-                #   connection string.
-                endpoint=(url.endpoint or "memory"),
-                # NOTE: This order will show that externals this the top level.
-                extras=(url.options | filter_data),
-            )
-        return cls.model_validate(
-            obj={
-                "extras": (values.pop("extras", {}) | filter_data),
-                **values,
-            }
-        )
-
-    @classmethod
-    def from_loader(cls, name: str, externals: DictData) -> Self:
-        """Construct Connection with Loader object with specific config name.
-
-        :param name: A config name.
-        :param externals: A external data that want to adding to extras.
-        """
-        loader: Loader = Loader(name, externals=externals)
-        # NOTE: Validate the config type match with current connection model
-        if loader.type != cls:
-            raise ValueError(f"Type {loader.type} does not match with {cls}")
-        return cls.from_dict(
-            {
-                "extras": (loader.data.pop("extras", {}) | externals),
-                **loader.data,
-            }
-        )
-
-    @field_validator("endpoint")
-    def __prepare_slash(cls, value: str) -> str:
-        """Prepare slash character that map double form URL model loading."""
-        if value.startswith("//"):
-            return value[1:]
-        return value
-
-
-class Conn(BaseConn):
-    """Conn (Connection) Model that implement any necessary methods. This object
-    should be the base for abstraction to any connection model object.
-    """
-
-    def get_spec(self) -> str:
-        """Return full connection url that construct from all fields."""
-        return (
-            f"{self.dialect}://{self.user or ''}"
-            f"{f':{self.pwd}' if self.pwd else ''}"
-            f"{self.host or ''}{f':{self.port}' if self.port else ''}"
-            f"/{self.endpoint}"
-        )
-
-    def ping(self) -> bool:
-        """Ping the connection that able to use with this field value."""
-        raise NotImplementedError("Ping does not implement")
-
-    def glob(self, pattern: str) -> Iterator[Any]:
-        """Return a list of object from the endpoint of this connection."""
-        raise NotImplementedError("Glob does not implement")
-
-    def find_object(self, _object: str):
-        raise NotImplementedError("Glob does not implement")
-
-
-class FlSys(Conn):
-    """File System Connection."""
-
-    dialect: Literal["local"] = "local"
-
-    def ping(self) -> bool:
-        return Path(self.endpoint).exists()
-
-    def glob(self, pattern: str) -> Iterator[Path]:
-        yield from Path(self.endpoint).rglob(pattern=pattern)
-
-    def find_object(self, _object: str) -> bool:
-        return (Path(self.endpoint) / _object).exists()
-
-
-class SFTP(Conn):
-    """SFTP Server Connection."""
-
-    dialect: Literal["sftp"] = "sftp"
-
-    def __client(self):
-        from .vendors.sftp import WrapSFTP
-
-        return WrapSFTP(
-            host=self.host,
-            port=self.port,
-            user=self.user,
-            pwd=self.pwd.get_secret_value(),
-        )
-
-    def ping(self) -> bool:
-        with self.__client().simple_client():
-            return True
-
-    def glob(self, pattern: str) -> Iterator[str]:
-        yield from self.__client().walk(pattern=pattern)
-
-
-class Db(Conn):
-    """RDBMS System Connection"""
-
-    def ping(self) -> bool:
-        from sqlalchemy import create_engine
-        from sqlalchemy.engine import URL, Engine
-        from sqlalchemy.exc import OperationalError
-
-        engine: Engine = create_engine(
-            url=URL.create(
-                self.dialect,
-                username=self.user,
-                password=self.pwd.get_secret_value() if self.pwd else None,
-                host=self.host,
-                port=self.port,
-                database=self.endpoint,
-                query={},
-            ),
-            execution_options={},
-        )
-        try:
-            return engine.connect()
-        except OperationalError as err:
-            logging.warning(str(err))
-            return False
-
-
-class SQLite(Db):
-    dialect: Literal["sqlite"]
-
-
-class ODBC(Conn): ...
-
-
-class Doc(Conn):
-    """No SQL System Connection"""
-
-
-class Mongo(Doc): ...
-
-
-class SSHCred(BaseModel):
-    ssh_host: str
-    ssh_user: str
-    ssh_password: Optional[SecretStr] = Field(default=None)
-    ssh_private_key: Optional[str] = Field(default=None)
-    ssh_private_key_pwd: Optional[SecretStr] = Field(default=None)
-    ssh_port: int = Field(default=22)
-
-
-class S3Cred(BaseModel):
-    aws_access_key: str
-    aws_secret_access_key: SecretStr
-    region: str = Field(default="ap-southeast-1")
-    role_arn: Optional[str] = Field(default=None)
-    role_name: Optional[str] = Field(default=None)
-    mfa_serial: Optional[str] = Field(default=None)
-
-
-class AZServPrinCred(BaseModel):
-    tenant: str
-    client_id: str
-    secret_id: SecretStr
-
-
-class GoogleCred(BaseModel):
-    google_json_path: str
-
-
-SubclassConn = TypeVar("SubclassConn", bound=Conn)

ddeutil/workflow/schedule.py

@@ -1,82 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
-# Licensed under the MIT License. See LICENSE in the project root for
-# license information.
-# ------------------------------------------------------------------------------
-from __future__ import annotations
-
-from datetime import datetime
-from typing import Annotated
-from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
-
-from ddeutil.workflow.vendors.__schedule import CronJob, CronRunner
-from pydantic import BaseModel, ConfigDict, Field
-from pydantic.functional_validators import field_validator
-from typing_extensions import Self
-
-from .__types import DictData
-from .loader import Loader
-
-
-class BaseSchedule(BaseModel):
-    """Base Schedule (Schedule) Model"""
-
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-
-    # NOTE: This is fields
-    cronjob: Annotated[CronJob, Field(description="Cron job of this schedule")]
-    tz: Annotated[str, Field(description="Timezone")] = "utc"
-    extras: Annotated[
-        DictData,
-        Field(default_factory=dict, description="Extras mapping of parameters"),
-    ]
-
-    @classmethod
-    def from_loader(
-        cls,
-        name: str,
-        externals: DictData,
-    ) -> Self:
-        loader: Loader = Loader(name, externals=externals)
-        if "cronjob" not in loader.data:
-            raise ValueError("Config does not set ``cronjob`` value")
-        return cls(cronjob=loader.data["cronjob"], extras=externals)
-
-    @field_validator("tz")
-    def __validate_tz(cls, value: str):
-        try:
-            _ = ZoneInfo(value)
-            return value
-        except ZoneInfoNotFoundError as err:
-            raise ValueError(f"Invalid timezone: {value}") from err
-
-    @field_validator("cronjob", mode="before")
-    def __prepare_cronjob(cls, value: str | CronJob) -> CronJob:
-        return CronJob(value) if isinstance(value, str) else value
-
-    def generate(self, start: str | datetime) -> CronRunner:
-        """Return Cron runner object."""
-        if not isinstance(start, datetime):
-            start: datetime = datetime.fromisoformat(start)
-        return self.cronjob.schedule(date=(start.astimezone(ZoneInfo(self.tz))))
-
-
-class Schedule(BaseSchedule):
-    """Schedule (Schedule) Model.
-
-    See Also:
-        * ``generate()`` is the main usecase of this schedule object.
-    """
-
-
-class ScheduleBkk(Schedule):
-    """Asia Bangkok Schedule (Schedule) timezone Model.
-
-    This model use for change timezone from utc to Asia/Bangkok
-    """
-
-    tz: Annotated[str, Field(description="Timezone")] = "Asia/Bangkok"
-
-
-class AwsSchedule(BaseSchedule):
-    """Implement Schedule for AWS Service."""

ddeutil/workflow/tasks/__init__.py

@@ -1,6 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
-# Licensed under the MIT License. See LICENSE in the project root for
-# license information.
-# ------------------------------------------------------------------------------
-from ._polars import *

ddeutil/workflow/tasks/_pandas.py

@@ -1,54 +0,0 @@
-import logging
-import math
-
-try:
-    import pandas as pd
-
-    logging.debug(f"Pandas version: {pd.__version__}")
-except ImportError as err:
-    raise ImportError(
-        "``split_iterable`` function want to use pandas package that does"
-        "not install on your interpreter."
-    ) from err
-
-
-def split_iterable(iterable, chunk_size=None, generator_flag: bool = True):
-    """
-    Split an iterable into mini batch with batch length of batch_number
-    supports batch of a pandas dataframe
-    usage:
-        >> for i in split_iterable([1,2,3,4,5], chunk_size=2):
-        >>    print(i)
-        [1, 2]
-        [3, 4]
-        [5]
-
-        for idx, mini_data in split_iterable(batch(df, chunk_size=10)):
-            print(idx)
-            print(mini_data)
-    """
-
-    chunk_size: int = chunk_size or 25000
-    num_chunks = math.ceil(len(iterable) / chunk_size)
-    if generator_flag:
-        for _ in range(num_chunks):
-            if isinstance(iterable, pd.DataFrame):
-                yield iterable.iloc[_ * chunk_size : (_ + 1) * chunk_size]
-            else:
-                yield iterable[_ * chunk_size : (_ + 1) * chunk_size]
-    else:
-        _chunks: list = []
-        for _ in range(num_chunks):
-            if isinstance(iterable, pd.DataFrame):
-                _chunks.append(
-                    iterable.iloc[_ * chunk_size : (_ + 1) * chunk_size]
-                )
-            else:
-                _chunks.append(iterable[_ * chunk_size : (_ + 1) * chunk_size])
-        return _chunks
-
-
-def chunks(dataframe: pd.DataFrame, n: int):
-    """Yield successive n-sized chunks from dataframe."""
-    for i in range(0, len(dataframe), n):
-        yield dataframe.iloc[i : i + n]

ddeutil/workflow/tasks/_polars.py

@@ -1,92 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
-# Licensed under the MIT License. See LICENSE in the project root for
-# license information.
-# ------------------------------------------------------------------------------
-from __future__ import annotations
-
-import logging
-from typing import Any
-from uuid import uuid4
-
-try:
-    import polars as pl
-
-    logging.debug(f"Polars version: {pl.__version__}")
-except ImportError:
-    raise ImportError(
-        "Please install polars if you want to use any relate task"
-    ) from None
-import pyarrow.parquet as pq
-from ddeutil.workflow.utils import tag
-from ddeutil.workflow.vendors.pl import PolarsCsv, PolarsParq
-
-
-def polars_dtype():
-    return {
-        "str": pl.Utf8,
-        "int": pl.Int32,
-    }
-
-
-@tag("polars-dir", name="el-csv-to-parquet")
-def csv_to_parquet_dir(
-    source: str,
-    sink: str,
-    conversion: dict[str, Any] | None = None,
-) -> dict[str, int]:
-    """Extract Load data from CSV to Parquet file.
-
-    :param source:
-    :param sink:
-    :param conversion:
-    """
-    print("Start EL for CSV to Parquet with Polars Engine")
-    print("---")
-    # STEP 01: Read the source data to Polars.
-    src_dataset: PolarsCsv = PolarsCsv.from_loader(name=source, externals={})
-    src_df: pl.DataFrame = src_dataset.load()
-    print(src_df)
-
-    # STEP 02: Schema conversion on Polars DataFrame.
-    conversion: dict[str, Any] = conversion or {}
-    if conversion:
-        src_df = src_df.with_columns(
-            *[pl.col(c).cast(col.type).alias(col.name) for c, col in conversion]
-        )
-        print("Start Schema Conversion ...")
-
-    # STEP 03: Write data to parquet file format.
-    sink = PolarsParq.from_loader(name=sink, externals={})
-    pq.write_to_dataset(
-        table=src_df.to_arrow(),
-        root_path=f"{sink.conn.endpoint}/{sink.object}",
-        compression="snappy",
-        basename_template=f"{sink.object}-{uuid4().hex}-{{i}}.snappy.parquet",
-    )
-    return {"records": src_df.select(pl.len()).item()}
-
-
-@tag("polars-dir-scan", name="el-csv-to-parquet")
-def csv_to_parquet_dir_scan(
-    source: str,
-    sink: str,
-    conversion: dict[str, Any] | None = None,
-) -> dict[str, int]:
-    print("Start EL for CSV to Parquet with Polars Engine")
-    print("---")
-    # STEP 01: Read the source data to Polars.
-    src_dataset: PolarsCsv = PolarsCsv.from_loader(name=source, externals={})
-    src_df: pl.LazyFrame = src_dataset.scan()
-
-    if conversion:
-        ...
-
-    sink = PolarsParq.from_loader(name=sink, externals={})
-    pq.write_to_dataset(
-        table=src_df.collect().to_arrow(),
-        root_path=f"{sink.conn.endpoint}/{sink.object}",
-        compression="snappy",
-        basename_template=f"{sink.object}-{uuid4().hex}-{{i}}.snappy.parquet",
-    )
-    return {"records": src_df.select(pl.len()).collect().item()}

ddeutil/workflow/vendors/__dataset.py

@@ -1,127 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
-# Licensed under the MIT License. See LICENSE in the project root for
-# license information.
-# ------------------------------------------------------------------------------
-from __future__ import annotations
-
-from datetime import datetime
-from typing import Annotated, Any, Optional
-
-from fmtutil import Datetime, FormatterGroupType, make_group
-from fmtutil.utils import escape_fmt_group
-from pydantic import BaseModel, Field
-from typing_extensions import Self
-
-from ..__types import DictData, TupleStr
-from ..conn import SubclassConn
-from ..loader import Loader
-
-EXCLUDED_EXTRAS: TupleStr = ("type",)
-OBJ_FMTS: FormatterGroupType = make_group({"datetime": Datetime})
-
-
-class BaseDataset(BaseModel):
-    """Base Dataset Model. This model implement only loading construction."""
-
-    conn: Annotated[SubclassConn, Field(description="Connection Model")]
-    endpoint: Annotated[
-        Optional[str],
-        Field(description="Endpoint of connection"),
-    ] = None
-    object: str = Field(description="Dataset object that want to contract")
-    features: list = Field(default_factory=list)
-    extras: dict[str, Any] = Field(default_factory=dict)
-
-    @classmethod
-    def from_loader(
-        cls,
-        name: str,
-        externals: DictData,
-    ) -> Self:
-        """Construct Connection with Loader object with specific config name.
-
-        :param name: A name of dataset that want to load from config file.
-        :param externals: An external parameters.
-        """
-        loader: Loader = Loader(name, externals=externals)
-
-        # NOTE: Validate the config type match with current dataset model
-        if loader.type != cls:
-            raise ValueError(f"Type {loader.type} does not match with {cls}")
-
-        filter_data: DictData = {
-            k: loader.data.pop(k)
-            for k in loader.data.copy()
-            if k not in cls.model_fields and k not in EXCLUDED_EXTRAS
-        }
-
-        if "conn" not in loader.data:
-            raise ValueError("Dataset config does not set ``conn`` value")
-
-        # NOTE: Start loading connection config
-        conn_name: str = loader.data.pop("conn")
-        conn_loader: Loader = Loader(conn_name, externals=externals)
-        conn_model: SubclassConn = conn_loader.type.from_loader(
-            name=conn_name, externals=externals
-        )
-
-        # NOTE: Override ``endpoint`` value to getter connection data.
-        if "endpoint" in loader.data:
-            # NOTE: Update endpoint path without Pydantic validator.
-            conn_model.__dict__["endpoint"] = loader.data["endpoint"]
-        else:
-            loader.data.update({"endpoint": conn_model.endpoint})
-        return cls.model_validate(
-            obj={
-                "extras": (
-                    loader.data.pop("extras", {}) | filter_data | externals
-                ),
-                "conn": conn_model,
-                **loader.data,
-            }
-        )
-
-
-class Dataset(BaseDataset):
-
-    def exists(self) -> bool:
-        raise NotImplementedError("Object exists does not implement")
-
-    def format_object(
-        self,
-        _object: str | None = None,
-        dt: str | datetime | None = None,
-    ) -> str:
-        """Format the object value that implement datetime"""
-        if dt is None:
-            dt = datetime.now()
-        dt: datetime = (
-            dt if isinstance(dt, datetime) else datetime.fromisoformat(dt)
-        )
-        return (
-            OBJ_FMTS({"datetime": dt})
-            .format(escape_fmt_group(_object or self.object))
-            .replace("\\", "")
-        )
-
-
-class FlDataset(Dataset):
-
-    def exists(self) -> bool:
-        return self.conn.find_object(self.object)
-
-
-class TblDataset(Dataset):
-
-    def exists(self) -> bool:
-        return self.conn.find_object(self.object)
-
-
-class FlDataFrame(Dataset):
-
-    def exists(self) -> bool:
-        return self.conn.find_object(self.object)
-
-
-class TblDataFrame(Dataset): ...