ddeutil-workflow 0.0.1__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.1"

ddeutil/workflow/__regex.py

@@ -0,0 +1,44 @@
+# -------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import re
+from re import (
+    IGNORECASE,
+    MULTILINE,
+    UNICODE,
+    VERBOSE,
+    Pattern,
+)
+
+
+class RegexConf:
+    """Regular expression config."""
+
+    # NOTE: Search caller
+    __re_caller: str = r"""
+        \$
+        {{
+            \s*(?P<caller>
+                [a-zA-Z0-9_.\s'\"\[\]\(\)\-\{}]+?
+            )\s*
+        }}
+    """
+    RE_CALLER: Pattern = re.compile(
+        __re_caller, MULTILINE | IGNORECASE | UNICODE | VERBOSE
+    )
+
+    # NOTE: Search task
+    __re_task_fmt: str = r"""
+        ^
+            (?P<path>[^/@]+)
+            /
+            (?P<func>[^@]+)
+            @
+            (?P<tag>.+)
+        $
+    """
+    RE_TASK_FMT: Pattern = re.compile(
+        __re_task_fmt, MULTILINE | IGNORECASE | UNICODE | VERBOSE
+    )

ddeutil/workflow/__types.py

@@ -0,0 +1,11 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+from typing import Any
+
+TupleStr = tuple[str, ...]
+DictData = dict[str, Any]

ddeutil/workflow/conn.py

@@ -0,0 +1,235 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import logging
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Annotated, Any, Literal, Optional, TypeVar
+
+from ddeutil.model.conn import Conn as ConnModel
+from pydantic import BaseModel, ConfigDict, Field
+from pydantic.functional_validators import field_validator
+from pydantic.types import SecretStr
+from typing_extensions import Self
+
+from .__types import DictData, TupleStr
+from .loader import Loader
+
+EXCLUDED_EXTRAS: TupleStr = (
+    "type",
+    "url",
+)
+
+
+class BaseConn(BaseModel):
+    """Base Conn (Connection) Model"""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    # NOTE: This is fields
+    dialect: str
+    host: Optional[str] = None
+    port: Optional[int] = None
+    user: Optional[str] = None
+    pwd: Optional[SecretStr] = None
+    endpoint: str
+    extras: Annotated[
+        DictData,
+        Field(default_factory=dict, description="Extras mapping of parameters"),
+    ]
+
+    @classmethod
+    def from_loader(
+        cls,
+        name: str,
+        externals: DictData,
+    ) -> Self:
+        """Construct Connection with Loader object with specific config name.
+
+        :param name:
+        :param externals:
+        """
+        loader: Loader = Loader(name, externals=externals)
+        # NOTE: Validate the config type match with current connection model
+        if loader.type != cls:
+            raise ValueError(f"Type {loader.type} does not match with {cls}")
+        filter_data: DictData = {
+            k: loader.data.pop(k)
+            for k in loader.data.copy()
+            if k not in cls.model_fields and k not in EXCLUDED_EXTRAS
+        }
+        if "url" in loader.data:
+            url: ConnModel = ConnModel.from_url(loader.data.pop("url"))
+            return cls(
+                dialect=url.dialect,
+                host=url.host,
+                port=url.port,
+                user=url.user,
+                pwd=url.pwd,
+                # NOTE:
+                #   I will replace None endpoint with memory value for SQLite
+                #   connection string.
+                endpoint=cls.__prepare_slash_from_url(url.endpoint or "memory"),
+                # NOTE: This order will show that externals this the top level.
+                extras=(url.options | filter_data | externals),
+            )
+        return cls.model_validate(
+            obj={
+                "extras": (
+                    loader.data.pop("extras", {}) | filter_data | externals
+                ),
+                **loader.data,
+            }
+        )
+
+    @classmethod
+    def __prepare_slash_from_url(cls, value: str) -> str:
+        if value.startswith("/"):
+            return value[1:]
+        return value
+
+    @field_validator("endpoint")
+    def __prepare_slash(cls, value: str) -> str:
+        if value.startswith("//"):
+            return value[1:]
+        return value
+
+
+class Conn(BaseConn):
+    """Conn (Connection) Model that implement any necessary methods. This object
+    should be the base for abstraction to any connection model object.
+    """
+
+    def get_spec(self) -> str:
+        """Return full connection url that construct from all fields."""
+        return (
+            f"{self.dialect}://{self.user or ''}"
+            f"{f':{self.pwd}' if self.pwd else ''}"
+            f"{self.host or ''}{f':{self.port}' if self.port else ''}"
+            f"/{self.endpoint}"
+        )
+
+    def ping(self) -> bool:
+        """Ping the connection that able to use with this field value."""
+        raise NotImplementedError("Ping does not implement")
+
+    def glob(self, pattern: str) -> Iterator[Any]:
+        """Return a list of object from the endpoint of this connection."""
+        raise NotImplementedError("Glob does not implement")
+
+    def find_object(self, _object: str):
+        raise NotImplementedError("Glob does not implement")
+
+
+class FlSys(Conn):
+    """File System Connection."""
+
+    dialect: Literal["local"] = "local"
+
+    def ping(self) -> bool:
+        return Path(self.endpoint).exists()
+
+    def glob(self, pattern: str) -> Iterator[Path]:
+        yield from Path(self.endpoint).rglob(pattern=pattern)
+
+    def find_object(self, _object: str) -> bool:
+        return (Path(self.endpoint) / _object).exists()
+
+
+class SFTP(Conn):
+    """SFTP Server Connection."""
+
+    dialect: Literal["sftp"] = "sftp"
+
+    def __client(self):
+        from .vendors.sftp_wrapped import WrapSFTP
+
+        return WrapSFTP(
+            host=self.host,
+            port=self.port,
+            user=self.user,
+            pwd=self.pwd.get_secret_value(),
+        )
+
+    def ping(self) -> bool:
+        with self.__client().simple_client():
+            return True
+
+    def glob(self, pattern: str) -> Iterator[str]:
+        yield from self.__client().walk(pattern=pattern)
+
+
+class Db(Conn):
+    """RDBMS System Connection"""
+
+    def ping(self) -> bool:
+        from sqlalchemy import create_engine
+        from sqlalchemy.engine import URL, Engine
+        from sqlalchemy.exc import OperationalError
+
+        engine: Engine = create_engine(
+            url=URL.create(
+                self.dialect,
+                username=self.user,
+                password=self.pwd.get_secret_value() if self.pwd else None,
+                host=self.host,
+                port=self.port,
+                database=self.endpoint,
+                query={},
+            ),
+            execution_options={},
+        )
+        try:
+            return engine.connect()
+        except OperationalError as err:
+            logging.warning(str(err))
+            return False
+
+
+class SQLite(Db):
+    dialect: Literal["sqlite"]
+
+
+class ODBC(Conn): ...
+
+
+class Doc(Conn):
+    """No SQL System Connection"""
+
+
+class Mongo(Doc): ...
+
+
+class SSHCred(BaseModel):
+    ssh_host: str
+    ssh_user: str
+    ssh_password: Optional[SecretStr] = Field(default=None)
+    ssh_private_key: Optional[str] = Field(default=None)
+    ssh_private_key_pwd: Optional[SecretStr] = Field(default=None)
+    ssh_port: int = Field(default=22)
+
+
+class S3Cred(BaseModel):
+    aws_access_key: str
+    aws_secret_access_key: SecretStr
+    region: str = Field(default="ap-southeast-1")
+    role_arn: Optional[str] = Field(default=None)
+    role_name: Optional[str] = Field(default=None)
+    mfa_serial: Optional[str] = Field(default=None)
+
+
+class AZServPrinCred(BaseModel):
+    tenant: str
+    client_id: str
+    secret_id: SecretStr
+
+
+class GoogleCred(BaseModel):
+    google_json_path: str
+
+
+SubclassConn = TypeVar("SubclassConn", bound=Conn)

ddeutil/workflow/dataset.py

@@ -0,0 +1,306 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Annotated, Any, Optional
+
+from fmtutil import Datetime, FormatterGroupType, make_group
+from fmtutil.utils import escape_fmt_group
+from pydantic import BaseModel, Field
+from typing_extensions import Self
+
+try:
+    import polars as pl
+except ImportError:
+    raise ImportError(
+        "Please install polars package\n\t\t$ pip install polars"
+    ) from None
+
+from .__types import DictData, TupleStr
+from .conn import SubclassConn
+from .loader import Loader
+
+EXCLUDED_EXTRAS: TupleStr = ("type",)
+OBJ_FMTS: FormatterGroupType = make_group(
+    {
+        "datetime": Datetime,
+    }
+)
+
+
+class BaseDataset(BaseModel):
+    """Base Dataset Model. This model implement only loading constructor."""
+
+    conn: Annotated[SubclassConn, Field(description="Connection Model")]
+    endpoint: Annotated[
+        Optional[str],
+        Field(description="Endpoint of connection"),
+    ] = None
+    object: str
+    features: list = Field(default_factory=list)
+    extras: dict[str, Any] = Field(default_factory=dict)
+
+    @classmethod
+    def from_loader(
+        cls,
+        name: str,
+        externals: DictData,
+    ) -> Self:
+        """Construct Connection with Loader object with specific config name.
+
+        :param name: A name of dataset that want to load from config file.
+        :param externals: An external parameters.
+        """
+        loader: Loader = Loader(name, externals=externals)
+
+        # NOTE: Validate the config type match with current dataset model
+        if loader.type != cls:
+            raise ValueError(f"Type {loader.type} does not match with {cls}")
+
+        filter_data: DictData = {
+            k: loader.data.pop(k)
+            for k in loader.data.copy()
+            if k not in cls.model_fields and k not in EXCLUDED_EXTRAS
+        }
+
+        if "conn" not in loader.data:
+            raise ValueError("Dataset config does not set ``conn`` value")
+
+        # NOTE: Start loading connection config
+        conn_name: str = loader.data.pop("conn")
+        conn_loader: Loader = Loader(conn_name, externals=externals)
+        conn_model: SubclassConn = conn_loader.type.from_loader(
+            name=conn_name, externals=externals
+        )
+
+        # NOTE: Override ``endpoint`` value to getter connection data.
+        if "endpoint" in loader.data:
+            # NOTE: Update endpoint path without Pydantic validator.
+            conn_model.__dict__["endpoint"] = loader.data["endpoint"]
+        else:
+            loader.data.update({"endpoint": conn_model.endpoint})
+        return cls.model_validate(
+            obj={
+                "extras": (
+                    loader.data.pop("extras", {}) | filter_data | externals
+                ),
+                "conn": conn_model,
+                **loader.data,
+            }
+        )
+
+
+class Dataset(BaseDataset):
+
+    def exists(self) -> bool:
+        raise NotImplementedError("Object exists does not implement")
+
+    def format_object(
+        self,
+        _object: str | None = None,
+        dt: str | datetime | None = None,
+    ) -> str:
+        """Format the object value that implement datetime"""
+        if dt is None:
+            dt = datetime.now()
+        dt: datetime = (
+            dt if isinstance(dt, datetime) else datetime.fromisoformat(dt)
+        )
+        return (
+            OBJ_FMTS({"datetime": dt})
+            .format(escape_fmt_group(_object or self.object))
+            .replace("\\", "")
+        )
+
+
+class FlDataset(Dataset):
+
+    def exists(self) -> bool:
+        return self.conn.find_object(self.object)
+
+
+class TblDataset(Dataset):
+
+    def exists(self) -> bool:
+        return self.conn.find_object(self.object)
+
+
+class FlDataFrame(Dataset):
+
+    def exists(self) -> bool:
+        return self.conn.find_object(self.object)
+
+
+class TblDataFrame(Dataset): ...
+
+
+class PandasCSV: ...
+
+
+class PandasJson: ...
+
+
+class PandasParq: ...
+
+
+class PandasDb: ...
+
+
+class PandasExcel: ...
+
+
+class PolarsCsvArgs(BaseModel):
+    """CSV file should use format rfc4180 as CSV standard format.
+
+    docs: [RFC4180](https://datatracker.ietf.org/doc/html/rfc4180)
+    """
+
+    header: bool = True
+    separator: str = ","
+    skip_rows: int = 0
+    encoding: str = "utf-8"
+
+
+class PolarsCsv(FlDataFrame):
+    extras: PolarsCsvArgs
+
+    def load_options(self) -> dict[str, Any]:
+        return {
+            "has_header": self.extras.header,
+            "separator": self.extras.separator,
+            "skip_rows": self.extras.skip_rows,
+            "encoding": self.extras.encoding,
+        }
+
+    def load(
+        self,
+        _object: str | None = None,
+        options: dict[str, Any] | None = None,
+        *,
+        override: bool = False,
+    ) -> pl.DataFrame:
+        """Load CSV file to Polars DataFrame with ``read_csv`` method."""
+        return pl.read_csv(
+            f"{self.conn.get_spec()}/{_object or self.object}",
+            **(
+                (options or {})
+                if override
+                else (self.load_options() | (options or {}))
+            ),
+        )
+
+    def scan(
+        self,
+        _object: str | None = None,
+        options: dict[str, Any] | None = None,
+    ) -> pl.LazyFrame:
+        """Load CSV file to Polars LazyFrame with ``scan_csv`` method."""
+        # FIXME: Save Csv does not support for the fsspec file url.
+        return pl.scan_csv(
+            f"{self.conn.endpoint}/{_object or self.object}",
+            **(self.load_options() | (options or {})),
+        )
+
+    def save_options(self) -> dict[str, Any]:
+        return {
+            "include_header": self.extras.header,
+            "separator": self.extras.separator,
+        }
+
+    def save(
+        self,
+        df: pl.DataFrame,
+        _object: str | None = None,
+        options: dict[str, Any] | None = None,
+    ) -> None:
+        """Save Polars Dataframe to CSV file with ``write_csv`` method."""
+        # FIXME: Save Csv does not support for the fsspec file url.
+        return df.write_csv(
+            f"{self.conn.endpoint}/{_object or self.object}",
+            **(self.save_options() | (options or {})),
+        )
+
+    def sink(
+        self,
+        df: pl.LazyFrame,
+        _object: str | None = None,
+        options: dict[str, Any] | None = None,
+    ) -> None:
+        """Save Polars Dataframe to CSV file with ``sink_csv`` method."""
+        # FIXME: Save Csv does not support for the fsspec file url.
+        return df.sink_csv(
+            f"{self.conn.endpoint}/{_object or self.object}",
+            **(self.save_options() | (options or {})),
+        )
+
+
+class PolarsJson(FlDataFrame):
+
+    def load(
+        self,
+        _object: str | None = None,
+        options: dict[str, Any] | None = None,
+        *,
+        dt: str | datetime | None = None,
+    ):
+        """Load Json file to Polars Dataframe with ``read_json`` method."""
+        # FIXME: Load Json does not support for the fsspec file url.
+        return pl.read_json(
+            f"{self.conn.endpoint}/"
+            f"{self.format_object(_object or self.object, dt=dt)}",
+            **(options or {}),
+        )
+
+    def save(
+        self,
+        df: pl.DataFrame,
+        _object: str | None = None,
+        options: dict[str, Any] | None = None,
+    ): ...
+
+
+class PolarsNdJson(FlDataFrame): ...
+
+
+class PolarsParqArgs(BaseModel):
+    compression: Optional[str] = None
+    use_pyarrow: bool = False
+    pyarrow_options: dict[str, Any] = Field(default_factory=dict)
+
+
+class PolarsParq(FlDataFrame):
+    extras: PolarsParqArgs
+
+    def save_options(self):
+        excluded: list[str] = []
+        if not self.extras.pyarrow_options:
+            excluded.append("pyarrow_options")
+        return self.extras.model_dump(exclude=excluded)
+
+    def save(
+        self,
+        df: pl.DataFrame,
+        _object: str | None = None,
+        options: dict[str, Any] | None = None,
+    ):
+        print(
+            f"Start write parquet to "
+            f"{self.conn.endpoint}/{_object or self.object}"
+        )
+        return df.write_parquet(
+            f"{self.conn.endpoint}/{_object or self.object}",
+            **(self.save_options() | (options or {})),
+        )
+
+
+class PostgresTbl(TblDataset): ...
+
+
+class SqliteTbl(TblDataset): ...
+
+
+class PolarsPostgres(TblDataFrame): ...

ddeutil/workflow/exceptions.py

@@ -0,0 +1,82 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+"""
+Define Errors Object for Node package
+"""
+from __future__ import annotations
+
+from typing import Union
+
+
+class BaseError(Exception):
+    """Base Error Object that use for catch any errors statement of
+    all step in this src
+    """
+
+
+class WorkflowBaseError(BaseError):
+    """Core Base Error object"""
+
+
+class ConfigNotFound(WorkflowBaseError):
+    """Error raise for a method not found the config file or data."""
+
+
+class ConfigArgumentError(WorkflowBaseError):
+    """Error raise for a wrong configuration argument."""
+
+    def __init__(self, argument: Union[str, tuple], message: str):
+        """Main Initialization that merge the argument and message input values
+        with specific content message together like
+
+            `__class__` with `argument`, `message`
+
+        :param argument: Union[str, tuple]
+        :param message: str
+        """
+        if isinstance(argument, tuple):
+            _last_arg: str = str(argument[-1])
+            _argument: str = (
+                (
+                    ", ".join([f"{_!r}" for _ in argument[:-1]])
+                    + f", and {_last_arg!r}"
+                )
+                if len(argument) > 1
+                else f"{_last_arg!r}"
+            )
+        else:
+            _argument: str = f"{argument!r}"
+        _message: str = f"with {_argument}, {message}"
+        super().__init__(_message)
+
+
+class ConnArgumentError(ConfigArgumentError):
+    """Error raise for wrong connection argument when loading or parsing"""
+
+
+class DsArgumentError(ConfigArgumentError):
+    """Error raise for wrong catalog argument when loading or parsing"""
+
+
+class NodeArgumentError(ConfigArgumentError):
+    """Error raise for wrong node argument when loading or parsing"""
+
+
+class ScdlArgumentError(ConfigArgumentError):
+    """Error raise for wrong schedule argument when loading or parsing"""
+
+
+class PipeArgumentError(ConfigArgumentError):
+    """Error raise for wrong pipeline argument when loading or parsing"""
+
+
+class PyException(Exception): ...
+
+
+class ShellException(Exception): ...
+
+
+class TaskException(Exception): ...

ddeutil/workflow/hooks/__init__.py

@@ -0,0 +1,9 @@
+from typing import Any
+
+from ddeutil.core import lazy
+
+registries: dict[str, Any] = {
+    "postgres-proc": {
+        "pysycopg": lazy("ddeutil.workflow.tasks._postgres.postgres_procedure"),
+    },
+}

ddeutil/workflow/hooks/_postgres.py

@@ -0,0 +1,2 @@
+def postgres_procedure():
+    return