ddeutil-workflow 0.0.3__py3-none-any.whl → 0.0.5__py3-none-any.whl

ddeutil/workflow/vendors/aws.py

@@ -1,185 +0,0 @@
-from typing import Any, Optional
-
-try:
-    import boto3
-    import botocore.exceptions
-except ImportError:
-    raise ImportError(
-        "Please install boto3 package if want to use boto wrapped object.\n\t\t"
-        "$ pip install boto3"
-    ) from None
-
-
-class WrapBoto3:
-    """Difference in boto3 between resource, client, and session
-    docs: https://stackoverflow.com/questions/42809096/
-        difference-in-boto3-between-resource-client-and-session
-
-    .. config::
-
-        ~/.aws/credentials
-
-        [my-user]
-        aws_access_key_id = AKIAxxx
-        aws_secret_access_key = xxx
-
-        [my-role]
-        source_profile = my-user
-        role_arn = arn:aws:iam::123456789012:role/the-role
-
-        ~/.aws/config
-
-        [profile my-role]
-        region = ap-southeast-2
-    """
-
-    def __init__(
-        self,
-        access_key_id: str,
-        secret_access_key: str,
-        region_name: Optional[str] = None,
-        *,
-        role_session_name: Optional[str] = None,
-        role_arn: Optional[str] = None,
-        mfa_serial: Optional[str] = None,
-    ):
-        self.access_key_id = access_key_id
-        self.secret_access_key = secret_access_key
-        self.region_name: str = region_name or "ap-southeast-1"
-
-        # Optional for session.
-        self.role_session_name: str = role_session_name or "AssumeRoleSession"
-        self.role_arn = role_arn
-        self.mfa_serial = mfa_serial
-
-        # Create credential
-        self.cred = self.make_cred()
-
-    def make_cred(self) -> dict[str, str]:
-        if self.role_arn is None:
-            return {
-                "AccessKeyId": self.access_key_id,
-                "SecretAccessKey": self.secret_access_key,
-            }
-        # NOTE: A low-level client representing AWS Security Token Service (STS)
-        # >>> sess = boto3.session.Session(
-        # ...   aws_access_key_id=ARN_ACCESS_KEY,
-        # ...   aws_secret_access_key=ARN_SECRET_KEY
-        # ... )
-        # >>> sts_client = sess.client('sts')
-        sts_client = boto3.client(
-            service_name="sts",
-            region_name=self.region_name,
-            aws_access_key_id=self.access_key_id,
-            aws_secret_access_key=self.secret_access_key,
-        )
-        mfa_optional: dict[str, str] = {}
-        if self.mfa_serial:
-            mfa_otp: str = str(input("Enter the MFA code: "))
-            mfa_optional = (
-                {"SerialNumber": self.mfa_serial, "TokenCode": mfa_otp},
-            )
-        assumed_role = sts_client.assume_role(
-            RoleArn=self.role_arn,
-            RoleSessionName=self.role_session_name,
-            DurationSeconds=3600,
-            **mfa_optional,
-        )
-        # NOTE: From the response that contains the assumed role, get the
-        # temporary credentials that can be used to make subsequent API calls
-        return assumed_role["Credentials"]
-
-    @property
-    def session(self):
-        """Can use by
-        ``s3 = self.session.client('s3')``
-        ``s3 = self.session.resource('s3')``
-        """
-        return boto3.session.Session(
-            aws_access_key_id=self.cred["AccessKeyId"],
-            aws_secret_access_key=self.cred["SecretAccessKey"],
-            aws_session_token=self.cred.get("SessionToken"),
-        )
-
-    @property
-    def s3(self):
-        return boto3.client(
-            service_name="s3",
-            region_name=self.region_name,
-            aws_access_key_id=self.cred["AccessKeyId"],
-            aws_secret_access_key=self.cred["SecretAccessKey"],
-            aws_session_token=self.cred.get("SessionToken"),
-        )
-
-    def list_objects(self, bucket: str, prefix: str):
-        objs: list[dict[str, Any]] = []
-        kwargs = {"Bucket": bucket, "Prefix": prefix}
-        while True:
-            resp = self.s3.list_objects_v2(**kwargs)
-            for obj in resp["Contents"]:
-                objs.append(obj)
-            try:
-                kwargs["ContinuationToken"] = resp["NextContinuationToken"]
-            except KeyError:
-                break
-        return objs
-
-    def paginate(
-        self,
-        bucket: str,
-        prefix: str,
-        *,
-        marker: Optional[str] = None,
-        search: Optional[str] = None,
-    ):
-        """
-        .. docs:
-            - https://boto3.amazonaws.com/v1/documentation/api/latest/
-                guide/paginators.html
-
-        .. search::
-            - "Contents[?Size > `100`][]"
-            - "Contents[?contains(LastModified, `'"2022-01-01"'`)]"
-            - "Contents[?LastModified>=`YYYY-MM-DD`].Key"
-            - "DeleteMarkers[?LastModified>=`2020-07-07T00:00:00`
-                && IsLatest==`true`].[Key,VersionId]"
-        """
-        paginator = self.s3.get_paginator("list_objects_v2")
-        page_iterator = paginator.paginate(
-            Bucket=bucket,
-            Prefix=prefix,
-            PaginationConfig={
-                # 'MaxItems': 10,
-                "PageSize": 10,
-                "StartingToken": marker,
-            },
-        )
-
-        for page in page_iterator:
-            print("# This is new page")
-            print("Contents Count:", len(page["Contents"]))
-            if "NextContinuationToken" in page.keys():
-                print(page["NextContinuationToken"])
-
-        # filtered_iterator = page_iterator.search("Contents[?Size > `100`][]")
-        # for key_data in filtered_iterator:
-        #     print(key_data)
-
-        # page_iterator = paginator.paginate(
-        #     Bucket=bucket,
-        #     Prefix=prefix,
-        #     PaginationConfig={
-        #         'MaxItems': 10,
-        #         'PageSize': 10,
-        #         'StartingToken': marker
-        #     }
-        # )
-
-    def exists(self, bucket: str, prefix: str) -> bool:
-        try:
-            self.s3.head_object(Bucket=bucket, Key=prefix)
-            return True
-        except botocore.exceptions.ClientError as err:
-            if err.response["Error"]["Code"]:
-                return False
-            raise

ddeutil/workflow/vendors/minio.py

@@ -1,11 +0,0 @@
-class WarpMinio:
-
-    def __init__(
-        self,
-        host: str,
-        access_key: str,
-        secret_access_key: str,
-    ):
-        self.host: str = host
-        self.access_key: str = access_key
-        self.secret_access_key: str = secret_access_key

ddeutil/workflow/vendors/pd.py

@@ -1,13 +0,0 @@
-class PandasCSV: ...
-
-
-class PandasJson: ...
-
-
-class PandasParq: ...
-
-
-class PandasDb: ...
-
-
-class PandasExcel: ...

ddeutil/workflow/vendors/pg.py

@@ -1,11 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
-# Licensed under the MIT License. See LICENSE in the project root for
-# license information.
-# ------------------------------------------------------------------------------
-from __future__ import annotations
-
-from .__dataset import TblDataset
-
-
-class PostgresTbl(TblDataset): ...

ddeutil/workflow/vendors/pl.py

@@ -1,172 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
-# Licensed under the MIT License. See LICENSE in the project root for
-# license information.
-# ------------------------------------------------------------------------------
-from __future__ import annotations
-
-from datetime import datetime
-from typing import Any, Optional
-
-from fmtutil import Datetime, FormatterGroupType, make_group
-from pydantic import BaseModel, Field
-
-try:
-    import polars as pl
-except ImportError:
-    raise ImportError(
-        "Please install polars package\n\t\t$ pip install polars"
-    ) from None
-
-from ..__types import TupleStr
-from .__dataset import FlDataFrame, TblDataFrame
-
-EXCLUDED_EXTRAS: TupleStr = ("type",)
-OBJ_FMTS: FormatterGroupType = make_group({"datetime": Datetime})
-
-
-class PolarsCsvArgs(BaseModel):
-    """CSV file should use format rfc4180 as CSV standard format.
-
-    docs: [RFC4180](https://datatracker.ietf.org/doc/html/rfc4180)
-    """
-
-    header: bool = True
-    separator: str = ","
-    skip_rows: int = 0
-    encoding: str = "utf-8"
-
-
-class PolarsCsv(FlDataFrame):
-    extras: PolarsCsvArgs
-
-    def load_options(self) -> dict[str, Any]:
-        return {
-            "has_header": self.extras.header,
-            "separator": self.extras.separator,
-            "skip_rows": self.extras.skip_rows,
-            "encoding": self.extras.encoding,
-        }
-
-    def load(
-        self,
-        _object: str | None = None,
-        options: dict[str, Any] | None = None,
-        *,
-        override: bool = False,
-    ) -> pl.DataFrame:
-        """Load CSV file to Polars DataFrame with ``read_csv`` method."""
-        return pl.read_csv(
-            f"{self.conn.get_spec()}/{_object or self.object}",
-            **(
-                (options or {})
-                if override
-                else (self.load_options() | (options or {}))
-            ),
-        )
-
-    def scan(
-        self,
-        _object: str | None = None,
-        options: dict[str, Any] | None = None,
-    ) -> pl.LazyFrame:
-        """Load CSV file to Polars LazyFrame with ``scan_csv`` method."""
-        # FIXME: Save Csv does not support for the fsspec file url.
-        return pl.scan_csv(
-            f"{self.conn.endpoint}/{_object or self.object}",
-            **(self.load_options() | (options or {})),
-        )
-
-    def save_options(self) -> dict[str, Any]:
-        return {
-            "include_header": self.extras.header,
-            "separator": self.extras.separator,
-        }
-
-    def save(
-        self,
-        df: pl.DataFrame,
-        _object: str | None = None,
-        options: dict[str, Any] | None = None,
-    ) -> None:
-        """Save Polars Dataframe to CSV file with ``write_csv`` method."""
-        # FIXME: Save Csv does not support for the fsspec file url.
-        return df.write_csv(
-            f"{self.conn.endpoint}/{_object or self.object}",
-            **(self.save_options() | (options or {})),
-        )
-
-    def sink(
-        self,
-        df: pl.LazyFrame,
-        _object: str | None = None,
-        options: dict[str, Any] | None = None,
-    ) -> None:
-        """Save Polars Dataframe to CSV file with ``sink_csv`` method."""
-        # FIXME: Save Csv does not support for the fsspec file url.
-        return df.sink_csv(
-            f"{self.conn.endpoint}/{_object or self.object}",
-            **(self.save_options() | (options or {})),
-        )
-
-
-class PolarsJson(FlDataFrame):
-
-    def load(
-        self,
-        _object: str | None = None,
-        options: dict[str, Any] | None = None,
-        *,
-        dt: str | datetime | None = None,
-    ):
-        """Load Json file to Polars Dataframe with ``read_json`` method."""
-        # FIXME: Load Json does not support for the fsspec file url.
-        return pl.read_json(
-            f"{self.conn.endpoint}/"
-            f"{self.format_object(_object or self.object, dt=dt)}",
-            **(options or {}),
-        )
-
-    def save(
-        self,
-        df: pl.DataFrame,
-        _object: str | None = None,
-        options: dict[str, Any] | None = None,
-    ): ...
-
-
-class PolarsNdJson(FlDataFrame): ...
-
-
-class PolarsParqArgs(BaseModel):
-    compression: Optional[str] = None
-    use_pyarrow: bool = False
-    pyarrow_options: dict[str, Any] = Field(default_factory=dict)
-
-
-class PolarsParq(FlDataFrame):
-    extras: PolarsParqArgs
-
-    def save_options(self):
-        excluded: list[str] = []
-        if not self.extras.pyarrow_options:
-            excluded.append("pyarrow_options")
-        return self.extras.model_dump(exclude=excluded)
-
-    def save(
-        self,
-        df: pl.DataFrame,
-        _object: str | None = None,
-        options: dict[str, Any] | None = None,
-    ):
-        print(
-            f"Start write parquet to "
-            f"{self.conn.endpoint}/{_object or self.object}"
-        )
-        return df.write_parquet(
-            f"{self.conn.endpoint}/{_object or self.object}",
-            **(self.save_options() | (options or {})),
-        )
-
-
-class PolarsPostgres(TblDataFrame): ...

ddeutil/workflow/vendors/sftp.py

@@ -1,209 +0,0 @@
-import contextlib
-from collections import deque
-from collections.abc import Generator, Iterator
-from ftplib import FTP
-from stat import S_ISDIR, S_ISREG
-from typing import Optional
-
-try:
-    import paramiko
-    from paramiko import SFTPAttributes, SFTPClient
-    from sshtunnel import BaseSSHTunnelForwarderError, SSHTunnelForwarder
-except ImportError:
-    raise ImportError(
-        "Please install paramiko and sshtunnel packages before using,\n\t\t"
-        "$ pip install paramiko sshtunnel"
-    ) from None
-
-
-class WrapFTP:
-    """Wrapped FTP Client.
-
-        FTP (File Transfer Protocol) is the standard for transferring files
-    between the Client and Server is one of the most popular formats.
-
-        But there is a disadvantage that the data sent and received is not
-    encrypted. There is an opportunity for a third party to read the information
-    along the transferring.
-    """
-
-    def __init__(
-        self,
-        host: str,
-        user: str,
-        pwd: str,
-        port: int = 21,
-    ):
-        self.host: str = host
-        self.port: int = port
-        self.user: str = user
-        self.pwd: str = pwd
-
-    def fpt_connect(self):
-        return FTP(
-            host=self.host,
-            user=self.user,
-            passwd=self.pwd,
-        )
-
-
-class WrapSFTP:
-    """Wrapped SFTP Client.
-
-        SFTP (Secure File Transfer Protocol) it is a standard that helps
-    increase security in file transfers. By encrypting data and commands before
-    transferring files between the client and server with SSH (Secure Shell),
-    we can be confident that the data we upload/download can be done safely.
-
-        It cannot be accessed by third parties or if the information is
-    obtained, it is encrypted and cannot be read.
-
-    See-Also:
-
-        This object will wrap the [Paramiko](https://www.paramiko.org/) package
-    with my connection interface.
-    """
-
-    def __init__(
-        self,
-        host: str,
-        user: Optional[str] = None,
-        port: Optional[int] = None,
-        *,
-        pwd: Optional[str] = None,
-        private_key: Optional[str] = None,
-        private_key_password: Optional[str] = None,
-    ) -> None:
-        self.host: str = host
-        self.user: str = user or ""
-        self.port: int = port or 22
-        self.pwd: Optional[str] = pwd
-
-        # Private key path like, ``/home/user/.ssh/id_rsa``.
-        self.private_key = private_key
-
-        # If this private key have password, private_key passphrase.
-        self.private_key_pwd = private_key_password
-
-    def get(self, remote_path, local_path):
-        with self.transport_client() as sftp:
-            sftp.get(remote_path, local_path)
-
-    def put(self, remote_path, local_path):
-        with self.transport_client() as sftp:
-            sftp.put(remote_path, local_path)
-
-    def rm(self, remote_path: str):
-        with self.transport_client() as sftp:
-            sftp.remove(remote_path)
-
-    def mkdir(self, remote_path: str):
-        with self.transport_client() as sftp:
-            sftp.mkdir(remote_path)
-
-    @contextlib.contextmanager
-    def ssh_tunnel(self) -> Iterator:
-        try:
-            with SSHTunnelForwarder(
-                (self.host, self.port),
-                ssh_username=self.user,
-                ssh_password=self.pwd,
-                ssh_pkey=self.private_key,
-                ssh_private_key_password=self.private_key_pwd,
-                local_bind_address=("0.0.0.0", 22),
-                # Use a suitable remote_bind_address that able to be DB host on
-                # that SSH Server.
-                remote_bind_address=("127.0.0.1", self.port),
-            ) as tunnel:
-                tunnel.check_tunnels()
-                client = paramiko.SSHClient()
-                if self.private_key:
-                    client.load_system_host_keys()
-                # NOTE: Add SSH key to known_hosts file.
-                client.set_missing_host_key_policy(
-                    paramiko.MissingHostKeyPolicy()
-                )
-
-                # NOTE: Start connect to SSH Server
-                client.connect(
-                    "127.0.0.1",
-                    port=tunnel.local_bind_port,
-                    **(
-                        {
-                            "username": self.user,
-                            "password": self.pwd,
-                            "allow_agent": False,
-                            "look_for_keys": False,
-                            "banner_timeout": 20,
-                        }
-                        if self.pwd
-                        else {}
-                    ),
-                )
-                with client.open_sftp() as sftp:
-                    yield sftp
-                client.close()
-        except BaseSSHTunnelForwarderError as err:
-            raise ValueError(
-                "This config data does not connect to the Server"
-            ) from err
-
-    @contextlib.contextmanager
-    def transport_client(self) -> Generator[SFTPClient, None, None]:
-        with paramiko.Transport(sock=(self.host, self.port)) as transport:
-            transport.connect(
-                hostkey=None,
-                username=self.user,
-                password=self.pwd,
-            )
-            with paramiko.SFTPClient.from_transport(transport) as sftp:
-                yield sftp
-
-    @contextlib.contextmanager
-    def simple_client(self) -> Generator[SFTPClient, None, None]:
-        client = paramiko.SSHClient()
-        client.set_missing_host_key_policy(paramiko.MissingHostKeyPolicy())
-        client.connect(
-            hostname=self.host,
-            port=self.port,
-            username=self.user,
-            password=self.pwd,
-        )
-        with client.open_sftp() as sftp:
-            yield sftp
-        client.close()
-
-    def glob(self, pattern: str) -> Iterator[str]:
-        with self.transport_client() as sftp:
-            try:
-                # NOTE: List files matching the pattern on the SFTP server
-                f: SFTPAttributes
-                for f in sftp.listdir_attr(pattern):
-                    yield pattern + f.filename
-            except FileNotFoundError:
-                raise FileNotFoundError(
-                    f"Pattern {pattern!r} does not found on SFTP server"
-                ) from None
-
-    def walk(self, pattern: str) -> Iterator[str]:
-        dirs: deque = deque([pattern])
-        with self.transport_client() as sftp:
-            while len(dirs) > 0:
-                d: str = dirs.popleft()
-                f: SFTPAttributes
-                for f in sftp.listdir_attr(d):
-                    rs: str = (
-                        (d + f.filename) if d == "/" else (d + "/" + f.filename)
-                    )
-                    if S_ISDIR(f.st_mode):
-                        dirs.append(rs)
-                    elif S_ISREG(f.st_mode):
-                        yield rs
-
-    @staticmethod
-    def isdir(path: SFTPAttributes):
-        try:
-            return S_ISDIR(path.st_mode)
-        except OSError:
-            # NOTE: Path does not exist, so by definition not a directory
-            return False

ddeutil_workflow-0.0.3.dist-info/RECORD

@@ -1,29 +0,0 @@
-ddeutil/workflow/__about__.py,sha256=smA9c0CTLewINRoxj2VBHoiYDESoFGtXYFDvRT31dgs,27
-ddeutil/workflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ddeutil/workflow/__regex.py,sha256=bOngaQ0zJgy3vfNwF2MlI8XhLu_Ei1Vz8y50iLj8ao4,1061
-ddeutil/workflow/__types.py,sha256=AkpQq6QlrclpurCZZVY9RMxoyS9z2WGzhaz_ikeTaCU,453
-ddeutil/workflow/conn.py,sha256=POtNcyqFNGxZnkg5J_H1OIvQVnnqG-ajmBBzjoHl9sg,7238
-ddeutil/workflow/exceptions.py,sha256=XAq82VHSMLNb4UjGatp7hYfjxFtMiKFtBqJyAhwTl-s,434
-ddeutil/workflow/loader.py,sha256=TXS4k2dqNycBYSTYcJ80WIsPMKNZbHNeBbcufX6lrJc,5483
-ddeutil/workflow/pipeline.py,sha256=fG6ta-SNx4OWS6n8w7YpYDadfnbqayj8A1uY03TvLUA,16942
-ddeutil/workflow/schedule.py,sha256=RMbTC7L32D3fJ5gYxJDCn-vPr2RYEBMSD0G2kj1Qows,2712
-ddeutil/workflow/utils.py,sha256=z7evB9kOsgTr30uVuL994bmOMDNZB5xDY2KjO7gL1dc,5379
-ddeutil/workflow/tasks/__init__.py,sha256=TIcw9JinrdepWgyazSMLk_QflUFms99ILI4GvLHUGD0,338
-ddeutil/workflow/tasks/_pandas.py,sha256=rqz5_VMSqkEdirk7i3EElZoqnRYFyyK_Z8_Zt8FyeTg,1693
-ddeutil/workflow/tasks/_polars.py,sha256=SYEBx-0I9tbY046QGSMokVugK8Fqjhiw4dzpL6y6Hww,2917
-ddeutil/workflow/vendors/__dataset.py,sha256=n9EwTIoVlgCKyCQQgQrijeoQgauOoKtdKiWCzSIErns,4065
-ddeutil/workflow/vendors/__dict.py,sha256=ETwkeA0qzKNgedfeRgAz1qShNXTIXIS4DXzJB4lM4jo,9962
-ddeutil/workflow/vendors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ddeutil/workflow/vendors/__schedule.py,sha256=cUIwtTli9G0klrNFxAIbG3VTiv6umRuNAZiKA-kOtpE,20690
-ddeutil/workflow/vendors/aws.py,sha256=zjq_LCu3ffVBRrxS2vqss9X24yrtuAEt9ouy2_WvS0o,5980
-ddeutil/workflow/vendors/az.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ddeutil/workflow/vendors/minio.py,sha256=pScLy38Du9moOrGaSBSFsoQRhiQ686FQyloOeLA0OQk,261
-ddeutil/workflow/vendors/pd.py,sha256=J6Nkb4RqUnz3NMfo3cHX-Udw3HPjqjUimojS86rR4os,116
-ddeutil/workflow/vendors/pg.py,sha256=TGwkV6nsarGLbiRTT_wB4uAy3xCR89EPPCMWqlWhFe8,422
-ddeutil/workflow/vendors/pl.py,sha256=B-l9zcZ9vATAKVMLv5tjKiWo5Qt8ZIv_aQzuVFinKbY,5087
-ddeutil/workflow/vendors/sftp.py,sha256=lQn4mnHhgvE9g1pbpoQF7HvZOxab8Z2XaDtSIJvumGM,7090
-ddeutil_workflow-0.0.3.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.3.dist-info/METADATA,sha256=KcsTd-FjufMK-4fhiIq27yeQUuA7NeB8TCkbXADQ1Dc,7992
-ddeutil_workflow-0.0.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-ddeutil_workflow-0.0.3.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.3.dist-info/RECORD,,