ddeutil-workflow 0.0.4__py3-none-any.whl → 0.0.6__py3-none-any.whl

ddeutil/workflow/vendors/pl.py

@@ -1,172 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
-# Licensed under the MIT License. See LICENSE in the project root for
-# license information.
-# ------------------------------------------------------------------------------
-from __future__ import annotations
-
-from datetime import datetime
-from typing import Any, Optional
-
-from fmtutil import Datetime, FormatterGroupType, make_group
-from pydantic import BaseModel, Field
-
-try:
-    import polars as pl
-except ImportError:
-    raise ImportError(
-        "Please install polars package\n\t\t$ pip install polars"
-    ) from None
-
-from ..__types import TupleStr
-from .__dataset import FlDataFrame, TblDataFrame
-
-EXCLUDED_EXTRAS: TupleStr = ("type",)
-OBJ_FMTS: FormatterGroupType = make_group({"datetime": Datetime})
-
-
-class PolarsCsvArgs(BaseModel):
-    """CSV file should use format rfc4180 as CSV standard format.
-
-    docs: [RFC4180](https://datatracker.ietf.org/doc/html/rfc4180)
-    """
-
-    header: bool = True
-    separator: str = ","
-    skip_rows: int = 0
-    encoding: str = "utf-8"
-
-
-class PolarsCsv(FlDataFrame):
-    extras: PolarsCsvArgs
-
-    def load_options(self) -> dict[str, Any]:
-        return {
-            "has_header": self.extras.header,
-            "separator": self.extras.separator,
-            "skip_rows": self.extras.skip_rows,
-            "encoding": self.extras.encoding,
-        }
-
-    def load(
-        self,
-        _object: str | None = None,
-        options: dict[str, Any] | None = None,
-        *,
-        override: bool = False,
-    ) -> pl.DataFrame:
-        """Load CSV file to Polars DataFrame with ``read_csv`` method."""
-        return pl.read_csv(
-            f"{self.conn.get_spec()}/{_object or self.object}",
-            **(
-                (options or {})
-                if override
-                else (self.load_options() | (options or {}))
-            ),
-        )
-
-    def scan(
-        self,
-        _object: str | None = None,
-        options: dict[str, Any] | None = None,
-    ) -> pl.LazyFrame:
-        """Load CSV file to Polars LazyFrame with ``scan_csv`` method."""
-        # FIXME: Save Csv does not support for the fsspec file url.
-        return pl.scan_csv(
-            f"{self.conn.endpoint}/{_object or self.object}",
-            **(self.load_options() | (options or {})),
-        )
-
-    def save_options(self) -> dict[str, Any]:
-        return {
-            "include_header": self.extras.header,
-            "separator": self.extras.separator,
-        }
-
-    def save(
-        self,
-        df: pl.DataFrame,
-        _object: str | None = None,
-        options: dict[str, Any] | None = None,
-    ) -> None:
-        """Save Polars Dataframe to CSV file with ``write_csv`` method."""
-        # FIXME: Save Csv does not support for the fsspec file url.
-        return df.write_csv(
-            f"{self.conn.endpoint}/{_object or self.object}",
-            **(self.save_options() | (options or {})),
-        )
-
-    def sink(
-        self,
-        df: pl.LazyFrame,
-        _object: str | None = None,
-        options: dict[str, Any] | None = None,
-    ) -> None:
-        """Save Polars Dataframe to CSV file with ``sink_csv`` method."""
-        # FIXME: Save Csv does not support for the fsspec file url.
-        return df.sink_csv(
-            f"{self.conn.endpoint}/{_object or self.object}",
-            **(self.save_options() | (options or {})),
-        )
-
-
-class PolarsJson(FlDataFrame):
-
-    def load(
-        self,
-        _object: str | None = None,
-        options: dict[str, Any] | None = None,
-        *,
-        dt: str | datetime | None = None,
-    ):
-        """Load Json file to Polars Dataframe with ``read_json`` method."""
-        # FIXME: Load Json does not support for the fsspec file url.
-        return pl.read_json(
-            f"{self.conn.endpoint}/"
-            f"{self.format_object(_object or self.object, dt=dt)}",
-            **(options or {}),
-        )
-
-    def save(
-        self,
-        df: pl.DataFrame,
-        _object: str | None = None,
-        options: dict[str, Any] | None = None,
-    ): ...
-
-
-class PolarsNdJson(FlDataFrame): ...
-
-
-class PolarsParqArgs(BaseModel):
-    compression: Optional[str] = None
-    use_pyarrow: bool = False
-    pyarrow_options: dict[str, Any] = Field(default_factory=dict)
-
-
-class PolarsParq(FlDataFrame):
-    extras: PolarsParqArgs
-
-    def save_options(self):
-        excluded: list[str] = []
-        if not self.extras.pyarrow_options:
-            excluded.append("pyarrow_options")
-        return self.extras.model_dump(exclude=excluded)
-
-    def save(
-        self,
-        df: pl.DataFrame,
-        _object: str | None = None,
-        options: dict[str, Any] | None = None,
-    ):
-        print(
-            f"Start write parquet to "
-            f"{self.conn.endpoint}/{_object or self.object}"
-        )
-        return df.write_parquet(
-            f"{self.conn.endpoint}/{_object or self.object}",
-            **(self.save_options() | (options or {})),
-        )
-
-
-class PolarsPostgres(TblDataFrame): ...

ddeutil/workflow/vendors/sftp.py

@@ -1,209 +0,0 @@
-import contextlib
-from collections import deque
-from collections.abc import Generator, Iterator
-from ftplib import FTP
-from stat import S_ISDIR, S_ISREG
-from typing import Optional
-
-try:
-    import paramiko
-    from paramiko import SFTPAttributes, SFTPClient
-    from sshtunnel import BaseSSHTunnelForwarderError, SSHTunnelForwarder
-except ImportError:
-    raise ImportError(
-        "Please install paramiko and sshtunnel packages before using,\n\t\t"
-        "$ pip install paramiko sshtunnel"
-    ) from None
-
-
-class WrapFTP:
-    """Wrapped FTP Client.
-
-        FTP (File Transfer Protocol) is the standard for transferring files
-    between the Client and Server is one of the most popular formats.
-
-        But there is a disadvantage that the data sent and received is not
-    encrypted. There is an opportunity for a third party to read the information
-    along the transferring.
-    """
-
-    def __init__(
-        self,
-        host: str,
-        user: str,
-        pwd: str,
-        port: int = 21,
-    ):
-        self.host: str = host
-        self.port: int = port
-        self.user: str = user
-        self.pwd: str = pwd
-
-    def fpt_connect(self):
-        return FTP(
-            host=self.host,
-            user=self.user,
-            passwd=self.pwd,
-        )
-
-
-class WrapSFTP:
-    """Wrapped SFTP Client.
-
-        SFTP (Secure File Transfer Protocol) it is a standard that helps
-    increase security in file transfers. By encrypting data and commands before
-    transferring files between the client and server with SSH (Secure Shell),
-    we can be confident that the data we upload/download can be done safely.
-
-        It cannot be accessed by third parties or if the information is
-    obtained, it is encrypted and cannot be read.
-
-    See-Also:
-
-        This object will wrap the [Paramiko](https://www.paramiko.org/) package
-    with my connection interface.
-    """
-
-    def __init__(
-        self,
-        host: str,
-        user: Optional[str] = None,
-        port: Optional[int] = None,
-        *,
-        pwd: Optional[str] = None,
-        private_key: Optional[str] = None,
-        private_key_password: Optional[str] = None,
-    ) -> None:
-        self.host: str = host
-        self.user: str = user or ""
-        self.port: int = port or 22
-        self.pwd: Optional[str] = pwd
-
-        # Private key path like, ``/home/user/.ssh/id_rsa``.
-        self.private_key = private_key
-
-        # If this private key have password, private_key passphrase.
-        self.private_key_pwd = private_key_password
-
-    def get(self, remote_path, local_path):
-        with self.transport_client() as sftp:
-            sftp.get(remote_path, local_path)
-
-    def put(self, remote_path, local_path):
-        with self.transport_client() as sftp:
-            sftp.put(remote_path, local_path)
-
-    def rm(self, remote_path: str):
-        with self.transport_client() as sftp:
-            sftp.remove(remote_path)
-
-    def mkdir(self, remote_path: str):
-        with self.transport_client() as sftp:
-            sftp.mkdir(remote_path)
-
-    @contextlib.contextmanager
-    def ssh_tunnel(self) -> Iterator:
-        try:
-            with SSHTunnelForwarder(
-                (self.host, self.port),
-                ssh_username=self.user,
-                ssh_password=self.pwd,
-                ssh_pkey=self.private_key,
-                ssh_private_key_password=self.private_key_pwd,
-                local_bind_address=("0.0.0.0", 22),
-                # Use a suitable remote_bind_address that able to be DB host on
-                # that SSH Server.
-                remote_bind_address=("127.0.0.1", self.port),
-            ) as tunnel:
-                tunnel.check_tunnels()
-                client = paramiko.SSHClient()
-                if self.private_key:
-                    client.load_system_host_keys()
-                # NOTE: Add SSH key to known_hosts file.
-                client.set_missing_host_key_policy(
-                    paramiko.MissingHostKeyPolicy()
-                )
-
-                # NOTE: Start connect to SSH Server
-                client.connect(
-                    "127.0.0.1",
-                    port=tunnel.local_bind_port,
-                    **(
-                        {
-                            "username": self.user,
-                            "password": self.pwd,
-                            "allow_agent": False,
-                            "look_for_keys": False,
-                            "banner_timeout": 20,
-                        }
-                        if self.pwd
-                        else {}
-                    ),
-                )
-                with client.open_sftp() as sftp:
-                    yield sftp
-                client.close()
-        except BaseSSHTunnelForwarderError as err:
-            raise ValueError(
-                "This config data does not connect to the Server"
-            ) from err
-
-    @contextlib.contextmanager
-    def transport_client(self) -> Generator[SFTPClient, None, None]:
-        with paramiko.Transport(sock=(self.host, self.port)) as transport:
-            transport.connect(
-                hostkey=None,
-                username=self.user,
-                password=self.pwd,
-            )
-            with paramiko.SFTPClient.from_transport(transport) as sftp:
-                yield sftp
-
-    @contextlib.contextmanager
-    def simple_client(self) -> Generator[SFTPClient, None, None]:
-        client = paramiko.SSHClient()
-        client.set_missing_host_key_policy(paramiko.MissingHostKeyPolicy())
-        client.connect(
-            hostname=self.host,
-            port=self.port,
-            username=self.user,
-            password=self.pwd,
-        )
-        with client.open_sftp() as sftp:
-            yield sftp
-        client.close()
-
-    def glob(self, pattern: str) -> Iterator[str]:
-        with self.transport_client() as sftp:
-            try:
-                # NOTE: List files matching the pattern on the SFTP server
-                f: SFTPAttributes
-                for f in sftp.listdir_attr(pattern):
-                    yield pattern + f.filename
-            except FileNotFoundError:
-                raise FileNotFoundError(
-                    f"Pattern {pattern!r} does not found on SFTP server"
-                ) from None
-
-    def walk(self, pattern: str) -> Iterator[str]:
-        dirs: deque = deque([pattern])
-        with self.transport_client() as sftp:
-            while len(dirs) > 0:
-                d: str = dirs.popleft()
-                f: SFTPAttributes
-                for f in sftp.listdir_attr(d):
-                    rs: str = (
-                        (d + f.filename) if d == "/" else (d + "/" + f.filename)
-                    )
-                    if S_ISDIR(f.st_mode):
-                        dirs.append(rs)
-                    elif S_ISREG(f.st_mode):
-                        yield rs
-
-    @staticmethod
-    def isdir(path: SFTPAttributes):
-        try:
-            return S_ISDIR(path.st_mode)
-        except OSError:
-            # NOTE: Path does not exist, so by definition not a directory
-            return False

ddeutil_workflow-0.0.4.dist-info/RECORD

@@ -1,29 +0,0 @@
-ddeutil/workflow/__about__.py,sha256=mI2ex1MyAxyhuxFGZTcowuzhaiztnKZbPh3obkmKpsY,27
-ddeutil/workflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ddeutil/workflow/__regex.py,sha256=bOngaQ0zJgy3vfNwF2MlI8XhLu_Ei1Vz8y50iLj8ao4,1061
-ddeutil/workflow/__types.py,sha256=AkpQq6QlrclpurCZZVY9RMxoyS9z2WGzhaz_ikeTaCU,453
-ddeutil/workflow/conn.py,sha256=POtNcyqFNGxZnkg5J_H1OIvQVnnqG-ajmBBzjoHl9sg,7238
-ddeutil/workflow/exceptions.py,sha256=XAq82VHSMLNb4UjGatp7hYfjxFtMiKFtBqJyAhwTl-s,434
-ddeutil/workflow/loader.py,sha256=TXS4k2dqNycBYSTYcJ80WIsPMKNZbHNeBbcufX6lrJc,5483
-ddeutil/workflow/pipeline.py,sha256=Uo26aC-IEdhSao0bxV_dy03fmv72rX9xENxiyzIEfC0,17789
-ddeutil/workflow/schedule.py,sha256=RMbTC7L32D3fJ5gYxJDCn-vPr2RYEBMSD0G2kj1Qows,2712
-ddeutil/workflow/utils.py,sha256=YWZA5npxzsx89O6bQ1-nxq3k_lbx4SDJURXkXwkmsmc,5556
-ddeutil/workflow/tasks/__init__.py,sha256=TIcw9JinrdepWgyazSMLk_QflUFms99ILI4GvLHUGD0,338
-ddeutil/workflow/tasks/_pandas.py,sha256=rqz5_VMSqkEdirk7i3EElZoqnRYFyyK_Z8_Zt8FyeTg,1693
-ddeutil/workflow/tasks/_polars.py,sha256=SYEBx-0I9tbY046QGSMokVugK8Fqjhiw4dzpL6y6Hww,2917
-ddeutil/workflow/vendors/__dataset.py,sha256=n9EwTIoVlgCKyCQQgQrijeoQgauOoKtdKiWCzSIErns,4065
-ddeutil/workflow/vendors/__dict.py,sha256=ETwkeA0qzKNgedfeRgAz1qShNXTIXIS4DXzJB4lM4jo,9962
-ddeutil/workflow/vendors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ddeutil/workflow/vendors/__schedule.py,sha256=cUIwtTli9G0klrNFxAIbG3VTiv6umRuNAZiKA-kOtpE,20690
-ddeutil/workflow/vendors/aws.py,sha256=zjq_LCu3ffVBRrxS2vqss9X24yrtuAEt9ouy2_WvS0o,5980
-ddeutil/workflow/vendors/az.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ddeutil/workflow/vendors/minio.py,sha256=pScLy38Du9moOrGaSBSFsoQRhiQ686FQyloOeLA0OQk,261
-ddeutil/workflow/vendors/pd.py,sha256=J6Nkb4RqUnz3NMfo3cHX-Udw3HPjqjUimojS86rR4os,116
-ddeutil/workflow/vendors/pg.py,sha256=TGwkV6nsarGLbiRTT_wB4uAy3xCR89EPPCMWqlWhFe8,422
-ddeutil/workflow/vendors/pl.py,sha256=B-l9zcZ9vATAKVMLv5tjKiWo5Qt8ZIv_aQzuVFinKbY,5087
-ddeutil/workflow/vendors/sftp.py,sha256=lQn4mnHhgvE9g1pbpoQF7HvZOxab8Z2XaDtSIJvumGM,7090
-ddeutil_workflow-0.0.4.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.4.dist-info/METADATA,sha256=-Sx33xY1RAQQ5OOSiOIIK6Qw8fcrAq-jiINbaXRFvjo,7992
-ddeutil_workflow-0.0.4.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-ddeutil_workflow-0.0.4.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.4.dist-info/RECORD,,