ddeutil-workflow 0.0.1__py3-none-any.whl

ddeutil/workflow/vendors/aws_warpped.py

@@ -0,0 +1,185 @@
+from typing import Any, Optional
+
+try:
+    import boto3
+    import botocore.exceptions
+except ImportError:
+    raise ImportError(
+        "Please install boto3 package if want to use boto wrapped object.\n\t\t"
+        "$ pip install boto3"
+    ) from None
+
+
+class WrapBoto3:
+    """Difference in boto3 between resource, client, and session
+    docs: https://stackoverflow.com/questions/42809096/
+        difference-in-boto3-between-resource-client-and-session
+
+    .. config::
+
+        ~/.aws/credentials
+
+        [my-user]
+        aws_access_key_id = AKIAxxx
+        aws_secret_access_key = xxx
+
+        [my-role]
+        source_profile = my-user
+        role_arn = arn:aws:iam::123456789012:role/the-role
+
+        ~/.aws/config
+
+        [profile my-role]
+        region = ap-southeast-2
+    """
+
+    def __init__(
+        self,
+        access_key_id: str,
+        secret_access_key: str,
+        region_name: Optional[str] = None,
+        *,
+        role_session_name: Optional[str] = None,
+        role_arn: Optional[str] = None,
+        mfa_serial: Optional[str] = None,
+    ):
+        self.access_key_id = access_key_id
+        self.secret_access_key = secret_access_key
+        self.region_name: str = region_name or "ap-southeast-1"
+
+        # Optional for session.
+        self.role_session_name: str = role_session_name or "AssumeRoleSession"
+        self.role_arn = role_arn
+        self.mfa_serial = mfa_serial
+
+        # Create credential
+        self.cred = self.make_cred()
+
+    def make_cred(self) -> dict[str, str]:
+        if self.role_arn is None:
+            return {
+                "AccessKeyId": self.access_key_id,
+                "SecretAccessKey": self.secret_access_key,
+            }
+        # NOTE: A low-level client representing AWS Security Token Service (STS)
+        # >>> sess = boto3.session.Session(
+        # ...   aws_access_key_id=ARN_ACCESS_KEY,
+        # ...   aws_secret_access_key=ARN_SECRET_KEY
+        # ... )
+        # >>> sts_client = sess.client('sts')
+        sts_client = boto3.client(
+            service_name="sts",
+            region_name=self.region_name,
+            aws_access_key_id=self.access_key_id,
+            aws_secret_access_key=self.secret_access_key,
+        )
+        mfa_optional: dict[str, str] = {}
+        if self.mfa_serial:
+            mfa_otp: str = str(input("Enter the MFA code: "))
+            mfa_optional = (
+                {"SerialNumber": self.mfa_serial, "TokenCode": mfa_otp},
+            )
+        assumed_role = sts_client.assume_role(
+            RoleArn=self.role_arn,
+            RoleSessionName=self.role_session_name,
+            DurationSeconds=3600,
+            **mfa_optional,
+        )
+        # NOTE: From the response that contains the assumed role, get the
+        # temporary credentials that can be used to make subsequent API calls
+        return assumed_role["Credentials"]
+
+    @property
+    def session(self):
+        """Can use by
+        ``s3 = self.session.client('s3')``
+        ``s3 = self.session.resource('s3')``
+        """
+        return boto3.session.Session(
+            aws_access_key_id=self.cred["AccessKeyId"],
+            aws_secret_access_key=self.cred["SecretAccessKey"],
+            aws_session_token=self.cred.get("SessionToken"),
+        )
+
+    @property
+    def s3(self):
+        return boto3.client(
+            service_name="s3",
+            region_name=self.region_name,
+            aws_access_key_id=self.cred["AccessKeyId"],
+            aws_secret_access_key=self.cred["SecretAccessKey"],
+            aws_session_token=self.cred.get("SessionToken"),
+        )
+
+    def list_objects(self, bucket: str, prefix: str):
+        objs: list[dict[str, Any]] = []
+        kwargs = {"Bucket": bucket, "Prefix": prefix}
+        while True:
+            resp = self.s3.list_objects_v2(**kwargs)
+            for obj in resp["Contents"]:
+                objs.append(obj)
+            try:
+                kwargs["ContinuationToken"] = resp["NextContinuationToken"]
+            except KeyError:
+                break
+        return objs
+
+    def paginate(
+        self,
+        bucket: str,
+        prefix: str,
+        *,
+        marker: Optional[str] = None,
+        search: Optional[str] = None,
+    ):
+        """
+        .. docs:
+            - https://boto3.amazonaws.com/v1/documentation/api/latest/
+                guide/paginators.html
+
+        .. search::
+            - "Contents[?Size > `100`][]"
+            - "Contents[?contains(LastModified, `'"2022-01-01"'`)]"
+            - "Contents[?LastModified>=`YYYY-MM-DD`].Key"
+            - "DeleteMarkers[?LastModified>=`2020-07-07T00:00:00`
+                && IsLatest==`true`].[Key,VersionId]"
+        """
+        paginator = self.s3.get_paginator("list_objects_v2")
+        page_iterator = paginator.paginate(
+            Bucket=bucket,
+            Prefix=prefix,
+            PaginationConfig={
+                # 'MaxItems': 10,
+                "PageSize": 10,
+                "StartingToken": marker,
+            },
+        )
+
+        for page in page_iterator:
+            print("# This is new page")
+            print("Contents Count:", len(page["Contents"]))
+            if "NextContinuationToken" in page.keys():
+                print(page["NextContinuationToken"])
+
+        # filtered_iterator = page_iterator.search("Contents[?Size > `100`][]")
+        # for key_data in filtered_iterator:
+        #     print(key_data)
+
+        # page_iterator = paginator.paginate(
+        #     Bucket=bucket,
+        #     Prefix=prefix,
+        #     PaginationConfig={
+        #         'MaxItems': 10,
+        #         'PageSize': 10,
+        #         'StartingToken': marker
+        #     }
+        # )
+
+    def exists(self, bucket: str, prefix: str) -> bool:
+        try:
+            self.s3.head_object(Bucket=bucket, Key=prefix)
+            return True
+        except botocore.exceptions.ClientError as err:
+            if err.response["Error"]["Code"]:
+                return False
+            raise

ddeutil/workflow/vendors/minio_warpped.py

@@ -0,0 +1,11 @@
+class WarpMinio:
+
+    def __init__(
+        self,
+        host: str,
+        access_key: str,
+        secret_access_key: str,
+    ):
+        self.host: str = host
+        self.access_key: str = access_key
+        self.secret_access_key: str = secret_access_key

ddeutil/workflow/vendors/sftp_wrapped.py

@@ -0,0 +1,209 @@
+import contextlib
+from collections import deque
+from collections.abc import Generator, Iterator
+from ftplib import FTP
+from stat import S_ISDIR, S_ISREG
+from typing import Optional
+
+try:
+    import paramiko
+    from paramiko import SFTPAttributes, SFTPClient
+    from sshtunnel import BaseSSHTunnelForwarderError, SSHTunnelForwarder
+except ImportError:
+    raise ImportError(
+        "Please install paramiko and sshtunnel packages before using,\n\t\t"
+        "$ pip install paramiko sshtunnel"
+    ) from None
+
+
+class WrapFTP:
+    """Wrapped FTP Client.
+
+        FTP (File Transfer Protocol) is the standard for transferring files
+    between the Client and Server is one of the most popular formats.
+
+        But there is a disadvantage that the data sent and received is not
+    encrypted. There is an opportunity for a third party to read the information
+    along the transferring.
+    """
+
+    def __init__(
+        self,
+        host: str,
+        user: str,
+        pwd: str,
+        port: int = 21,
+    ):
+        self.host: str = host
+        self.port: int = port
+        self.user: str = user
+        self.pwd: str = pwd
+
+    def fpt_connect(self):
+        return FTP(
+            host=self.host,
+            user=self.user,
+            passwd=self.pwd,
+        )
+
+
+class WrapSFTP:
+    """Wrapped SFTP Client.
+
+        SFTP (Secure File Transfer Protocol) it is a standard that helps
+    increase security in file transfers. By encrypting data and commands before
+    transferring files between the client and server with SSH (Secure Shell),
+    we can be confident that the data we upload/download can be done safely.
+
+        It cannot be accessed by third parties or if the information is
+    obtained, it is encrypted and cannot be read.
+
+    See-Also:
+
+        This object will wrap the [Paramiko](https://www.paramiko.org/) package
+    with my connection interface.
+    """
+
+    def __init__(
+        self,
+        host: str,
+        user: Optional[str] = None,
+        port: Optional[int] = None,
+        *,
+        pwd: Optional[str] = None,
+        private_key: Optional[str] = None,
+        private_key_password: Optional[str] = None,
+    ) -> None:
+        self.host: str = host
+        self.user: str = user or ""
+        self.port: int = port or 22
+        self.pwd: Optional[str] = pwd
+
+        # Private key path like, ``/home/user/.ssh/id_rsa``.
+        self.private_key = private_key
+
+        # If this private key have password, private_key passphrase.
+        self.private_key_pwd = private_key_password
+
+    def get(self, remote_path, local_path):
+        with self.transport_client() as sftp:
+            sftp.get(remote_path, local_path)
+
+    def put(self, remote_path, local_path):
+        with self.transport_client() as sftp:
+            sftp.put(remote_path, local_path)
+
+    def rm(self, remote_path: str):
+        with self.transport_client() as sftp:
+            sftp.remove(remote_path)
+
+    def mkdir(self, remote_path: str):
+        with self.transport_client() as sftp:
+            sftp.mkdir(remote_path)
+
+    @contextlib.contextmanager
+    def ssh_tunnel(self) -> Iterator:
+        try:
+            with SSHTunnelForwarder(
+                (self.host, self.port),
+                ssh_username=self.user,
+                ssh_password=self.pwd,
+                ssh_pkey=self.private_key,
+                ssh_private_key_password=self.private_key_pwd,
+                local_bind_address=("0.0.0.0", 22),
+                # Use a suitable remote_bind_address that able to be DB host on
+                # that SSH Server.
+                remote_bind_address=("127.0.0.1", self.port),
+            ) as tunnel:
+                tunnel.check_tunnels()
+                client = paramiko.SSHClient()
+                if self.private_key:
+                    client.load_system_host_keys()
+                # NOTE: Add SSH key to known_hosts file.
+                client.set_missing_host_key_policy(
+                    paramiko.MissingHostKeyPolicy()
+                )
+
+                # NOTE: Start connect to SSH Server
+                client.connect(
+                    "127.0.0.1",
+                    port=tunnel.local_bind_port,
+                    **(
+                        {
+                            "username": self.user,
+                            "password": self.pwd,
+                            "allow_agent": False,
+                            "look_for_keys": False,
+                            "banner_timeout": 20,
+                        }
+                        if self.pwd
+                        else {}
+                    ),
+                )
+                with client.open_sftp() as sftp:
+                    yield sftp
+                client.close()
+        except BaseSSHTunnelForwarderError as err:
+            raise ValueError(
+                "This config data does not connect to the Server"
+            ) from err
+
+    @contextlib.contextmanager
+    def transport_client(self) -> Generator[SFTPClient, None, None]:
+        with paramiko.Transport(sock=(self.host, self.port)) as transport:
+            transport.connect(
+                hostkey=None,
+                username=self.user,
+                password=self.pwd,
+            )
+            with paramiko.SFTPClient.from_transport(transport) as sftp:
+                yield sftp
+
+    @contextlib.contextmanager
+    def simple_client(self) -> Generator[SFTPClient, None, None]:
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.MissingHostKeyPolicy())
+        client.connect(
+            hostname=self.host,
+            port=self.port,
+            username=self.user,
+            password=self.pwd,
+        )
+        with client.open_sftp() as sftp:
+            yield sftp
+        client.close()
+
+    def glob(self, pattern: str) -> Iterator[str]:
+        with self.transport_client() as sftp:
+            try:
+                # NOTE: List files matching the pattern on the SFTP server
+                f: SFTPAttributes
+                for f in sftp.listdir_attr(pattern):
+                    yield pattern + f.filename
+            except FileNotFoundError:
+                raise FileNotFoundError(
+                    f"Pattern {pattern!r} does not found on SFTP server"
+                ) from None
+
+    def walk(self, pattern: str) -> Iterator[str]:
+        dirs: deque = deque([pattern])
+        with self.transport_client() as sftp:
+            while len(dirs) > 0:
+                d: str = dirs.popleft()
+                f: SFTPAttributes
+                for f in sftp.listdir_attr(d):
+                    rs: str = (
+                        (d + f.filename) if d == "/" else (d + "/" + f.filename)
+                    )
+                    if S_ISDIR(f.st_mode):
+                        dirs.append(rs)
+                    elif S_ISREG(f.st_mode):
+                        yield rs
+
+    @staticmethod
+    def isdir(path: SFTPAttributes):
+        try:
+            return S_ISDIR(path.st_mode)
+        except OSError:
+            # NOTE: Path does not exist, so by definition not a directory
+            return False

ddeutil_workflow-0.0.1.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Data Developer & Engineering
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ddeutil_workflow-0.0.1.dist-info/METADATA

@@ -0,0 +1,251 @@
+Metadata-Version: 2.1
+Name: ddeutil-workflow
+Version: 0.0.1
+Summary: Data Developer & Engineer Workflow Utility Objects
+Author-email: ddeutils <korawich.anu@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/ddeutils/ddeutil-workflow/
+Project-URL: Source Code, https://github.com/ddeutils/ddeutil-workflow/
+Keywords: data,workflow,utility,pipeline
+Classifier: Topic :: Utilities
+Classifier: Natural Language :: English
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.9.13
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: fmtutil
+Requires-Dist: ddeutil-io
+Requires-Dist: ddeutil-model
+Requires-Dist: python-dotenv
+Provides-Extra: test
+Requires-Dist: sqlalchemy ==2.0.30 ; extra == 'test'
+Requires-Dist: paramiko ==3.4.0 ; extra == 'test'
+Requires-Dist: sshtunnel ==0.4.0 ; extra == 'test'
+Requires-Dist: boto3 ==1.34.109 ; extra == 'test'
+Requires-Dist: fsspec ==2024.5.0 ; extra == 'test'
+Requires-Dist: polars ==0.20.26 ; extra == 'test'
+Requires-Dist: pyarrow ==16.1.0 ; extra == 'test'
+
+# Data Utility: _Workflow_
+
+[![test](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml)
+[![python support version](https://img.shields.io/pypi/pyversions/ddeutil-workflow)](https://pypi.org/project/ddeutil-workflow/)
+[![size](https://img.shields.io/github/languages/code-size/ddeutils/ddeutil-workflow)](https://github.com/ddeutils/ddeutil-workflow)
+
+**Table of Contents**:
+
+- [Installation](#installation)
+- [Getting Started](#getting-started)
+  - [Connection](#connection)
+  - [Dataset](#dataset)
+  - [Schedule](#schedule)
+- [Examples](#examples)
+  - [Python](#python)
+  - [Tasks (EL)](#tasks-extract--load)
+  - [Hooks (T)](#hooks-transform)
+
+This **Utility Workflow** objects was created for easy to make a simple metadata
+driven pipeline that able to **ETL, T, EL, or ELT** by `.yaml` file.
+
+I think we should not create the multiple pipeline per use-case if we able to
+write some dynamic pipeline that just change the input parameters per use-case
+instead. This way we can handle a lot of pipelines in our orgs with metadata only.
+It called **Metadata Driven**.
+
+Next, we should get some monitoring tools for manage logging that return from
+pipeline running. Because it not show us what is a use-case that running data
+pipeline.
+
+> [!NOTE]
+> _Disclaimer_: I inspire the dynamic statement from the GitHub Action `.yml` files
+> and all of config file from several data orchestration framework tools from my
+> experience on Data Engineer.
+
+## Installation
+
+```shell
+pip install ddeutil-workflow
+```
+
+This project need `ddeutil-io`, `ddeutil-model` extension namespace packages.
+
+## Getting Started
+
+The first step, you should start create the connections and datasets for In and
+Out of you data that want to use in pipeline of workflow. Some of this component
+is similar component of the **Airflow** because I like it concepts.
+
+### Connection
+
+The connection for worker able to do any thing.
+
+```yaml
+conn_postgres_data:
+  type: conn.Postgres
+  url: 'postgres//username:${ENV_PASS}@hostname:port/database?echo=True&time_out=10'
+```
+
+```python
+from ddeutil.workflow.conn import Conn
+
+conn = Conn.from_loader(name='conn_postgres_data', externals={})
+assert conn.ping()
+```
+
+### Dataset
+
+The dataset is define any objects on the connection.
+
+```yaml
+ds_postgres_customer_tbl:
+  type: dataset.PostgresTbl
+  conn: 'conn_postgres_data'
+  features:
+    id: serial primary key
+    name: varchar( 100 ) not null
+```
+
+```python
+from ddeutil.workflow.dataset import PostgresTbl
+
+dataset = PostgresTbl.from_loader(name='ds_postgres_customer_tbl', externals={})
+assert dataset.exists()
+```
+
+### Schedule
+
+```yaml
+schd_for_node:
+  type: schedule.Scdl
+  cron: "*/5 * * * *"
+```
+
+```python
+from ddeutil.workflow.schedule import Scdl
+
+scdl = Scdl.from_loader(name='schd_for_node', externals={})
+assert '*/5 * * * *' == str(scdl.cronjob)
+
+cron_iterate = scdl.generate('2022-01-01 00:00:00')
+assert '2022-01-01 00:05:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
+assert '2022-01-01 00:10:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
+assert '2022-01-01 00:15:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
+assert '2022-01-01 00:20:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
+assert '2022-01-01 00:25:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
+```
+
+## Examples
+
+This is examples that use workflow file for running common Data Engineering
+use-case.
+
+### Python
+
+The state of doing lists that worker should to do. It be collection of the stage.
+
+```yaml
+run_py_local:
+  type: ddeutil.workflow.pipe.Pipeline
+  params:
+    author-run: utils.receive.string
+    run-date: utils.receive.datetime
+  jobs:
+    first-job:
+      stages:
+        - name: Printing Information
+          id: define-func
+          run: |
+            x = '${{ params.author-run }}'
+            print(f'Hello {x}')
+
+            def echo(name: str):
+              print(f'Hello {name}')
+
+        - name: Run Sequence and use var from Above
+          vars:
+            x: ${{ params.author-run }}
+          run: |
+            print(f'Receive x from above with {x}')
+            # Change x value
+            x: int = 1
+
+        - name: Call Function
+          vars:
+            echo: ${{ stages.define-func.outputs.echo }}
+          run: |
+            echo('Caller')
+```
+
+```python
+from ddeutil.workflow.pipeline import Pipeline
+
+pipe = Pipeline.from_loader(name='run_py_local', externals={})
+pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
+```
+
+```shell
+> Hello Local Workflow
+> Receive x from above with Local Workflow
+> Hello Caller
+```
+
+### Tasks (Extract & Load)
+
+```yaml
+pipe_el_pg_to_lake:
+  type: ddeutil.workflow.pipe.Pipeline
+  params:
+    run-date: utils.receive.datetime
+    author-email: utils.receive.string
+  jobs:
+    extract-load:
+      stages:
+        - name: "Extract Load from Postgres to Lake"
+          id: extract
+          task: tasks/postgres-to-delta@polars
+          with:
+            source:
+              conn: conn_postgres_url
+              query: |
+                select * from ${{ params.name }}
+                where update_date = '${{ params.datetime }}'
+            sink:
+              conn: conn_az_lake
+              endpoint: "/${{ params.name }}"
+```
+
+### Hooks (Transform)
+
+```yaml
+pipe_hook_mssql_proc:
+  type: ddeutil.workflow.pipe.Pipeline
+  params:
+    run_date: utils.receive.datetime
+    sp_name: utils.receive.string
+    source_name: utils.receive.string
+    target_name: utils.receive.string
+  jobs:
+    transform:
+      stages:
+        - name: "Transform Data in MS SQL Server"
+          hook: hooks/mssql-proc@odbc
+          with:
+            exec: ${{ params.sp_name }}
+            params:
+              run_mode: "T"
+              run_date: ${{ params.run_date }}
+              source: ${{ params.source_name }}
+              target: ${{ params.target_name }}
+```
+
+## License
+
+This project was licensed under the terms of the [MIT license](LICENSE).