ddeutil-workflow 0.0.7__py3-none-any.whl → 0.0.9__py3-none-any.whl

ddeutil/workflow/app.py

@@ -1,41 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
-# Licensed under the MIT License. See LICENSE in the project root for
-# license information.
-# ------------------------------------------------------------------------------
-import functools
-import time
-
-import schedule
-
-
-def catch_exceptions(cancel_on_failure=False):
-    def catch_exceptions_decorator(job_func):
-        @functools.wraps(job_func)
-        def wrapper(*args, **kwargs):
-            try:
-                return job_func(*args, **kwargs)
-            except Exception as err:
-                print(err)
-
-                if cancel_on_failure:
-                    return schedule.CancelJob
-
-        return wrapper
-
-    return catch_exceptions_decorator
-
-
-@catch_exceptions(cancel_on_failure=True)
-def bad_task():
-    return 1 / 0
-
-
-schedule.every(5).seconds.do(bad_task)
-
-if __name__ == "__main__":
-    while True:
-        schedule.run_pending()
-        time.sleep(1)
-        if not schedule.get_jobs():
-            break

ddeutil_workflow-0.0.7.dist-info/METADATA

@@ -1,341 +0,0 @@
-Metadata-Version: 2.1
-Name: ddeutil-workflow
-Version: 0.0.7
-Summary: Data Developer & Engineer Workflow Utility Objects
-Author-email: ddeutils <korawich.anu@gmail.com>
-License: MIT
-Project-URL: Homepage, https://github.com/ddeutils/ddeutil-workflow/
-Project-URL: Source Code, https://github.com/ddeutils/ddeutil-workflow/
-Keywords: data,workflow,utility,pipeline
-Classifier: Topic :: Utilities
-Classifier: Natural Language :: English
-Classifier: Development Status :: 4 - Beta
-Classifier: Intended Audience :: Developers
-Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Requires-Python: >=3.9.13
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: fmtutil
-Requires-Dist: ddeutil-io
-Requires-Dist: python-dotenv ==1.0.1
-Provides-Extra: api
-Requires-Dist: fastapi[standard] ==0.112.0 ; extra == 'api'
-Requires-Dist: apscheduler[sqlalchemy] <4.0.0,==3.10.4 ; extra == 'api'
-Requires-Dist: croniter ==3.0.3 ; extra == 'api'
-Provides-Extra: app
-Requires-Dist: schedule <2.0.0,==1.2.2 ; extra == 'app'
-
-# Workflow
-
-[![test](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml)
-[![python support version](https://img.shields.io/pypi/pyversions/ddeutil-workflow)](https://pypi.org/project/ddeutil-workflow/)
-[![size](https://img.shields.io/github/languages/code-size/ddeutils/ddeutil-workflow)](https://github.com/ddeutils/ddeutil-workflow)
-[![gh license](https://img.shields.io/github/license/ddeutils/ddeutil-workflow)](https://github.com/ddeutils/ddeutil-workflow/blob/main/LICENSE)
-
-**Table of Contents**:
-
-- [Installation](#installation)
-- [Getting Started](#getting-started)
-  - [On](#on)
-  - [Pipeline](#pipeline)
-- [Usage](#usage)
-  - [Python & Bash](#python--bash)
-  - [Hook (EL)](#hook-extract--load)
-  - [Hook (T)](#hook-transform)
-- [Configuration](#configuration)
-- [Deployment](#deployment)
-
-This **Workflow** objects was created for easy to make a simple metadata
-driven for data pipeline orchestration that able to use for **ETL, T, EL, or
-ELT** by a `.yaml` file template.
-
-In my opinion, I think it should not create duplicate pipeline codes if I can
-write with dynamic input parameters on the one template pipeline that just change
-the input parameters per use-case instead.
-This way I can handle a lot of logical pipelines in our orgs with only metadata
-configuration. It called **Metadata Driven Data Pipeline**.
-
-Next, we should get some monitoring tools for manage logging that return from
-pipeline running. Because it not show us what is a use-case that running data
-pipeline.
-
-> [!NOTE]
-> _Disclaimer_: I inspire the dynamic statement from the GitHub Action `.yml` files
-> and all of config file from several data orchestration framework tools from my
-> experience on Data Engineer.
-
-## Installation
-
-```shell
-pip install ddeutil-workflow
-```
-
-This project need `ddeutil-io` extension namespace packages. If you want to install
-this package with application add-ons, you should add `app` in installation;
-
-```shell
-pip install ddeutil-workflow[app]
-```
-
-```shell
-pip install ddeutil-workflow[api]
-```
-
-## Getting Started
-
-The first step, you should start create the connections and datasets for In and
-Out of you data that want to use in pipeline of workflow. Some of this component
-is similar component of the **Airflow** because I like it orchestration concepts.
-
-The main feature of this project is the `Pipeline` object that can call any
-registries function. The pipeline can handle everything that you want to do, it
-will passing parameters and catching the output for re-use it to next step.
-
-> [!IMPORTANT]
-> In the future of this project, I will drop the connection and dataset to
-> dynamic registries instead of main features because it have a lot of maintain
-> vendor codes and deps. (I do not have time to handle this features)
-
-### On
-
-The **On** is schedule object.
-
-```yaml
-on_every_5_min:
-  type: on.On
-  cron: "*/5 * * * *"
-```
-
-```python
-from ddeutil.workflow.on import On
-
-schedule = On.from_loader(name='on_every_5_min', externals={})
-assert '*/5 * * * *' == str(schedule.cronjob)
-
-cron_iter = schedule.generate('2022-01-01 00:00:00')
-assert '2022-01-01 00:05:00' f"{cron_iter.next:%Y-%m-%d %H:%M:%S}"
-assert '2022-01-01 00:10:00' f"{cron_iter.next:%Y-%m-%d %H:%M:%S}"
-assert '2022-01-01 00:15:00' f"{cron_iter.next:%Y-%m-%d %H:%M:%S}"
-assert '2022-01-01 00:20:00' f"{cron_iter.next:%Y-%m-%d %H:%M:%S}"
-```
-
-### Pipeline
-
-The **Pipeline** object that is the core feature of this project.
-
-```yaml
-run_py_local:
-  type: ddeutil.workflow.pipeline.Pipeline
-  on: 'on_every_5_min'
-  params:
-    author-run:
-      type: str
-    run-date:
-      type: datetime
-```
-
-```python
-from ddeutil.workflow.pipeline import Pipeline
-
-pipe = Pipeline.from_loader(name='run_py_local', externals={})
-pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
-```
-
-> [!NOTE]
-> The above parameter use short declarative statement. You can pass a parameter
-> type to the key of a parameter name.
-> ```yaml
-> params:
->   author-run: str
->   run-date: datetime
-> ```
->
-> And for the type, you can remove `ddeutil.workflow` prefix because we can find
-> it by looping search from `WORKFLOW_CORE_REGISTRY` value.
-
-## Usage
-
-This is examples that use workflow file for running common Data Engineering
-use-case.
-
-> [!IMPORTANT]
-> I recommend you to use `task` stage for all actions that you want to do with
-> pipeline object.
-
-### Python & Bash
-
-```yaml
-run_py_local:
-  type: pipeline.Pipeline
-  params:
-    author-run: str
-    run-date: datetime
-  jobs:
-    first-job:
-      stages:
-        - name: "Printing Information"
-          id: define-func
-          run: |
-            x = '${{ params.author-run }}'
-            print(f'Hello {x}')
-
-            def echo(name: str):
-              print(f'Hello {name}')
-
-        - name: "Run Sequence and use var from Above"
-          vars:
-            x: ${{ params.author-run }}
-          run: |
-            print(f'Receive x from above with {x}')
-            # Change x value
-            x: int = 1
-
-        - name: "Call Function"
-          vars:
-            echo: ${{ stages.define-func.outputs.echo }}
-          run: |
-            echo('Caller')
-    second-job:
-      stages:
-        - name: "Echo Bash Script"
-          id: shell-echo
-          bash: |
-            echo "Hello World from Shell"
-```
-
-```python
-from ddeutil.workflow.pipeline import Pipeline
-
-pipe = Pipeline.from_loader(name='run_py_local', externals={})
-pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
-```
-
-```shell
-> Hello Local Workflow
-> Receive x from above with Local Workflow
-> Hello Caller
-> Hello World from Shell
-```
-
-### Hook (Extract & Load)
-
-```yaml
-pipe_el_pg_to_lake:
-  type: pipeline.Pipeline
-  params:
-    run-date: datetime
-    author-email: str
-  jobs:
-    extract-load:
-      stages:
-        - name: "Extract Load from Postgres to Lake"
-          id: extract-load
-          uses: tasks/postgres-to-delta@polars
-          with:
-            source:
-              conn: conn_postgres_url
-              query: |
-                select * from ${{ params.name }}
-                where update_date = '${{ params.datetime }}'
-            sink:
-              conn: conn_az_lake
-              endpoint: "/${{ params.name }}"
-```
-
-Implement hook:
-
-```python
-from ddeutil.workflow.utils import tag
-
-@tag('polars', alias='postgres-to-delta')
-def postgres_to_delta(source, sink):
-    return {
-        "source": source, "sink": sink
-    }
-```
-
-### Hook (Transform)
-
-```yaml
-pipeline_hook_mssql_proc:
-  type: pipeline.Pipeline
-  params:
-    run_date: datetime
-    sp_name: str
-    source_name: str
-    target_name: str
-  jobs:
-    transform:
-      stages:
-        - name: "Transform Data in MS SQL Server"
-          id: transform
-          uses: tasks/mssql-proc@odbc
-          with:
-            exec: ${{ params.sp_name }}
-            params:
-              run_mode: "T"
-              run_date: ${{ params.run_date }}
-              source: ${{ params.source_name }}
-              target: ${{ params.target_name }}
-```
-
-Implement hook:
-
-```python
-from ddeutil.workflow.utils import tag
-
-@tag('odbc', alias='mssql-proc')
-def odbc_mssql_procedure(_exec: str, params: dict):
-    return {
-        "exec": _exec, "params": params
-    }
-```
-
-## Configuration
-
-```bash
-export WORKFLOW_ROOT_PATH=.
-export WORKFLOW_CORE_REGISTRY=ddeutil.workflow,tests.utils
-export WORKFLOW_CORE_REGISTRY_FILTER=ddeutil.workflow.utils
-export WORKFLOW_CORE_PATH_CONF=conf
-export WORKFLOW_CORE_TIMEZONE=Asia/Bangkok
-export WORKFLOW_CORE_DEFAULT_STAGE_ID=true
-
-export WORKFLOW_CORE_MAX_PIPELINE_POKING=4
-export WORKFLOW_CORE_MAX_JOB_PARALLEL=2
-```
-
-Application config:
-
-```bash
-export WORKFLOW_APP_DB_URL=postgresql+asyncpg://user:pass@localhost:5432/schedule
-export WORKFLOW_APP_INTERVAL=10
-```
-
-## Deployment
-
-This package able to run as a application service for receive manual trigger
-from the master node via RestAPI or use to be Scheduler background service
-like crontab job but via Python API.
-
-### Schedule Service
-
-```shell
-(venv) $ python src.ddeutil.workflow.app
-```
-
-### API Server
-
-```shell
-(venv) $ uvicorn src.ddeutil.workflow.api:app --host 0.0.0.0 --port 80 --reload
-```
-
-> [!NOTE]
-> If this package already deploy, it able to use
-> `uvicorn ddeutil.workflow.api:app --host 0.0.0.0 --port 80`

ddeutil_workflow-0.0.7.dist-info/RECORD

@@ -1,20 +0,0 @@
-ddeutil/workflow/__about__.py,sha256=b23XabBwtuoPOLmS_Hj_gSA4LZ0fRfAkACM6c3szVoc,27
-ddeutil/workflow/__init__.py,sha256=4PEL3RdHmUowK0Dz-tK7fO0wvFX4u9CLd0Up7b3lrAQ,760
-ddeutil/workflow/__types.py,sha256=SYMoxbENQX8uPsiCZkjtpHAqqHOh8rUrarAFicAJd0E,1773
-ddeutil/workflow/api.py,sha256=d2Mmv9jTtN3FITIy-2mivyAKdBOGZxtkNWRMPbCLlFI,3341
-ddeutil/workflow/app.py,sha256=GbdwvUkE8lO2Ze4pZ0-J-7p9mcZAaORfjkHwW_oZIP0,1076
-ddeutil/workflow/exceptions.py,sha256=BH7COn_3uz3z7oJBZOQGiuo8osBFgeXL8HYymnjCOPQ,671
-ddeutil/workflow/loader.py,sha256=_ZD-XP5P7VbUeqItrUVPaKIZu6dMUZ2aywbCbReW1hQ,2778
-ddeutil/workflow/log.py,sha256=_GJEdJr7bqpcQDxZjrqHd-hkiW3NKFaVoR6voE6Ty0o,952
-ddeutil/workflow/on.py,sha256=YoEqDbzJUwqOA3JRltbvlYr0rNTtxdmb7cWMxl8U19k,6717
-ddeutil/workflow/pipeline.py,sha256=dKF09TFS_v5TCD-5o8tp1UhB5sGuWIQu4zl_UFtlIC0,25951
-ddeutil/workflow/repeat.py,sha256=sNoRfbOR4cYm_edrSvlVy9N8Dk_osLIq9FC5GMZz32M,4621
-ddeutil/workflow/route.py,sha256=Ck_O1xJwI-vKkMJr37El0-1PGKlwKF8__DDNWVQrf0A,2079
-ddeutil/workflow/scheduler.py,sha256=FqmkvWCqwJ4eRf8aDn5Ce4FcNWqmcvu2aTTfL34lfgs,22184
-ddeutil/workflow/stage.py,sha256=z05bKk2QFQDXjidSnQYCVOdceSpSO13sHXE0B1UH6XA,14978
-ddeutil/workflow/utils.py,sha256=pDM2jaYVP-USH0pLd_XmHOguxVPGVzZ76hOh1AZdINU,18495
-ddeutil_workflow-0.0.7.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.7.dist-info/METADATA,sha256=ba2nH57cpHB2P4ldQCRT8ZWDj3r1OPx9a1dgcB0a2Ws,9702
-ddeutil_workflow-0.0.7.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
-ddeutil_workflow-0.0.7.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.7.dist-info/RECORD,,