ddeutil-workflow 0.0.3__tar.gz → 0.0.5__tar.gz

{ddeutil_workflow-0.0.3/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.3
+Version: 0.0.5
 Summary: Data Developer & Engineer Workflow Utility Objects
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -9,7 +9,7 @@ Project-URL: Source Code, https://github.com/ddeutils/ddeutil-workflow/
 Keywords: data,workflow,utility,pipeline
 Classifier: Topic :: Utilities
 Classifier: Natural Language :: English
-Classifier: Development Status :: 3 - Alpha
+Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
@@ -23,21 +23,16 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: fmtutil
 Requires-Dist: ddeutil-io
-Requires-Dist: python-dotenv
-Provides-Extra: test
-Requires-Dist: sqlalchemy==2.0.30; extra == "test"
-Requires-Dist: paramiko==3.4.0; extra == "test"
-Requires-Dist: sshtunnel==0.4.0; extra == "test"
-Requires-Dist: boto3==1.34.117; extra == "test"
-Requires-Dist: fsspec==2024.5.0; extra == "test"
-Requires-Dist: polars==0.20.31; extra == "test"
-Requires-Dist: pyarrow==16.1.0; extra == "test"
+Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: schedule==1.2.2
 
 # Data Utility: _Workflow_
 
 [![test](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml)
 [![python support version](https://img.shields.io/pypi/pyversions/ddeutil-workflow)](https://pypi.org/project/ddeutil-workflow/)
 [![size](https://img.shields.io/github/languages/code-size/ddeutils/ddeutil-workflow)](https://github.com/ddeutils/ddeutil-workflow)
+[![gh license](https://img.shields.io/github/license/ddeutils/ddeutil-workflow)](https://github.com/ddeutils/ddeutil-workflow/blob/main/LICENSE)
+
 
 **Table of Contents**:
 
@@ -46,10 +41,11 @@ Requires-Dist: pyarrow==16.1.0; extra == "test"
   - [Connection](#connection)
   - [Dataset](#dataset)
   - [Schedule](#schedule)
-- [Examples](#examples)
-  - [Python](#python)
+- [Pipeline Examples](#examples)
+  - [Python & Shell](#python--shell)
   - [Tasks (EL)](#tasks-extract--load)
-  - [Hooks (T)](#hooks-transform)
+  - [Hooks (T)](#tasks-transform)
+- [Configuration](#configuration)
 
 This **Utility Workflow** objects was created for easy to make a simple metadata
 driven pipeline that able to **ETL, T, EL, or ELT** by `.yaml` file.
@@ -80,7 +76,7 @@ This project need `ddeutil-io`, `ddeutil-model` extension namespace packages.
 
 The first step, you should start create the connections and datasets for In and
 Out of you data that want to use in pipeline of workflow. Some of this component
-is similar component of the **Airflow** because I like it concepts.
+is similar component of the **Airflow** because I like it orchestration concepts.
 
 The main feature of this project is the `Pipeline` object that can call any
 registries function. The pipeline can handle everything that you want to do, it
@@ -91,44 +87,7 @@ will passing parameters and catching the output for re-use it to next step.
 > dynamic registries instead of main features because it have a lot of maintain
 > vendor codes and deps. (I do not have time to handle this features)
 
-### Connection
-
-The connection for worker able to do any thing.
-
-```yaml
-conn_postgres_data:
-  type: conn.Postgres
-  url: 'postgres//username:${ENV_PASS}@hostname:port/database?echo=True&time_out=10'
-```
-
-```python
-from ddeutil.workflow.conn import Conn
-
-conn = Conn.from_loader(name='conn_postgres_data', externals={})
-assert conn.ping()
-```
-
-### Dataset
-
-The dataset is define any objects on the connection. This feature was implemented
-on `/vendors` because it has a lot of tools that can interact with any data systems
-in the data tool stacks.
-
-```yaml
-ds_postgres_customer_tbl:
-  type: dataset.PostgresTbl
-  conn: 'conn_postgres_data'
-  features:
-    id: serial primary key
-    name: varchar( 100 ) not null
-```
-
-```python
-from ddeutil.workflow.vendors.pg import PostgresTbl
-
-dataset = PostgresTbl.from_loader(name='ds_postgres_customer_tbl', externals={})
-assert dataset.exists()
-```
+---
 
 ### Schedule
 
@@ -139,7 +98,7 @@ schd_for_node:
 ```
 
 ```python
-from ddeutil.workflow.schedule import Schedule
+from ddeutil.workflow.on import Schedule
 
 scdl = Schedule.from_loader(name='schd_for_node', externals={})
 assert '*/5 * * * *' == str(scdl.cronjob)
@@ -152,18 +111,35 @@ assert '2022-01-01 00:20:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
 assert '2022-01-01 00:25:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
 ```
 
+---
+
+### Pipeline
+
+```yaml
+run_py_local:
+  type: ddeutil.workflow.pipeline.Pipeline
+  ...
+```
+
+```python
+from ddeutil.workflow.pipeline import Pipeline
+
+pipe = Pipeline.from_loader(name='run_py_local', externals={})
+pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
+```
+
 ## Examples
 
 This is examples that use workflow file for running common Data Engineering
 use-case.
 
-### Python
+### Python & Shell
 
 The state of doing lists that worker should to do. It be collection of the stage.
 
 ```yaml
 run_py_local:
-  type: ddeutil.workflow.pipe.Pipeline
+  type: ddeutil.workflow.pipeline.Pipeline
   params:
     author-run:
       type: str
@@ -194,6 +170,12 @@ run_py_local:
             echo: ${{ stages.define-func.outputs.echo }}
           run: |
             echo('Caller')
+    second-job:
+      stages:
+        - name: Echo Shell Script
+          id: shell-echo
+          shell: |
+            echo "Hello World from Shell"
 ```
 
 ```python
@@ -207,13 +189,16 @@ pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
 > Hello Local Workflow
 > Receive x from above with Local Workflow
 > Hello Caller
+> Hello World from Shell
 ```
 
+---
+
 ### Tasks (Extract & Load)
 
 ```yaml
 pipe_el_pg_to_lake:
-  type: ddeutil.workflow.pipe.Pipeline
+  type: ddeutil.workflow.pipeline.Pipeline
   params:
     run-date:
       type: datetime
@@ -236,11 +221,15 @@ pipe_el_pg_to_lake:
               endpoint: "/${{ params.name }}"
 ```
 
+---
+
 ### Tasks (Transform)
 
+> I recommend you to use task for all actions that you want to do.
+
 ```yaml
 pipe_hook_mssql_proc:
-  type: ddeutil.workflow.pipe.Pipeline
+  type: ddeutil.workflow.pipeline.Pipeline
   params:
     run_date: datetime
     sp_name: str
@@ -261,6 +250,16 @@ pipe_hook_mssql_proc:
               target: ${{ params.target_name }}
 ```
 
+> [!NOTE]
+> The above parameter use short declarative statement. You can pass a parameter
+> type to the key of a parameter name.
+
+## Configuration
+
+```text
+
+```
+
 ## License
 
 This project was licensed under the terms of the [MIT license](LICENSE).

{ddeutil_workflow-0.0.3 → ddeutil_workflow-0.0.5}/README.md

@@ -3,6 +3,8 @@
 [![test](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml)
 [![python support version](https://img.shields.io/pypi/pyversions/ddeutil-workflow)](https://pypi.org/project/ddeutil-workflow/)
 [![size](https://img.shields.io/github/languages/code-size/ddeutils/ddeutil-workflow)](https://github.com/ddeutils/ddeutil-workflow)
+[![gh license](https://img.shields.io/github/license/ddeutils/ddeutil-workflow)](https://github.com/ddeutils/ddeutil-workflow/blob/main/LICENSE)
+
 
 **Table of Contents**:
 
@@ -11,10 +13,11 @@
   - [Connection](#connection)
   - [Dataset](#dataset)
   - [Schedule](#schedule)
-- [Examples](#examples)
-  - [Python](#python)
+- [Pipeline Examples](#examples)
+  - [Python & Shell](#python--shell)
   - [Tasks (EL)](#tasks-extract--load)
-  - [Hooks (T)](#hooks-transform)
+  - [Hooks (T)](#tasks-transform)
+- [Configuration](#configuration)
 
 This **Utility Workflow** objects was created for easy to make a simple metadata
 driven pipeline that able to **ETL, T, EL, or ELT** by `.yaml` file.
@@ -45,7 +48,7 @@ This project need `ddeutil-io`, `ddeutil-model` extension namespace packages.
 
 The first step, you should start create the connections and datasets for In and
 Out of you data that want to use in pipeline of workflow. Some of this component
-is similar component of the **Airflow** because I like it concepts.
+is similar component of the **Airflow** because I like it orchestration concepts.
 
 The main feature of this project is the `Pipeline` object that can call any
 registries function. The pipeline can handle everything that you want to do, it
@@ -56,44 +59,7 @@ will passing parameters and catching the output for re-use it to next step.
 > dynamic registries instead of main features because it have a lot of maintain
 > vendor codes and deps. (I do not have time to handle this features)
 
-### Connection
-
-The connection for worker able to do any thing.
-
-```yaml
-conn_postgres_data:
-  type: conn.Postgres
-  url: 'postgres//username:${ENV_PASS}@hostname:port/database?echo=True&time_out=10'
-```
-
-```python
-from ddeutil.workflow.conn import Conn
-
-conn = Conn.from_loader(name='conn_postgres_data', externals={})
-assert conn.ping()
-```
-
-### Dataset
-
-The dataset is define any objects on the connection. This feature was implemented
-on `/vendors` because it has a lot of tools that can interact with any data systems
-in the data tool stacks.
-
-```yaml
-ds_postgres_customer_tbl:
-  type: dataset.PostgresTbl
-  conn: 'conn_postgres_data'
-  features:
-    id: serial primary key
-    name: varchar( 100 ) not null
-```
-
-```python
-from ddeutil.workflow.vendors.pg import PostgresTbl
-
-dataset = PostgresTbl.from_loader(name='ds_postgres_customer_tbl', externals={})
-assert dataset.exists()
-```
+---
 
 ### Schedule
 
@@ -104,7 +70,7 @@ schd_for_node:
 ```
 
 ```python
-from ddeutil.workflow.schedule import Schedule
+from ddeutil.workflow.on import Schedule
 
 scdl = Schedule.from_loader(name='schd_for_node', externals={})
 assert '*/5 * * * *' == str(scdl.cronjob)
@@ -117,18 +83,35 @@ assert '2022-01-01 00:20:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
 assert '2022-01-01 00:25:00' f"{cron_iterate.next:%Y-%m-%d %H:%M:%S}"
 ```
 
+---
+
+### Pipeline
+
+```yaml
+run_py_local:
+  type: ddeutil.workflow.pipeline.Pipeline
+  ...
+```
+
+```python
+from ddeutil.workflow.pipeline import Pipeline
+
+pipe = Pipeline.from_loader(name='run_py_local', externals={})
+pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
+```
+
 ## Examples
 
 This is examples that use workflow file for running common Data Engineering
 use-case.
 
-### Python
+### Python & Shell
 
 The state of doing lists that worker should to do. It be collection of the stage.
 
 ```yaml
 run_py_local:
-  type: ddeutil.workflow.pipe.Pipeline
+  type: ddeutil.workflow.pipeline.Pipeline
   params:
     author-run:
       type: str
@@ -159,6 +142,12 @@ run_py_local:
             echo: ${{ stages.define-func.outputs.echo }}
           run: |
             echo('Caller')
+    second-job:
+      stages:
+        - name: Echo Shell Script
+          id: shell-echo
+          shell: |
+            echo "Hello World from Shell"
 ```
 
 ```python
@@ -172,13 +161,16 @@ pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
 > Hello Local Workflow
 > Receive x from above with Local Workflow
 > Hello Caller
+> Hello World from Shell
 ```
 
+---
+
 ### Tasks (Extract & Load)
 
 ```yaml
 pipe_el_pg_to_lake:
-  type: ddeutil.workflow.pipe.Pipeline
+  type: ddeutil.workflow.pipeline.Pipeline
   params:
     run-date:
       type: datetime
@@ -201,11 +193,15 @@ pipe_el_pg_to_lake:
               endpoint: "/${{ params.name }}"
 ```
 
+---
+
 ### Tasks (Transform)
 
+> I recommend you to use task for all actions that you want to do.
+
 ```yaml
 pipe_hook_mssql_proc:
-  type: ddeutil.workflow.pipe.Pipeline
+  type: ddeutil.workflow.pipeline.Pipeline
   params:
     run_date: datetime
     sp_name: str
@@ -226,6 +222,16 @@ pipe_hook_mssql_proc:
               target: ${{ params.target_name }}
 ```
 
+> [!NOTE]
+> The above parameter use short declarative statement. You can pass a parameter
+> type to the key of a parameter name.
+
+## Configuration
+
+```text
+
+```
+
 ## License
 
 This project was licensed under the terms of the [MIT license](LICENSE).

{ddeutil_workflow-0.0.3 → ddeutil_workflow-0.0.5}/pyproject.toml

@@ -12,8 +12,7 @@ keywords = ['data', 'workflow', 'utility', 'pipeline']
 classifiers = [
     "Topic :: Utilities",
     "Natural Language :: English",
-    "Development Status :: 3 - Alpha",
-    # "Development Status :: 4 - Beta",
+    "Development Status :: 4 - Beta",
     # "Development Status :: 5 - Production/Stable",
     "Intended Audience :: Developers",
     "Operating System :: OS Independent",
@@ -28,7 +27,8 @@ requires-python = ">=3.9.13"
 dependencies = [
     "fmtutil",
     "ddeutil-io",
-    "python-dotenv",
+    "python-dotenv==1.0.1",
+    "schedule==1.2.2",
 ]
 dynamic = ["version"]
 
@@ -36,21 +36,6 @@ dynamic = ["version"]
 Homepage = "https://github.com/ddeutils/ddeutil-workflow/"
 "Source Code" = "https://github.com/ddeutils/ddeutil-workflow/"
 
-[project.optional-dependencies]
-test = [
-    "sqlalchemy==2.0.30",
-    # SFTP
-    "paramiko==3.4.0",
-    "sshtunnel==0.4.0",
-    # AWS Client
-    "boto3==1.34.117",
-    # Open files
-    "fsspec==2024.5.0",
-    # Polars
-    "polars==0.20.31",
-    "pyarrow==16.1.0",
-]
-
 [tool.setuptools.dynamic]
 version = {attr = "ddeutil.workflow.__about__.__version__"}
 
@@ -80,8 +65,8 @@ addopts = [
 ]
 filterwarnings = ["error"]
 log_cli = true
-log_cli_level = "INFO"
-log_cli_format = "%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)"
+log_cli_level = "DEBUG"
+log_cli_format = "%(asctime)s [%(levelname)7s] %(message)s (%(filename)s:%(lineno)s)"
 log_cli_date_format = "%Y-%m-%d %H:%M:%S"
 
 [tool.black]

ddeutil_workflow-0.0.5/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.5"