ddeutil-workflow 0.0.7__tar.gz → 0.0.8__tar.gz

{ddeutil_workflow-0.0.7 → ddeutil_workflow-0.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.7
+Version: 0.0.8
 Summary: Data Developer & Engineer Workflow Utility Objects
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -79,13 +79,10 @@ pip install ddeutil-workflow
 This project need `ddeutil-io` extension namespace packages. If you want to install
 this package with application add-ons, you should add `app` in installation;
 
-```shell
-pip install ddeutil-workflow[app]
-```
-
-```shell
-pip install ddeutil-workflow[api]
-```
+| Usecase            | Install Optional          |
+|--------------------|---------------------------|
+| Scheduler Service  | `ddeutil-workflow[app]`   |
+| FastAPI Server     | `ddeutil-workflow[api]`   |
 
 ## Getting Started
 
@@ -168,11 +165,12 @@ use-case.
 > I recommend you to use `task` stage for all actions that you want to do with
 > pipeline object.
 
-### Python & Bash
-
 ```yaml
 run_py_local:
   type: pipeline.Pipeline
+  on:
+    - cronjob: '* * * * *'
+      timezone: "Asia/Bangkok"
   params:
     author-run: str
     run-date: datetime
@@ -182,12 +180,11 @@ run_py_local:
         - name: "Printing Information"
           id: define-func
           run: |
-            x = '${{ params.author-run }}'
-            print(f'Hello {x}')
+            x = '${{ params.run-date | fmt("%Y%m%d") }}'
+            print(f'Hello at {x}')
 
             def echo(name: str):
               print(f'Hello {name}')
-
         - name: "Run Sequence and use var from Above"
           vars:
             x: ${{ params.author-run }}
@@ -195,7 +192,6 @@ run_py_local:
             print(f'Receive x from above with {x}')
             # Change x value
             x: int = 1
-
         - name: "Call Function"
           vars:
             echo: ${{ stages.define-func.outputs.echo }}
@@ -210,93 +206,23 @@ run_py_local:
 ```
 
 ```python
+from datetime import datetime
 from ddeutil.workflow.pipeline import Pipeline
 
-pipe = Pipeline.from_loader(name='run_py_local', externals={})
-pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
+pipe: Pipeline = Pipeline.from_loader(name='run_py_local', externals={})
+pipe.execute(params={
+    'author-run': 'Local Workflow',
+    'run-date': datetime(2024, 1, 1),
+})
 ```
 
 ```shell
-> Hello Local Workflow
+> Hello at 20240101
 > Receive x from above with Local Workflow
 > Hello Caller
 > Hello World from Shell
 ```
 
-### Hook (Extract & Load)
-
-```yaml
-pipe_el_pg_to_lake:
-  type: pipeline.Pipeline
-  params:
-    run-date: datetime
-    author-email: str
-  jobs:
-    extract-load:
-      stages:
-        - name: "Extract Load from Postgres to Lake"
-          id: extract-load
-          uses: tasks/postgres-to-delta@polars
-          with:
-            source:
-              conn: conn_postgres_url
-              query: |
-                select * from ${{ params.name }}
-                where update_date = '${{ params.datetime }}'
-            sink:
-              conn: conn_az_lake
-              endpoint: "/${{ params.name }}"
-```
-
-Implement hook:
-
-```python
-from ddeutil.workflow.utils import tag
-
-@tag('polars', alias='postgres-to-delta')
-def postgres_to_delta(source, sink):
-    return {
-        "source": source, "sink": sink
-    }
-```
-
-### Hook (Transform)
-
-```yaml
-pipeline_hook_mssql_proc:
-  type: pipeline.Pipeline
-  params:
-    run_date: datetime
-    sp_name: str
-    source_name: str
-    target_name: str
-  jobs:
-    transform:
-      stages:
-        - name: "Transform Data in MS SQL Server"
-          id: transform
-          uses: tasks/mssql-proc@odbc
-          with:
-            exec: ${{ params.sp_name }}
-            params:
-              run_mode: "T"
-              run_date: ${{ params.run_date }}
-              source: ${{ params.source_name }}
-              target: ${{ params.target_name }}
-```
-
-Implement hook:
-
-```python
-from ddeutil.workflow.utils import tag
-
-@tag('odbc', alias='mssql-proc')
-def odbc_mssql_procedure(_exec: str, params: dict):
-    return {
-        "exec": _exec, "params": params
-    }
-```
-
 ## Configuration
 
 ```bash
@@ -306,7 +232,6 @@ export WORKFLOW_CORE_REGISTRY_FILTER=ddeutil.workflow.utils
 export WORKFLOW_CORE_PATH_CONF=conf
 export WORKFLOW_CORE_TIMEZONE=Asia/Bangkok
 export WORKFLOW_CORE_DEFAULT_STAGE_ID=true
-
 export WORKFLOW_CORE_MAX_PIPELINE_POKING=4
 export WORKFLOW_CORE_MAX_JOB_PARALLEL=2
 ```

{ddeutil_workflow-0.0.7 → ddeutil_workflow-0.0.8}/README.md

@@ -46,13 +46,10 @@ pip install ddeutil-workflow
 This project need `ddeutil-io` extension namespace packages. If you want to install
 this package with application add-ons, you should add `app` in installation;
 
-```shell
-pip install ddeutil-workflow[app]
-```
-
-```shell
-pip install ddeutil-workflow[api]
-```
+| Usecase            | Install Optional          |
+|--------------------|---------------------------|
+| Scheduler Service  | `ddeutil-workflow[app]`   |
+| FastAPI Server     | `ddeutil-workflow[api]`   |
 
 ## Getting Started
 
@@ -135,11 +132,12 @@ use-case.
 > I recommend you to use `task` stage for all actions that you want to do with
 > pipeline object.
 
-### Python & Bash
-
 ```yaml
 run_py_local:
   type: pipeline.Pipeline
+  on:
+    - cronjob: '* * * * *'
+      timezone: "Asia/Bangkok"
   params:
     author-run: str
     run-date: datetime
@@ -149,12 +147,11 @@ run_py_local:
         - name: "Printing Information"
           id: define-func
           run: |
-            x = '${{ params.author-run }}'
-            print(f'Hello {x}')
+            x = '${{ params.run-date | fmt("%Y%m%d") }}'
+            print(f'Hello at {x}')
 
             def echo(name: str):
               print(f'Hello {name}')
-
         - name: "Run Sequence and use var from Above"
           vars:
             x: ${{ params.author-run }}
@@ -162,7 +159,6 @@ run_py_local:
             print(f'Receive x from above with {x}')
             # Change x value
             x: int = 1
-
         - name: "Call Function"
           vars:
             echo: ${{ stages.define-func.outputs.echo }}
@@ -177,93 +173,23 @@ run_py_local:
 ```
 
 ```python
+from datetime import datetime
 from ddeutil.workflow.pipeline import Pipeline
 
-pipe = Pipeline.from_loader(name='run_py_local', externals={})
-pipe.execute(params={'author-run': 'Local Workflow', 'run-date': '2024-01-01'})
+pipe: Pipeline = Pipeline.from_loader(name='run_py_local', externals={})
+pipe.execute(params={
+    'author-run': 'Local Workflow',
+    'run-date': datetime(2024, 1, 1),
+})
 ```
 
 ```shell
-> Hello Local Workflow
+> Hello at 20240101
 > Receive x from above with Local Workflow
 > Hello Caller
 > Hello World from Shell
 ```
 
-### Hook (Extract & Load)
-
-```yaml
-pipe_el_pg_to_lake:
-  type: pipeline.Pipeline
-  params:
-    run-date: datetime
-    author-email: str
-  jobs:
-    extract-load:
-      stages:
-        - name: "Extract Load from Postgres to Lake"
-          id: extract-load
-          uses: tasks/postgres-to-delta@polars
-          with:
-            source:
-              conn: conn_postgres_url
-              query: |
-                select * from ${{ params.name }}
-                where update_date = '${{ params.datetime }}'
-            sink:
-              conn: conn_az_lake
-              endpoint: "/${{ params.name }}"
-```
-
-Implement hook:
-
-```python
-from ddeutil.workflow.utils import tag
-
-@tag('polars', alias='postgres-to-delta')
-def postgres_to_delta(source, sink):
-    return {
-        "source": source, "sink": sink
-    }
-```
-
-### Hook (Transform)
-
-```yaml
-pipeline_hook_mssql_proc:
-  type: pipeline.Pipeline
-  params:
-    run_date: datetime
-    sp_name: str
-    source_name: str
-    target_name: str
-  jobs:
-    transform:
-      stages:
-        - name: "Transform Data in MS SQL Server"
-          id: transform
-          uses: tasks/mssql-proc@odbc
-          with:
-            exec: ${{ params.sp_name }}
-            params:
-              run_mode: "T"
-              run_date: ${{ params.run_date }}
-              source: ${{ params.source_name }}
-              target: ${{ params.target_name }}
-```
-
-Implement hook:
-
-```python
-from ddeutil.workflow.utils import tag
-
-@tag('odbc', alias='mssql-proc')
-def odbc_mssql_procedure(_exec: str, params: dict):
-    return {
-        "exec": _exec, "params": params
-    }
-```
-
 ## Configuration
 
 ```bash
@@ -273,7 +199,6 @@ export WORKFLOW_CORE_REGISTRY_FILTER=ddeutil.workflow.utils
 export WORKFLOW_CORE_PATH_CONF=conf
 export WORKFLOW_CORE_TIMEZONE=Asia/Bangkok
 export WORKFLOW_CORE_DEFAULT_STAGE_ID=true
-
 export WORKFLOW_CORE_MAX_PIPELINE_POKING=4
 export WORKFLOW_CORE_MAX_JOB_PARALLEL=2
 ```

{ddeutil_workflow-0.0.7 → ddeutil_workflow-0.0.8}/pyproject.toml

@@ -62,6 +62,10 @@ concurrency = ["thread", "multiprocessing"]
 source = ["ddeutil.workflow", "tests"]
 omit = [
     "scripts/",
+    "src/ddeutil/workflow/api.py",
+    "src/ddeutil/workflow/app.py",
+    "src/ddeutil/workflow/repeat.py",
+    "src/ddeutil/workflow/route.py",
     "tests/utils.py",
     "tests/tasks/dummy.py",
 ]
@@ -79,7 +83,7 @@ addopts = [
 filterwarnings = ["error"]
 log_cli = true
 log_cli_level = "DEBUG"
-log_cli_format = "%(asctime)s [%(levelname)-7s] %(message)-75s (%(filename)s:%(lineno)s)"
+log_cli_format = "%(asctime)s [%(levelname)-7s] %(message)-100s (%(filename)s:%(lineno)s)"
 log_cli_date_format = "%Y%m%d %H:%M:%S"
 
 [tool.black]

ddeutil_workflow-0.0.8/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.8"

{ddeutil_workflow-0.0.7 → ddeutil_workflow-0.0.8}/src/ddeutil/workflow/app.py

@@ -3,6 +3,8 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+from __future__ import annotations
+
 import functools
 import time
 
@@ -10,6 +12,8 @@ import schedule
 
 
 def catch_exceptions(cancel_on_failure=False):
+    """Catch exception error from scheduler job."""
+
     def catch_exceptions_decorator(job_func):
         @functools.wraps(job_func)
         def wrapper(*args, **kwargs):

{ddeutil_workflow-0.0.7 → ddeutil_workflow-0.0.8}/src/ddeutil/workflow/exceptions.py

@@ -3,9 +3,6 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""
-Define Errors Object for Node package
-"""
 from __future__ import annotations
 
 
@@ -24,4 +21,4 @@ class JobException(WorkflowException): ...
 class PipelineException(WorkflowException): ...
 
 
-class ParamValueException(ValueError): ...
+class ParamValueException(WorkflowException): ...

{ddeutil_workflow-0.0.7 → ddeutil_workflow-0.0.8}/src/ddeutil/workflow/log.py

@@ -6,11 +6,16 @@
 from __future__ import annotations
 
 import logging
+from datetime import datetime
 from functools import lru_cache
+from typing import Union
 
+from pydantic import BaseModel, Field
 from rich.console import Console
 from rich.logging import RichHandler
 
+from .__types import DictData
+
 console = Console(color_system="256", width=200, style="blue")
 
 
@@ -28,3 +33,47 @@ def get_logger(module_name):
     logger.addHandler(handler)
     logger.setLevel(logging.DEBUG)
     return logger
+
+
+class BaseLog(BaseModel):
+    """Base logging model."""
+
+    parent_id: str
+    id: str
+    input: DictData
+    output: DictData
+    update_time: datetime = Field(default_factory=datetime.now)
+
+
+class StageLog(BaseLog): ...
+
+
+class JobLog(BaseLog): ...
+
+
+class PipelineLog(BaseLog): ...
+
+
+Log = Union[
+    StageLog,
+    JobLog,
+    PipelineLog,
+]
+
+
+def push_log_memory(log: DictData):
+    """Push message log to globals log queue."""
+    print(log)
+
+
+LOGS_REGISTRY = {
+    "memory": push_log_memory,
+}
+
+
+def push_log(log: DictData, mode: str = "memory"):
+    return LOGS_REGISTRY[mode](log)
+
+
+def save_log():
+    """Save log that push to queue to target saving"""