ddeutil-workflow 0.0.33__tar.gz → 0.0.35__tar.gz

{ddeutil_workflow-0.0.33/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ddeutil-workflow
-Version: 0.0.33
+Version: 0.0.35
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -62,7 +62,7 @@ configuration. It called **Metadata Driven Data Workflow**.
 1. The Minimum frequency unit of scheduling is **1 minute** :warning:
 2. Can not re-run only failed stage and its pending downstream :rotating_light:
 3. All parallel tasks inside workflow engine use Multi-Threading
-   (Python 3.13 unlock GIL :unlock:)
+   (🐍 Python 3.13 unlock GIL :unlock:)
 
 ---
 
@@ -74,49 +74,52 @@ you should to set the data layer separate this core program before run this appl
 
 ```mermaid
 flowchart LR
-    subgraph Interface
-    A((User))
-        subgraph Docker Container
+    A((fa:fa-user User))
+
+    subgraph Docker Container
+        direction TB
         G@{ shape: rounded, label: "Observe<br>Application" }
-        end
     end
 
-    A --->|action| B(Workflow<br>Application)
-    B ---> |response| A
-    B -..-> |response| G
-    G -..-> |request| B
-
     subgraph Docker Container
-    B
+        direction TB
+        B@{ shape: rounded, label: "Workflow<br>Application" }
     end
 
+    A <-->|action &<br>response| B
+    B -...-> |response| G
+    G -...-> |request| B
+
     subgraph Data Context
-    D@{ shape: processes, label: "Logs" }
-    E@{ shape: lin-cyl, label: "Audit<br>Logs" }
+        D@{ shape: processes, label: "Logs" }
+        E@{ shape: lin-cyl, label: "Audit<br>Logs" }
     end
 
     subgraph Git Context
-    F@{ shape: tag-rect, label: "YAML<br>files" }
+        F@{ shape: tag-rect, label: "YAML<br>files" }
     end
 
-    B --->|disable| F
-    F --->|read| B
+    A ---> |push| H(Repo)
+    H -.-> |pull| F
+
+    B <-->|disable &<br>read| F
 
-    B --->|write| E
-    E --->|read| B
-    B --->|write| D
+    B <-->|read &<br>write| E
+
+    B -->|write| D
 
     D -.->|read| G
     E -.->|read| G
 ```
 
 > [!WARNING]
-> _Disclaimer_: I inspire the dynamic statement from the [**GitHub Action**](https://github.com/features/actions)
-> with `.yml` files and all configs file from several data orchestration framework
-> tools from my experience on Data Engineer. :grimacing:
->
-> Other workflow tools that I interest on them and pick some interested feature
-> implement to this package:
+> _**Disclaimer**_: I inspire the dynamic YAML statement from the [**GitHub Action**](https://github.com/features/actions),
+> and all configs pattern from several data orchestration framework tools from
+> my data engineering experience. :grimacing:
+
+> [!NOTE]
+> Other workflow orchestration tools that I interest and pick them to be inspiration
+> some for this package:
 >
 > - [Google **Workflows**](https://cloud.google.com/workflows)
 > - [AWS **Step Functions**](https://aws.amazon.com/step-functions/)
@@ -138,9 +141,9 @@ This is examples that use workflow file for running common Data Engineering
 use-case.
 
 > [!IMPORTANT]
-> I recommend you to use the `hook` stage for all actions that you want to do
+> I recommend you to use the `call` stage for all actions that you want to do
 > with workflow activity that you want to orchestrate. Because it is able to
-> dynamic an input argument with the same hook function that make you use less
+> dynamic an input argument with the same call function that make you use less
 > time to maintenance your data workflows.
 
 ```yaml
@@ -182,7 +185,7 @@ run-py-local:
                  writing_mode: flatten
                  aws_s3_path: my-data/open-data/${{ params.source-extract }}
 
-                 # This Authentication code should implement with your custom hook
+                 # This Authentication code should implement with your custom call
                  # function. The template allow you to use environment variable.
                  aws_access_client_id: ${AWS_ACCESS_CLIENT_ID}
                  aws_access_client_secret: ${AWS_ACCESS_CLIENT_SECRET}
@@ -247,7 +250,7 @@ it will use default value and do not raise any error to you.
 | Name                         | Component | Default                                                                                                                         | Description                                                                                                        |
 |:-----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------|
 | **ROOT_PATH**                |   Core    | `.`                                                                                                                             | The root path of the workflow application.                                                                         |
-| **REGISTRY**                 |   Core    | `.`                                                                                                                             | List of importable string for the hook stage.                                                                      |
+| **REGISTRY**                 |   Core    | `.`                                                                                                                             | List of importable string for the call stage.                                                                      |
 | **REGISTRY_FILTER**          |   Core    | `ddeutil.workflow.templates`                                                                                                    | List of importable string for the filter template.                                                                 |
 | **CONF_PATH**                |   Core    | `conf`                                                                                                                          | The config path that keep all template `.yaml` files.                                                              |
 | **TIMEZONE**                 |   Core    | `Asia/Bangkok`                                                                                                                  | A Timezone string value that will pass to `ZoneInfo` object.                                                       |
@@ -264,9 +267,10 @@ it will use default value and do not raise any error to you.
 | **PATH**                     |    Log    | `./logs`                                                                                                                        | The log path of the workflow saving log.                                                                           |
 | **DEBUG_MODE**               |    Log    | `true`                                                                                                                          | A flag that enable logging with debug level mode.                                                                  |
 | **FORMAT**                   |    Log    | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` |                                                                                                                    |
+| **FORMAT_FILE**              |    Log    | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})`                                                   |                                                                                                                    |
 | **DATETIME_FORMAT**          |    Log    | `%Y-%m-%d %H:%M:%S`                                                                                                             |                                                                                                                    |
-| **ENABLE_ROTATED_FILE**      |    Log    | `false`                                                                                                                         |                                                                                                                    |
-| **PATH**                     |   Audit   | `./logs`                                                                                                                        |                                                                                                                    |
+| **ENABLE_WRITE**             |    Log    | `false`                                                                                                                         |                                                                                                                    |
+| **PATH**                     |   Audit   | `./audits`                                                                                                                      |                                                                                                                    |
 | **ENABLE_WRITE**             |   Audit   | `true`                                                                                                                          | A flag that enable logging object saving log to its destination.                                                   |
 | **MAX_PROCESS**              |    App    | `2`                                                                                                                             | The maximum process worker number that run in scheduler app module.                                                |
 | **MAX_SCHEDULE_PER_PROCESS** |    App    | `100`                                                                                                                           | A schedule per process that run parallel.                                                                          |

{ddeutil_workflow-0.0.33 → ddeutil_workflow-0.0.35}/README.md

@@ -30,7 +30,7 @@ configuration. It called **Metadata Driven Data Workflow**.
 1. The Minimum frequency unit of scheduling is **1 minute** :warning:
 2. Can not re-run only failed stage and its pending downstream :rotating_light:
 3. All parallel tasks inside workflow engine use Multi-Threading
-   (Python 3.13 unlock GIL :unlock:)
+   (🐍 Python 3.13 unlock GIL :unlock:)
 
 ---
 
@@ -42,49 +42,52 @@ you should to set the data layer separate this core program before run this appl
 
 ```mermaid
 flowchart LR
-    subgraph Interface
-    A((User))
-        subgraph Docker Container
+    A((fa:fa-user User))
+
+    subgraph Docker Container
+        direction TB
         G@{ shape: rounded, label: "Observe<br>Application" }
-        end
     end
 
-    A --->|action| B(Workflow<br>Application)
-    B ---> |response| A
-    B -..-> |response| G
-    G -..-> |request| B
-
     subgraph Docker Container
-    B
+        direction TB
+        B@{ shape: rounded, label: "Workflow<br>Application" }
     end
 
+    A <-->|action &<br>response| B
+    B -...-> |response| G
+    G -...-> |request| B
+
     subgraph Data Context
-    D@{ shape: processes, label: "Logs" }
-    E@{ shape: lin-cyl, label: "Audit<br>Logs" }
+        D@{ shape: processes, label: "Logs" }
+        E@{ shape: lin-cyl, label: "Audit<br>Logs" }
     end
 
     subgraph Git Context
-    F@{ shape: tag-rect, label: "YAML<br>files" }
+        F@{ shape: tag-rect, label: "YAML<br>files" }
     end
 
-    B --->|disable| F
-    F --->|read| B
+    A ---> |push| H(Repo)
+    H -.-> |pull| F
+
+    B <-->|disable &<br>read| F
 
-    B --->|write| E
-    E --->|read| B
-    B --->|write| D
+    B <-->|read &<br>write| E
+
+    B -->|write| D
 
     D -.->|read| G
     E -.->|read| G
 ```
 
 > [!WARNING]
-> _Disclaimer_: I inspire the dynamic statement from the [**GitHub Action**](https://github.com/features/actions)
-> with `.yml` files and all configs file from several data orchestration framework
-> tools from my experience on Data Engineer. :grimacing:
->
-> Other workflow tools that I interest on them and pick some interested feature
-> implement to this package:
+> _**Disclaimer**_: I inspire the dynamic YAML statement from the [**GitHub Action**](https://github.com/features/actions),
+> and all configs pattern from several data orchestration framework tools from
+> my data engineering experience. :grimacing:
+
+> [!NOTE]
+> Other workflow orchestration tools that I interest and pick them to be inspiration
+> some for this package:
 >
 > - [Google **Workflows**](https://cloud.google.com/workflows)
 > - [AWS **Step Functions**](https://aws.amazon.com/step-functions/)
@@ -106,9 +109,9 @@ This is examples that use workflow file for running common Data Engineering
 use-case.
 
 > [!IMPORTANT]
-> I recommend you to use the `hook` stage for all actions that you want to do
+> I recommend you to use the `call` stage for all actions that you want to do
 > with workflow activity that you want to orchestrate. Because it is able to
-> dynamic an input argument with the same hook function that make you use less
+> dynamic an input argument with the same call function that make you use less
 > time to maintenance your data workflows.
 
 ```yaml
@@ -150,7 +153,7 @@ run-py-local:
                  writing_mode: flatten
                  aws_s3_path: my-data/open-data/${{ params.source-extract }}
 
-                 # This Authentication code should implement with your custom hook
+                 # This Authentication code should implement with your custom call
                  # function. The template allow you to use environment variable.
                  aws_access_client_id: ${AWS_ACCESS_CLIENT_ID}
                  aws_access_client_secret: ${AWS_ACCESS_CLIENT_SECRET}
@@ -215,7 +218,7 @@ it will use default value and do not raise any error to you.
 | Name                         | Component | Default                                                                                                                         | Description                                                                                                        |
 |:-----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------|
 | **ROOT_PATH**                |   Core    | `.`                                                                                                                             | The root path of the workflow application.                                                                         |
-| **REGISTRY**                 |   Core    | `.`                                                                                                                             | List of importable string for the hook stage.                                                                      |
+| **REGISTRY**                 |   Core    | `.`                                                                                                                             | List of importable string for the call stage.                                                                      |
 | **REGISTRY_FILTER**          |   Core    | `ddeutil.workflow.templates`                                                                                                    | List of importable string for the filter template.                                                                 |
 | **CONF_PATH**                |   Core    | `conf`                                                                                                                          | The config path that keep all template `.yaml` files.                                                              |
 | **TIMEZONE**                 |   Core    | `Asia/Bangkok`                                                                                                                  | A Timezone string value that will pass to `ZoneInfo` object.                                                       |
@@ -232,9 +235,10 @@ it will use default value and do not raise any error to you.
 | **PATH**                     |    Log    | `./logs`                                                                                                                        | The log path of the workflow saving log.                                                                           |
 | **DEBUG_MODE**               |    Log    | `true`                                                                                                                          | A flag that enable logging with debug level mode.                                                                  |
 | **FORMAT**                   |    Log    | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` |                                                                                                                    |
+| **FORMAT_FILE**              |    Log    | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})`                                                   |                                                                                                                    |
 | **DATETIME_FORMAT**          |    Log    | `%Y-%m-%d %H:%M:%S`                                                                                                             |                                                                                                                    |
-| **ENABLE_ROTATED_FILE**      |    Log    | `false`                                                                                                                         |                                                                                                                    |
-| **PATH**                     |   Audit   | `./logs`                                                                                                                        |                                                                                                                    |
+| **ENABLE_WRITE**             |    Log    | `false`                                                                                                                         |                                                                                                                    |
+| **PATH**                     |   Audit   | `./audits`                                                                                                                      |                                                                                                                    |
 | **ENABLE_WRITE**             |   Audit   | `true`                                                                                                                          | A flag that enable logging object saving log to its destination.                                                   |
 | **MAX_PROCESS**              |    App    | `2`                                                                                                                             | The maximum process worker number that run in scheduler app module.                                                |
 | **MAX_SCHEDULE_PER_PROCESS** |    App    | `100`                                                                                                                           | A schedule per process that run parallel.                                                                          |

ddeutil_workflow-0.0.35/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.35"

{ddeutil_workflow-0.0.33 → ddeutil_workflow-0.0.35}/src/ddeutil/workflow/__init__.py

@@ -9,6 +9,13 @@ from .audit import (
     Audit,
     get_audit,
 )
+from .caller import (
+    ReturnTagFunc,
+    TagFunc,
+    extract_call,
+    make_registry,
+    tag,
+)
 from .conf import (
     Config,
     Loader,
@@ -28,17 +35,15 @@ from .exceptions import (
     UtilException,
     WorkflowException,
 )
-from .hook import (
-    ReturnTagFunc,
-    TagFunc,
-    extract_hook,
-    make_registry,
-    tag,
-)
 from .job import (
     Job,
     Strategy,
 )
+from .logs import (
+    TraceLog,
+    get_dt_tznow,
+    get_trace,
+)
 from .params import (
     ChoiceParam,
     DatetimeParam,
@@ -46,7 +51,11 @@ from .params import (
     Param,
     StrParam,
 )
-from .result import Result
+from .result import (
+    Result,
+    Status,
+    default_gen_id,
+)
 from .scheduler import (
     Schedule,
     ScheduleWorkflow,
@@ -54,10 +63,10 @@ from .scheduler import (
     schedule_runner,
     schedule_task,
 )
-from .stage import (
+from .stages import (
     BashStage,
+    CallStage,
     EmptyStage,
-    HookStage,
     PyStage,
     Stage,
     TriggerStage,

{ddeutil_workflow-0.0.33 → ddeutil_workflow-0.0.35}/src/ddeutil/workflow/api/api.py

@@ -20,6 +20,7 @@ from ..conf import config, get_logger
 from ..scheduler import ReleaseThread, ReleaseThreads
 from ..workflow import ReleaseQueue, WorkflowTask
 from .repeat import repeat_at
+from .routes import log
 
 load_dotenv()
 logger = get_logger("ddeutil.workflow")
@@ -77,22 +78,26 @@ async def health():
     return {"message": "Workflow API already start up"}
 
 
-# NOTE: Enable the workflow route.
+# NOTE Add the logs route by default.
+app.include_router(log, prefix=config.prefix_path)
+
+
+# NOTE: Enable the workflows route.
 if config.enable_route_workflow:
-    from .route import workflow_route
+    from .routes import workflow
 
-    app.include_router(workflow_route, prefix=config.prefix_path)
+    app.include_router(workflow, prefix=config.prefix_path)
 
 
-# NOTE: Enable the schedule route.
+# NOTE: Enable the schedules route.
 if config.enable_route_schedule:
     from ..audit import get_audit
     from ..scheduler import schedule_task
-    from .route import schedule_route
+    from .routes import schedule
 
-    app.include_router(schedule_route, prefix=config.prefix_path)
+    app.include_router(schedule, prefix=config.prefix_path)
 
-    @schedule_route.on_event("startup")
+    @schedule.on_event("startup")
     @repeat_at(cron="* * * * *", delay=2)
     def scheduler_listener():
         """Schedule broker every minute at 02 second."""
@@ -109,7 +114,7 @@ if config.enable_route_schedule:
                 log=get_audit(),
             )
 
-    @schedule_route.on_event("startup")
+    @schedule.on_event("startup")
     @repeat_at(cron="*/5 * * * *", delay=10)
     def monitoring():
         logger.debug("[MONITOR]: Start monitoring threading.")

ddeutil_workflow-0.0.35/src/ddeutil/workflow/api/routes/__init__.py

@@ -0,0 +1,8 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from .logs import log_route as log
+from .schedules import schedule_route as schedule
+from .workflows import workflow_route as workflow

ddeutil_workflow-0.0.35/src/ddeutil/workflow/api/routes/logs.py

@@ -0,0 +1,36 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+from fastapi import APIRouter
+from fastapi.responses import UJSONResponse
+
+from ...conf import get_logger
+from ...logs import get_trace_obj
+
+logger = get_logger("ddeutil.workflow")
+
+
+# NOTE: Start create the schedule routes.
+#
+log_route = APIRouter(
+    prefix="/logs",
+    tags=["logs"],
+    default_response_class=UJSONResponse,
+)
+
+
+@log_route.get(path="/")
+async def get_logs():
+    return {
+        "message": "Getting logs",
+        "audits": list(get_trace_obj().find_logs()),
+    }
+
+
+@log_route.get(path="/{run_id}")
+async def get_log_with_run_id(run_id: str):
+    return get_trace_obj().find_log_with_id(run_id)

ddeutil_workflow-0.0.33/src/ddeutil/workflow/api/route.py → ddeutil_workflow-0.0.35/src/ddeutil/workflow/api/routes/schedules.py

@@ -6,30 +6,17 @@
 from __future__ import annotations
 
 import copy
-from dataclasses import asdict
 from datetime import datetime, timedelta
-from typing import Any
 
 from fastapi import APIRouter, HTTPException, Request
 from fastapi import status as st
 from fastapi.responses import UJSONResponse
-from pydantic import BaseModel
 
-from ..__types import DictData
-from ..audit import Audit, get_audit
-from ..conf import Loader, config, get_logger
-from ..result import Result
-from ..scheduler import Schedule
-from ..workflow import Workflow
+from ...conf import config, get_logger
+from ...scheduler import Schedule
 
 logger = get_logger("ddeutil.workflow")
 
-workflow_route = APIRouter(
-    prefix="/workflows",
-    tags=["workflows"],
-    default_response_class=UJSONResponse,
-)
-
 schedule_route = APIRouter(
     prefix="/schedules",
     tags=["schedules"],
@@ -37,122 +24,6 @@ schedule_route = APIRouter(
 )
 
 
-@workflow_route.get(path="/")
-async def get_workflows() -> DictData:
-    """Return all workflow workflows that exists in config path."""
-    workflows: DictData = dict(Loader.finds(Workflow))
-    return {
-        "message": f"Getting all workflows: {len(workflows)}",
-        "count": len(workflows),
-        "workflows": workflows,
-    }
-
-
-@workflow_route.get(path="/{name}")
-async def get_workflow_by_name(name: str) -> DictData:
-    """Return model of workflow that passing an input workflow name."""
-    try:
-        workflow: Workflow = Workflow.from_loader(name=name, externals={})
-    except ValueError as err:
-        logger.exception(err)
-        raise HTTPException(
-            status_code=st.HTTP_404_NOT_FOUND,
-            detail=(
-                f"Workflow workflow name: {name!r} does not found in /conf path"
-            ),
-        ) from None
-    return workflow.model_dump(
-        by_alias=True,
-        exclude_none=True,
-        exclude_unset=True,
-        exclude_defaults=True,
-    )
-
-
-class ExecutePayload(BaseModel):
-    params: dict[str, Any]
-
-
-@workflow_route.post(path="/{name}/execute", status_code=st.HTTP_202_ACCEPTED)
-async def execute_workflow(name: str, payload: ExecutePayload) -> DictData:
-    """Return model of workflow that passing an input workflow name."""
-    try:
-        workflow: Workflow = Workflow.from_loader(name=name, externals={})
-    except ValueError:
-        raise HTTPException(
-            status_code=st.HTTP_404_NOT_FOUND,
-            detail=(
-                f"Workflow workflow name: {name!r} does not found in /conf path"
-            ),
-        ) from None
-
-    # NOTE: Start execute manually
-    try:
-        result: Result = workflow.execute(params=payload.params)
-    except Exception as err:
-        raise HTTPException(
-            status_code=st.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"{type(err)}: {err}",
-        ) from None
-
-    return asdict(result)
-
-
-@workflow_route.get(path="/{name}/logs")
-async def get_workflow_logs(name: str):
-    try:
-        return {
-            "message": f"Getting workflow {name!r} logs",
-            "logs": [
-                log.model_dump(
-                    by_alias=True,
-                    exclude_none=True,
-                    exclude_unset=True,
-                    exclude_defaults=True,
-                )
-                for log in get_audit().find_logs(name=name)
-            ],
-        }
-    except FileNotFoundError:
-        raise HTTPException(
-            status_code=st.HTTP_404_NOT_FOUND,
-            detail=f"Does not found log for workflow {name!r}",
-        ) from None
-
-
-@workflow_route.get(path="/{name}/logs/{release}")
-async def get_workflow_release_log(name: str, release: str):
-    try:
-        log: Audit = get_audit().find_log_with_release(
-            name=name, release=datetime.strptime(release, "%Y%m%d%H%M%S")
-        )
-    except FileNotFoundError:
-        raise HTTPException(
-            status_code=st.HTTP_404_NOT_FOUND,
-            detail=(
-                f"Does not found log for workflow {name!r} "
-                f"with release {release!r}"
-            ),
-        ) from None
-    return {
-        "message": f"Getting workflow {name!r} log in release {release}",
-        "log": log.model_dump(
-            by_alias=True,
-            exclude_none=True,
-            exclude_unset=True,
-            exclude_defaults=True,
-        ),
-    }
-
-
-@workflow_route.delete(
-    path="/{name}/logs/{release}",
-    status_code=st.HTTP_204_NO_CONTENT,
-)
-async def del_workflow_release_log(name: str, release: str):
-    return {"message": f"Deleted workflow {name!r} log in release {release}"}
-
-
 @schedule_route.get(path="/{name}")
 async def get_schedules(name: str):
     try: