ddeutil-workflow 0.0.10__tar.gz → 0.0.11__tar.gz

{ddeutil_workflow-0.0.10 → ddeutil_workflow-0.0.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.10
+Version: 0.0.11
 Summary: Lightweight workflow orchestration with less dependencies
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -24,10 +24,9 @@ License-File: LICENSE
 Requires-Dist: ddeutil-io
 Requires-Dist: python-dotenv==1.0.1
 Requires-Dist: typer<1.0.0,==0.12.5
-Provides-Extra: schedule
-Requires-Dist: schedule<2.0.0,==1.2.2; extra == "schedule"
+Requires-Dist: schedule<2.0.0,==1.2.2
 Provides-Extra: api
-Requires-Dist: fastapi[standard]<1.0.0,==0.112.2; extra == "api"
+Requires-Dist: fastapi<1.0.0,==0.112.2; extra == "api"
 
 # Workflow
 
@@ -38,22 +37,22 @@ Requires-Dist: fastapi[standard]<1.0.0,==0.112.2; extra == "api"
 [![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 
 The **Lightweight workflow orchestration** with less dependencies the was created
-for easy to make a simple metadata driven for data pipeline orchestration.
+for easy to make a simple metadata driven for data workflow orchestration.
 It can to use for data operator by a `.yaml` template.
 
 > [!WARNING]
 > This package provide only orchestration workload. That mean you should not use
 > workflow stage to process any large data which use lot of compute usecase.
 
-In my opinion, I think it should not create duplicate pipeline codes if I can
-write with dynamic input parameters on the one template pipeline that just change
+In my opinion, I think it should not create duplicate workflow codes if I can
+write with dynamic input parameters on the one template workflow that just change
 the input parameters per use-case instead.
-This way I can handle a lot of logical pipelines in our orgs with only metadata
-configuration. It called **Metadata Driven Data Pipeline**.
+This way I can handle a lot of logical workflows in our orgs with only metadata
+configuration. It called **Metadata Driven Data Workflow**.
 
 Next, we should get some monitoring tools for manage logging that return from
-pipeline running. Because it not show us what is a use-case that running data
-pipeline.
+workflow running. Because it not show us what is a use-case that running data
+workflow.
 
 > [!NOTE]
 > _Disclaimer_: I inspire the dynamic statement from the GitHub Action `.yml` files
@@ -75,8 +74,7 @@ this package with application add-ons, you should add `app` in installation;
 | Usecase           | Install Optional                         | Support            |
 |-------------------|------------------------------------------|--------------------|
 | Python & CLI      | `pip install ddeutil-workflow`           | :heavy_check_mark: |
-| Scheduler Service | `pip install ddeutil-workflow[schedule]` | :x:                |
-| FastAPI Server    | `pip install ddeutil-workflow[api]`      | :x:                |
+| FastAPI Server    | `pip install ddeutil-workflow[api]`      | :heavy_check_mark: |
 
 
 > I added this feature to the main milestone.
@@ -97,67 +95,65 @@ use-case.
 
 > [!IMPORTANT]
 > I recommend you to use the `hook` stage for all actions that you want to do
-> with pipeline activity that you want to orchestrate. Because it able to dynamic
+> with workflow activity that you want to orchestrate. Because it able to dynamic
 > an input argument with the same hook function that make you use less time to
-> maintenance your data pipelines.
+> maintenance your data workflows.
 
 ```yaml
 run_py_local:
-  type: pipeline.Pipeline
-  on:
-    - cronjob: '*/5 * * * *'
-      timezone: "Asia/Bangkok"
-  params:
-    author-run: str
-    run-date: datetime
-  jobs:
-    getting-api-data:
-      stages:
-        - name: "Retrieve API Data"
-          id: retrieve-api
-          uses: tasks/get-api-with-oauth-to-s3@requests
-          with:
-            url: https://open-data/
-            auth: ${API_ACCESS_REFRESH_TOKEN}
-            aws_s3_path: my-data/open-data/
-
-            # This Authentication code should implement with your custom hook function.
-            # The template allow you to use environment variable.
-            aws_access_client_id: ${AWS_ACCESS_CLIENT_ID}
-            aws_access_client_secret: ${AWS_ACCESS_CLIENT_SECRET}
+   type: Workflow
+   on:
+      # If workflow deploy to schedule, it will running every 5 minutes
+      # with Asia/Bangkok timezone.
+      - cronjob: '*/5 * * * *'
+        timezone: "Asia/Bangkok"
+   params:
+      # Incoming execution parameters will validate with this type. It allow
+      # to set default value or templating.
+      author-run: str
+      run-date: datetime
+   jobs:
+      getting-api-data:
+         stages:
+            - name: "Retrieve API Data"
+              id: retrieve-api
+              uses: tasks/get-api-with-oauth-to-s3@requests
+              with:
+                 url: https://open-data/
+                 auth: ${API_ACCESS_REFRESH_TOKEN}
+                 aws_s3_path: my-data/open-data/
+
+                 # This Authentication code should implement with your custom hook function.
+                 # The template allow you to use environment variable.
+                 aws_access_client_id: ${AWS_ACCESS_CLIENT_ID}
+                 aws_access_client_secret: ${AWS_ACCESS_CLIENT_SECRET}
 ```
 
 ## Configuration
 
-| Environment                         | Component | Default                      | Description                                                                |
-|-------------------------------------|-----------|------------------------------|----------------------------------------------------------------------------|
-| `WORKFLOW_ROOT_PATH`                | Core      | .                            | The root path of the workflow application                                  |
-| `WORKFLOW_CORE_REGISTRY`            | Core      | ddeutil.workflow,tests.utils | List of importable string for the hook stage                               |
-| `WORKFLOW_CORE_REGISTRY_FILTER`     | Core      | ddeutil.workflow.utils       | List of importable string for the filter template                          |
-| `WORKFLOW_CORE_PATH_CONF`           | Core      | conf                         | The config path that keep all template `.yaml` files                       |
-| `WORKFLOW_CORE_TIMEZONE`            | Core      | Asia/Bangkok                 | A Timezone string value that will pass to `ZoneInfo` object                |
-| `WORKFLOW_CORE_STAGE_DEFAULT_ID`    | Core      | true                         | A flag that enable default stage ID that use for catch an execution output |
-| `WORKFLOW_CORE_STAGE_RAISE_ERROR`   | Core      | true                         | A flag that all stage raise StageException from stage execution            |
-| `WORKFLOW_CORE_MAX_PIPELINE_POKING` | Core      | 4                            |                                                                            |
-| `WORKFLOW_CORE_MAX_JOB_PARALLEL`    | Core      | 2                            | The maximum job number that able to run parallel in pipeline executor      |
-| `WORKFLOW_LOG_DEBUG_MODE`           | Log       | true                         | A flag that enable logging with debug level mode                           |
-| `WORKFLOW_LOG_ENABLE_WRITE`         | Log       | true                         | A flag that enable logging object saving log to its destination            |
-
-
-**Application**:
-
-| Environment                         | Default                          | Description                                                             |
-|-------------------------------------|----------------------------------|-------------------------------------------------------------------------|
-| `WORKFLOW_APP_PROCESS_WORKER`       | 2                                | The maximum process worker number that run in scheduler app module      |
-| `WORKFLOW_APP_SCHEDULE_PER_PROCESS` | 100                              | A schedule per process that run parallel                                |
-| `WORKFLOW_APP_STOP_BOUNDARY_DELTA`  | '{"minutes": 5, "seconds": 20}'  | A time delta value that use to stop scheduler app in json string format |
-
-**API server**:
-
-| Environment                          | Default | Description                                                                       |
-|--------------------------------------|---------|-----------------------------------------------------------------------------------|
-| `WORKFLOW_API_ENABLE_ROUTE_WORKFLOW` | true    | A flag that enable workflow route to manage execute manually and workflow logging |
-| `WORKFLOW_API_ENABLE_ROUTE_SCHEDULE` | true    | A flag that enable run scheduler                                                  |
+| Environment                         | Component | Default                          | Description                                                                |
+|-------------------------------------|-----------|----------------------------------|----------------------------------------------------------------------------|
+| `WORKFLOW_ROOT_PATH`                | Core      | .                                | The root path of the workflow application                                  |
+| `WORKFLOW_CORE_REGISTRY`            | Core      | src.ddeutil.workflow,tests.utils | List of importable string for the hook stage                               |
+| `WORKFLOW_CORE_REGISTRY_FILTER`     | Core      | ddeutil.workflow.utils           | List of importable string for the filter template                          |
+| `WORKFLOW_CORE_PATH_CONF`           | Core      | conf                             | The config path that keep all template `.yaml` files                       |
+| `WORKFLOW_CORE_TIMEZONE`            | Core      | Asia/Bangkok                     | A Timezone string value that will pass to `ZoneInfo` object                |
+| `WORKFLOW_CORE_STAGE_DEFAULT_ID`    | Core      | true                             | A flag that enable default stage ID that use for catch an execution output |
+| `WORKFLOW_CORE_STAGE_RAISE_ERROR`   | Core      | true                             | A flag that all stage raise StageException from stage execution            |
+| `WORKFLOW_CORE_MAX_NUM_POKING`      | Core      | 4                                |                                                                            |
+| `WORKFLOW_CORE_MAX_JOB_PARALLEL`    | Core      | 2                                | The maximum job number that able to run parallel in workflow executor      |
+| `WORKFLOW_LOG_DEBUG_MODE`           | Log       | true                             | A flag that enable logging with debug level mode                           |
+| `WORKFLOW_LOG_ENABLE_WRITE`         | Log       | true                             | A flag that enable logging object saving log to its destination            |
+| `WORKFLOW_APP_PROCESS_WORKER`       | Schedule  | 2                                | The maximum process worker number that run in scheduler app module         |
+| `WORKFLOW_APP_SCHEDULE_PER_PROCESS` | Schedule  | 100                              | A schedule per process that run parallel                                   |
+| `WORKFLOW_APP_STOP_BOUNDARY_DELTA`  | Schedule  | '{"minutes": 5, "seconds": 20}'  | A time delta value that use to stop scheduler app in json string format    |
+
+**API Application**:
+
+| Environment                          | Component | Default | Description                                                                       |
+|--------------------------------------|-----------|---------|-----------------------------------------------------------------------------------|
+| `WORKFLOW_API_ENABLE_ROUTE_WORKFLOW` | API       | true    | A flag that enable workflow route to manage execute manually and workflow logging |
+| `WORKFLOW_API_ENABLE_ROUTE_SCHEDULE` | API       | true    | A flag that enable run scheduler                                                  |
 
 ## Deployment
 
@@ -165,18 +161,18 @@ This package able to run as a application service for receive manual trigger
 from the master node via RestAPI or use to be Scheduler background service
 like crontab job but via Python API.
 
-### Schedule Service
+### Schedule App
 
 ```shell
-(venv) $ python src.ddeutil.workflow.app
+(venv) $ ddeutil-workflow schedule
 ```
 
 ### API Server
 
 ```shell
-(venv) $ uvicorn src.ddeutil.workflow.api:app --host 0.0.0.0 --port 80 --reload
+(venv) $ uvicorn src.ddeutil.workflow.api:app --host 127.0.0.1 --port 80
 ```
 
 > [!NOTE]
 > If this package already deploy, it able to use
-> `uvicorn ddeutil.workflow.api:app --host 0.0.0.0 --port 80 --workers 4`
+> `uvicorn ddeutil.workflow.api:app --host 127.0.0.1 --port 80 --workers 4`

ddeutil_workflow-0.0.11/README.md

@@ -0,0 +1,148 @@
+# Workflow
+
+[![test](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/ddeutils/ddeutil-workflow/actions/workflows/tests.yml)
+[![python support version](https://img.shields.io/pypi/pyversions/ddeutil-workflow)](https://pypi.org/project/ddeutil-workflow/)
+[![size](https://img.shields.io/github/languages/code-size/ddeutils/ddeutil-workflow)](https://github.com/ddeutils/ddeutil-workflow)
+[![gh license](https://img.shields.io/github/license/ddeutils/ddeutil-workflow)](https://github.com/ddeutils/ddeutil-workflow/blob/main/LICENSE)
+[![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+
+The **Lightweight workflow orchestration** with less dependencies the was created
+for easy to make a simple metadata driven for data workflow orchestration.
+It can to use for data operator by a `.yaml` template.
+
+> [!WARNING]
+> This package provide only orchestration workload. That mean you should not use
+> workflow stage to process any large data which use lot of compute usecase.
+
+In my opinion, I think it should not create duplicate workflow codes if I can
+write with dynamic input parameters on the one template workflow that just change
+the input parameters per use-case instead.
+This way I can handle a lot of logical workflows in our orgs with only metadata
+configuration. It called **Metadata Driven Data Workflow**.
+
+Next, we should get some monitoring tools for manage logging that return from
+workflow running. Because it not show us what is a use-case that running data
+workflow.
+
+> [!NOTE]
+> _Disclaimer_: I inspire the dynamic statement from the GitHub Action `.yml` files
+> and all of config file from several data orchestration framework tools from my
+> experience on Data Engineer.
+
+**Rules of This Workflow engine**:
+
+1. Minimum unit of scheduling is 1 minute
+2. Cannot re-run only failed stage and its pending downstream
+3. All parallel tasks inside workflow engine use Threading
+   (Because Python 3.13 unlock GIL)
+
+## Installation
+
+This project need `ddeutil-io` extension namespace packages. If you want to install
+this package with application add-ons, you should add `app` in installation;
+
+| Usecase           | Install Optional                         | Support            |
+|-------------------|------------------------------------------|--------------------|
+| Python & CLI      | `pip install ddeutil-workflow`           | :heavy_check_mark: |
+| FastAPI Server    | `pip install ddeutil-workflow[api]`      | :heavy_check_mark: |
+
+
+> I added this feature to the main milestone.
+>
+> **Docker Images** supported:
+>
+> | Docker Image                | Python Version | Support |
+> |-----------------------------|----------------|---------|
+> | ddeutil-workflow:latest     | `3.9`          | :x:     |
+> | ddeutil-workflow:python3.10 | `3.10`         | :x:     |
+> | ddeutil-workflow:python3.11 | `3.11`         | :x:     |
+> | ddeutil-workflow:python3.12 | `3.12`         | :x:     |
+
+## Usage
+
+This is examples that use workflow file for running common Data Engineering
+use-case.
+
+> [!IMPORTANT]
+> I recommend you to use the `hook` stage for all actions that you want to do
+> with workflow activity that you want to orchestrate. Because it able to dynamic
+> an input argument with the same hook function that make you use less time to
+> maintenance your data workflows.
+
+```yaml
+run_py_local:
+   type: Workflow
+   on:
+      # If workflow deploy to schedule, it will running every 5 minutes
+      # with Asia/Bangkok timezone.
+      - cronjob: '*/5 * * * *'
+        timezone: "Asia/Bangkok"
+   params:
+      # Incoming execution parameters will validate with this type. It allow
+      # to set default value or templating.
+      author-run: str
+      run-date: datetime
+   jobs:
+      getting-api-data:
+         stages:
+            - name: "Retrieve API Data"
+              id: retrieve-api
+              uses: tasks/get-api-with-oauth-to-s3@requests
+              with:
+                 url: https://open-data/
+                 auth: ${API_ACCESS_REFRESH_TOKEN}
+                 aws_s3_path: my-data/open-data/
+
+                 # This Authentication code should implement with your custom hook function.
+                 # The template allow you to use environment variable.
+                 aws_access_client_id: ${AWS_ACCESS_CLIENT_ID}
+                 aws_access_client_secret: ${AWS_ACCESS_CLIENT_SECRET}
+```
+
+## Configuration
+
+| Environment                         | Component | Default                          | Description                                                                |
+|-------------------------------------|-----------|----------------------------------|----------------------------------------------------------------------------|
+| `WORKFLOW_ROOT_PATH`                | Core      | .                                | The root path of the workflow application                                  |
+| `WORKFLOW_CORE_REGISTRY`            | Core      | src.ddeutil.workflow,tests.utils | List of importable string for the hook stage                               |
+| `WORKFLOW_CORE_REGISTRY_FILTER`     | Core      | ddeutil.workflow.utils           | List of importable string for the filter template                          |
+| `WORKFLOW_CORE_PATH_CONF`           | Core      | conf                             | The config path that keep all template `.yaml` files                       |
+| `WORKFLOW_CORE_TIMEZONE`            | Core      | Asia/Bangkok                     | A Timezone string value that will pass to `ZoneInfo` object                |
+| `WORKFLOW_CORE_STAGE_DEFAULT_ID`    | Core      | true                             | A flag that enable default stage ID that use for catch an execution output |
+| `WORKFLOW_CORE_STAGE_RAISE_ERROR`   | Core      | true                             | A flag that all stage raise StageException from stage execution            |
+| `WORKFLOW_CORE_MAX_NUM_POKING`      | Core      | 4                                |                                                                            |
+| `WORKFLOW_CORE_MAX_JOB_PARALLEL`    | Core      | 2                                | The maximum job number that able to run parallel in workflow executor      |
+| `WORKFLOW_LOG_DEBUG_MODE`           | Log       | true                             | A flag that enable logging with debug level mode                           |
+| `WORKFLOW_LOG_ENABLE_WRITE`         | Log       | true                             | A flag that enable logging object saving log to its destination            |
+| `WORKFLOW_APP_PROCESS_WORKER`       | Schedule  | 2                                | The maximum process worker number that run in scheduler app module         |
+| `WORKFLOW_APP_SCHEDULE_PER_PROCESS` | Schedule  | 100                              | A schedule per process that run parallel                                   |
+| `WORKFLOW_APP_STOP_BOUNDARY_DELTA`  | Schedule  | '{"minutes": 5, "seconds": 20}'  | A time delta value that use to stop scheduler app in json string format    |
+
+**API Application**:
+
+| Environment                          | Component | Default | Description                                                                       |
+|--------------------------------------|-----------|---------|-----------------------------------------------------------------------------------|
+| `WORKFLOW_API_ENABLE_ROUTE_WORKFLOW` | API       | true    | A flag that enable workflow route to manage execute manually and workflow logging |
+| `WORKFLOW_API_ENABLE_ROUTE_SCHEDULE` | API       | true    | A flag that enable run scheduler                                                  |
+
+## Deployment
+
+This package able to run as a application service for receive manual trigger
+from the master node via RestAPI or use to be Scheduler background service
+like crontab job but via Python API.
+
+### Schedule App
+
+```shell
+(venv) $ ddeutil-workflow schedule
+```
+
+### API Server
+
+```shell
+(venv) $ uvicorn src.ddeutil.workflow.api:app --host 127.0.0.1 --port 80
+```
+
+> [!NOTE]
+> If this package already deploy, it able to use
+> `uvicorn ddeutil.workflow.api:app --host 127.0.0.1 --port 80 --workers 4`

{ddeutil_workflow-0.0.10 → ddeutil_workflow-0.0.11}/pyproject.toml

@@ -28,15 +28,13 @@ dependencies = [
     "ddeutil-io",
     "python-dotenv==1.0.1",
     "typer==0.12.5,<1.0.0",
+    "schedule==1.2.2,<2.0.0",
 ]
 dynamic = ["version"]
 
 [project.optional-dependencies]
-schedule = [
-    "schedule==1.2.2,<2.0.0",
-]
 api = [
-    "fastapi[standard]==0.112.2,<1.0.0",
+    "fastapi==0.112.2,<1.0.0",
 ]
 
 [project.urls]

ddeutil_workflow-0.0.11/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.11"

{ddeutil_workflow-0.0.10 → ddeutil_workflow-0.0.11}/src/ddeutil/workflow/__init__.py

@@ -6,12 +6,13 @@
 from .exceptions import (
     JobException,
     ParamValueException,
-    PipelineException,
     StageException,
     UtilException,
+    WorkflowException,
 )
+from .job import Job, Strategy
 from .on import On, interval2crontab
-from .pipeline import Job, Pipeline, Strategy
+from .scheduler import Workflow
 from .stage import Stage, handler_result
 from .utils import (
     Param,

ddeutil_workflow-0.0.11/src/ddeutil/workflow/api.py

@@ -0,0 +1,157 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import os
+import uuid
+from collections.abc import AsyncIterator
+from datetime import datetime, timedelta
+from queue import Empty, Queue
+from threading import Thread
+from typing import TypedDict
+
+from ddeutil.core import str2bool
+from dotenv import load_dotenv
+from fastapi import FastAPI
+from fastapi.middleware.gzip import GZipMiddleware
+from fastapi.responses import UJSONResponse
+from pydantic import BaseModel
+
+from .__about__ import __version__
+from .log import get_logger
+from .repeat import repeat_at, repeat_every
+from .scheduler import WorkflowTask
+
+load_dotenv()
+logger = get_logger("ddeutil.workflow")
+
+
+class State(TypedDict):
+    upper_queue: Queue
+    upper_result: dict[str, str]
+    scheduler: list[str]
+    workflow_threads: dict[str, Thread]
+    workflow_tasks: list[WorkflowTask]
+    workflow_queue: dict[str, list[datetime]]
+    workflow_running: dict[str, list[datetime]]
+
+
+@contextlib.asynccontextmanager
+async def lifespan(a: FastAPI) -> AsyncIterator[State]:
+    a.state.upper_queue = Queue()
+    a.state.upper_result = {}
+    a.state.scheduler = []
+    a.state.workflow_threads = {}
+    a.state.workflow_tasks = []
+    a.state.workflow_queue = {}
+    a.state.workflow_running = {}
+
+    await asyncio.create_task(broker_upper_messages())
+
+    yield {
+        "upper_queue": a.state.upper_queue,
+        "upper_result": a.state.upper_result,
+        # NOTE: Scheduler value should be contain a key of workflow workflow and
+        #   list of datetime of queue and running.
+        #
+        #   ... {
+        #   ...     '<workflow-name>': (
+        #   ...         [<running-datetime>, ...], [<queue-datetime>, ...]
+        #   ...     )
+        #   ... }
+        #
+        "scheduler": a.state.scheduler,
+        "workflow_queue": a.state.workflow_queue,
+        "workflow_running": a.state.workflow_running,
+        "workflow_threads": a.state.workflow_threads,
+        "workflow_tasks": a.state.workflow_tasks,
+    }
+
+
+app = FastAPI(
+    titile="Workflow API",
+    description=(
+        "This is workflow FastAPI web application that use to manage manual "
+        "execute or schedule workflow via RestAPI."
+    ),
+    version=__version__,
+    lifespan=lifespan,
+    default_response_class=UJSONResponse,
+)
+app.add_middleware(GZipMiddleware, minimum_size=1000)
+
+
+@repeat_every(seconds=10)
+async def broker_upper_messages():
+    """Broker for receive message from the `/upper` path and change it to upper
+    case. This broker use interval running in background every 10 seconds.
+    """
+    for _ in range(10):
+        try:
+            obj = app.state.upper_queue.get_nowait()
+            app.state.upper_result[obj["request_id"]] = obj["text"].upper()
+            logger.info(f"Upper message: {app.state.upper_result}")
+        except Empty:
+            pass
+    await asyncio.sleep(0.0001)
+
+
+class Payload(BaseModel):
+    text: str
+
+
+async def get_result(request_id: str) -> dict[str, str]:
+    """Get data from output dict that global."""
+    while True:
+        if request_id in app.state.upper_result:
+            result: str = app.state.upper_result[request_id]
+            del app.state.upper_result[request_id]
+            return {"message": result}
+        await asyncio.sleep(0.0025)
+
+
+@app.get("/")
+@app.get("/api")
+async def health():
+    return {"message": "Workflow API already start up"}
+
+
+@app.post("/api")
+async def message_upper(payload: Payload):
+    """Convert message from any case to the upper case."""
+    request_id: str = str(uuid.uuid4())
+    app.state.upper_queue.put(
+        {"text": payload.text, "request_id": request_id},
+    )
+    return await get_result(request_id)
+
+
+if str2bool(os.getenv("WORKFLOW_API_ENABLE_ROUTE_WORKFLOW", "true")):
+    from .route import workflow
+
+    app.include_router(workflow)
+
+if str2bool(os.getenv("WORKFLOW_API_ENABLE_ROUTE_SCHEDULE", "true")):
+    from .route import schedule
+    from .scheduler import workflow_task
+
+    app.include_router(schedule)
+
+    @schedule.on_event("startup")
+    @repeat_at(cron="* * * * *", delay=2)
+    def schedule_broker_up():
+        logger.debug(
+            f"[SCHEDULER]: Start listening schedule from queue "
+            f"{app.state.scheduler}"
+        )
+        if app.state.workflow_tasks:
+            workflow_task(
+                app.state.workflow_tasks,
+                stop=datetime.now() + timedelta(minutes=1),
+                threads=app.state.workflow_threads,
+            )

{ddeutil_workflow-0.0.10 → ddeutil_workflow-0.0.11}/src/ddeutil/workflow/cli.py

@@ -29,21 +29,21 @@ cli.add_typer(
 
 @cli.command()
 def run(
-    pipeline: Annotated[
+    workflow: Annotated[
         str,
-        Argument(help="A pipeline name that want to run manually"),
+        Argument(help="A workflow name that want to run manually"),
     ],
     params: Annotated[
         str,
         Argument(
-            help="A json string for parameters of this pipeline execution."
+            help="A json string for parameters of this workflow execution."
         ),
     ],
 ):
-    """Run pipeline workflow manually with an input custom parameters that able
-    to receive with pipeline params config.
+    """Run workflow workflow manually with an input custom parameters that able
+    to receive with workflow params config.
     """
-    logger.info(f"Running pipeline name: {pipeline}")
+    logger.info(f"Running workflow name: {workflow}")
     logger.info(f"... with Parameters: {json.dumps(json.loads(params))}")
 
 
@@ -63,7 +63,7 @@ def schedule(
     externals: Annotated[
         Optional[str],
         Argument(
-            help="A json string for parameters of this pipeline execution."
+            help="A json string for parameters of this workflow execution."
         ),
     ] = None,
 ):
@@ -77,20 +77,20 @@ def schedule(
             tz=ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
         )
 
-    from .scheduler import workflow
+    from .scheduler import workflow_runner
 
     # NOTE: Start running workflow scheduler application.
-    workflow_rs: list[str] = workflow(
+    workflow_rs: list[str] = workflow_runner(
         stop=stop, excluded=excluded, externals=json.loads(externals)
     )
     logger.info(f"Application run success: {workflow_rs}")
 
 
-@cli_log.command("pipeline-get")
-def pipeline_log_get(
+@cli_log.command("workflow-get")
+def workflow_log_get(
     name: Annotated[
         str,
-        Argument(help="A pipeline name that want to getting log"),
+        Argument(help="A workflow name that want to getting log"),
     ],
     limit: Annotated[
         int,
@@ -113,8 +113,8 @@ class LogMode(str, Enum):
     delete = "delete"
 
 
-@cli_log.command("pipeline-delete")
-def pipeline_log_delete(
+@cli_log.command("workflow-delete")
+def workflow_log_delete(
     mode: Annotated[
         LogMode,
         Argument(case_sensitive=True),

{ddeutil_workflow-0.0.10 → ddeutil_workflow-0.0.11}/src/ddeutil/workflow/exceptions.py

@@ -6,22 +6,22 @@
 from __future__ import annotations
 
 
-class WorkflowException(Exception): ...
+class BaseWorkflowException(Exception): ...
 
 
-class UtilException(WorkflowException): ...
+class UtilException(BaseWorkflowException): ...
 
 
-class StageException(WorkflowException): ...
+class StageException(BaseWorkflowException): ...
 
 
-class JobException(WorkflowException): ...
+class JobException(BaseWorkflowException): ...
 
 
-class PipelineException(WorkflowException): ...
+class WorkflowException(BaseWorkflowException): ...
 
 
-class PipelineFailException(WorkflowException): ...
+class WorkflowFailException(WorkflowException): ...
 
 
 class ParamValueException(WorkflowException): ...