ddeutil-workflow 0.0.36__tar.gz → 0.0.38__tar.gz

{ddeutil_workflow-0.0.36 → ddeutil_workflow-0.0.38}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.36
+Version: 0.0.38
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -31,6 +31,10 @@ Provides-Extra: api
 Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "api"
 Requires-Dist: httpx; extra == "api"
 Requires-Dist: ujson; extra == "api"
+Provides-Extra: async
+Requires-Dist: aiofiles; extra == "async"
+Requires-Dist: aiohttp; extra == "async"
+Dynamic: license-file
 
 # Workflow Orchestration
 
@@ -61,10 +65,10 @@ configuration. It called **Metadata Driven Data Workflow**.
 
 **:pushpin: <u>Rules of This Workflow engine</u>**:
 
-1. The Minimum frequency unit of scheduling is **1 minute** :warning:
-2. Can not re-run only failed stage and its pending downstream :rotating_light:
-3. All parallel tasks inside workflow engine use Multi-Threading
-   (🐍 Python 3.13 unlock GIL :unlock:)
+1. The Minimum frequency unit of scheduling is **1 Minute** 🕘
+2. **Can not** re-run only failed stage and its pending downstream ↩️
+3. All parallel tasks inside workflow engine use **Multi-Threading**
+   (Python 3.13 unlock GIL 🐍🔓)
 
 ---
 
@@ -165,6 +169,8 @@ run-py-local:
       run-date: datetime
    jobs:
       getting-api-data:
+         runs-on:
+            type: local
          stages:
             - name: "Retrieve API Data"
               id: retrieve-api

{ddeutil_workflow-0.0.36 → ddeutil_workflow-0.0.38}/README.md

@@ -27,10 +27,10 @@ configuration. It called **Metadata Driven Data Workflow**.
 
 **:pushpin: <u>Rules of This Workflow engine</u>**:
 
-1. The Minimum frequency unit of scheduling is **1 minute** :warning:
-2. Can not re-run only failed stage and its pending downstream :rotating_light:
-3. All parallel tasks inside workflow engine use Multi-Threading
-   (🐍 Python 3.13 unlock GIL :unlock:)
+1. The Minimum frequency unit of scheduling is **1 Minute** 🕘
+2. **Can not** re-run only failed stage and its pending downstream ↩️
+3. All parallel tasks inside workflow engine use **Multi-Threading**
+   (Python 3.13 unlock GIL 🐍🔓)
 
 ---
 
@@ -131,6 +131,8 @@ run-py-local:
       run-date: datetime
    jobs:
       getting-api-data:
+         runs-on:
+            type: local
          stages:
             - name: "Retrieve API Data"
               id: retrieve-api

{ddeutil_workflow-0.0.36 → ddeutil_workflow-0.0.38}/pyproject.toml

@@ -40,6 +40,10 @@ api = [
     "httpx",
     "ujson",
 ]
+async = [
+    "aiofiles",
+    "aiohttp",
+]
 
 [project.urls]
 Homepage = "https://github.com/ddeutils/ddeutil-workflow/"
@@ -63,12 +67,14 @@ commit_prefix_force_fix = true
 branch = true
 relative_files = true
 concurrency = ["thread", "multiprocessing"]
-source = ["ddeutil.workflow"]
+source = ["src.ddeutil.workflow"]
 omit = [
     "src/ddeutil/workflow/__about__.py",
     "src/ddeutil/workflow/__cron.py",
+    "src/ddeutil/workflow/context.py",
     "src/ddeutil/workflow/api/__init__.py",
     "src/ddeutil/workflow/api/api.py",
+    "src/ddeutil/workflow/api/log.py",
     "src/ddeutil/workflow/api/repeat.py",
     "src/ddeutil/workflow/api/routes/__init__.py",
     "src/ddeutil/workflow/api/routes/job.py",
@@ -100,6 +106,7 @@ addopts = [
 filterwarnings = [
     "error",
     "ignore::DeprecationWarning",
+    "ignore::pytest.PytestUnraisableExceptionWarning",
 ]
 log_cli = true
 log_cli_level = "DEBUG"

ddeutil_workflow-0.0.38/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.38"

{ddeutil_workflow-0.0.36 → ddeutil_workflow-0.0.38}/src/ddeutil/workflow/__init__.py

@@ -39,6 +39,8 @@ from .job import (
     Job,
     RunsOn,
     Strategy,
+    local_execute,
+    local_execute_strategy,
 )
 from .logs import (
     TraceData,
@@ -69,6 +71,8 @@ from .stages import (
     BashStage,
     CallStage,
     EmptyStage,
+    ForEachStage,
+    ParallelStage,
     PyStage,
     Stage,
     TriggerStage,
@@ -89,7 +93,6 @@ from .templates import (
 from .utils import (
     batch,
     cross_product,
-    dash2underscore,
     delay,
     filter_func,
     gen_id,

{ddeutil_workflow-0.0.36 → ddeutil_workflow-0.0.38}/src/ddeutil/workflow/api/api.py

@@ -27,7 +27,7 @@ from .repeat import repeat_at
 from .routes import job, log
 
 load_dotenv()
-logger = get_logger("ddeutil.workflow")
+logger = get_logger("uvicorn.error")
 
 
 class State(TypedDict):
@@ -151,6 +151,7 @@ if config.enable_route_schedule:
 async def validation_exception_handler(
     request: Request, exc: RequestValidationError
 ):
+    _ = request
     return UJSONResponse(
         status_code=st.HTTP_422_UNPROCESSABLE_ENTITY,
         content=jsonable_encoder({"detail": exc.errors(), "body": exc.body}),
@@ -164,4 +165,5 @@ if __name__ == "__main__":
         app,
         host="0.0.0.0",
         port=80,
+        log_level="DEBUG",
     )

ddeutil_workflow-0.0.38/src/ddeutil/workflow/api/log.py

@@ -0,0 +1,59 @@
+from ..conf import config
+
+LOGGING_CONFIG = {  # pragma: no cov
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "standard": {
+            "format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
+        },
+        "custom_formatter": {
+            "format": config.log_format,
+            "datefmt": config.log_datetime_format,
+        },
+    },
+    "root": {
+        "level": "DEBUG" if config.debug else "INFO",
+    },
+    "handlers": {
+        "default": {
+            "formatter": "standard",
+            "class": "logging.StreamHandler",
+            "stream": "ext://sys.stderr",
+        },
+        "stream_handler": {
+            "formatter": "custom_formatter",
+            "class": "logging.StreamHandler",
+            "stream": "ext://sys.stdout",
+        },
+        "file_handler": {
+            "formatter": "custom_formatter",
+            "class": "logging.handlers.RotatingFileHandler",
+            "filename": "logs/app.log",
+            "maxBytes": 1024 * 1024 * 1,
+            "backupCount": 3,
+        },
+    },
+    "loggers": {
+        "uvicorn": {
+            "handlers": ["default", "file_handler"],
+            "level": "DEBUG" if config.debug else "INFO",
+            "propagate": False,
+        },
+        "uvicorn.access": {
+            "handlers": ["stream_handler", "file_handler"],
+            "level": "DEBUG" if config.debug else "INFO",
+            "propagate": False,
+        },
+        "uvicorn.error": {
+            "handlers": ["stream_handler", "file_handler"],
+            "level": "DEBUG" if config.debug else "INFO",
+            "propagate": False,
+        },
+        # "uvicorn.asgi": {
+        #     "handlers": ["stream_handler", "file_handler"],
+        #     "level": "TRACE",
+        #     "propagate": False,
+        # },
+    },
+}

{ddeutil_workflow-0.0.36 → ddeutil_workflow-0.0.38}/src/ddeutil/workflow/api/repeat.py

@@ -15,7 +15,7 @@ from starlette.concurrency import run_in_threadpool
 from ..__cron import CronJob
 from ..conf import config, get_logger
 
-logger = get_logger("ddeutil.workflow")
+logger = get_logger("uvicorn.error")
 
 
 def get_cronjob_delta(cron: str) -> float:

{ddeutil_workflow-0.0.36 → ddeutil_workflow-0.0.38}/src/ddeutil/workflow/api/routes/job.py

@@ -17,7 +17,7 @@ from ...exceptions import JobException
 from ...job import Job
 from ...result import Result
 
-logger = get_logger("ddeutil.workflow")
+logger = get_logger("uvicorn.error")
 
 
 job_route = APIRouter(
@@ -45,6 +45,7 @@ async def job_execute(
         run_id=result.run_id,
         parent_run_id=result.parent_run_id,
     )
+    context: DictData = {}
     try:
         job.set_outputs(
             job.execute(
@@ -52,7 +53,7 @@ async def job_execute(
                 run_id=rs.run_id,
                 parent_run_id=rs.parent_run_id,
             ).context,
-            to=params,
+            to=context,
         )
     except JobException as err:
         rs.trace.error(f"[WORKFLOW]: {err.__class__.__name__}: {err}")
@@ -70,4 +71,5 @@ async def job_execute(
             exclude_defaults=True,
         ),
         "params": params,
+        "context": context,
     }

ddeutil_workflow-0.0.38/src/ddeutil/workflow/api/routes/logs.py

@@ -0,0 +1,173 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+"""This route include audit and trace log paths."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Path, Query
+from fastapi import status as st
+from fastapi.responses import UJSONResponse
+
+from ...audit import get_audit
+from ...logs import get_trace_obj
+
+log_route = APIRouter(
+    prefix="/logs",
+    tags=["logs"],
+    default_response_class=UJSONResponse,
+)
+
+
+@log_route.get(
+    path="/traces/",
+    response_class=UJSONResponse,
+    status_code=st.HTTP_200_OK,
+    summary="Read all trace logs.",
+    tags=["trace"],
+)
+async def get_traces(
+    offset: int = Query(default=0, gt=0),
+    limit: int = Query(default=100, gt=0),
+):
+    """Return all trace logs from the current trace log path that config with
+    `WORKFLOW_LOG_PATH` environment variable name.
+    """
+    return {
+        "message": (
+            f"Getting trace logs with offset: {offset} and limit: {limit}"
+        ),
+        "traces": [
+            trace.model_dump(
+                by_alias=True,
+                exclude_none=True,
+                exclude_unset=True,
+                exclude_defaults=True,
+            )
+            for trace in get_trace_obj().find_logs()
+        ],
+    }
+
+
+@log_route.get(
+    path="/traces/{run_id}",
+    response_class=UJSONResponse,
+    status_code=st.HTTP_200_OK,
+    summary="Read trace log with specific running ID.",
+    tags=["trace"],
+)
+async def get_trace_with_id(run_id: str):
+    """Return trace log with specific running ID from the current trace log path
+    that config with `WORKFLOW_LOG_PATH` environment variable name.
+
+    - **run_id**: A running ID that want to search a trace log from the log
+        path.
+    """
+    return {
+        "message": f"Getting trace log with specific running ID: {run_id}",
+        "trace": (
+            get_trace_obj()
+            .find_log_with_id(run_id)
+            .model_dump(
+                by_alias=True,
+                exclude_none=True,
+                exclude_unset=True,
+                exclude_defaults=True,
+            )
+        ),
+    }
+
+
+@log_route.get(
+    path="/audits/",
+    response_class=UJSONResponse,
+    status_code=st.HTTP_200_OK,
+    summary="Read all audit logs.",
+    tags=["audit"],
+)
+async def get_audits():
+    """Return all audit logs from the current audit log path that config with
+    `WORKFLOW_AUDIT_PATH` environment variable name.
+    """
+    return {
+        "message": "Getting audit logs",
+        "audits": list(get_audit().find_audits(name="demo")),
+    }
+
+
+@log_route.get(
+    path="/audits/{workflow}/",
+    response_class=UJSONResponse,
+    status_code=st.HTTP_200_OK,
+    summary="Read all audit logs with specific workflow name.",
+    tags=["audit"],
+)
+async def get_audit_with_workflow(workflow: str):
+    """Return all audit logs with specific workflow name from the current audit
+    log path that config with `WORKFLOW_AUDIT_PATH` environment variable name.
+
+    - **workflow**: A specific workflow name that want to find audit logs.
+    """
+    return {
+        "message": f"Getting audit logs with workflow name {workflow}",
+        "audits": list(get_audit().find_audits(name="demo")),
+    }
+
+
+@log_route.get(
+    path="/audits/{workflow}/{release}",
+    response_class=UJSONResponse,
+    status_code=st.HTTP_200_OK,
+    summary="Read all audit logs with specific workflow name and release date.",
+    tags=["audit"],
+)
+async def get_audit_with_workflow_release(
+    workflow: str = Path(...),
+    release: str = Path(...),
+):
+    """Return all audit logs with specific workflow name and release date from
+    the current audit log path that config with `WORKFLOW_AUDIT_PATH`
+    environment variable name.
+
+    - **workflow**: A specific workflow name that want to find audit logs.
+    - **release**: A release date with a string format `%Y%m%d%H%M%S`.
+    """
+    return {
+        "message": (
+            f"Getting audit logs with workflow name {workflow} and release "
+            f"{release}"
+        ),
+        "audits": list(get_audit().find_audits(name="demo")),
+    }
+
+
+@log_route.get(
+    path="/audits/{workflow}/{release}/{run_id}",
+    response_class=UJSONResponse,
+    status_code=st.HTTP_200_OK,
+    summary=(
+        "Read all audit logs with specific workflow name, release date "
+        "and running ID."
+    ),
+    tags=["audit"],
+)
+async def get_audit_with_workflow_release_run_id(
+    workflow: str, release: str, run_id: str
+):
+    """Return all audit logs with specific workflow name and release date from
+    the current audit log path that config with `WORKFLOW_AUDIT_PATH`
+    environment variable name.
+
+    - **workflow**: A specific workflow name that want to find audit logs.
+    - **release**: A release date with a string format `%Y%m%d%H%M%S`.
+    - **run_id**: A running ID that want to search audit log from this release
+        date.
+    """
+    return {
+        "message": (
+            f"Getting audit logs with workflow name {workflow}, release "
+            f"{release}, and running ID {run_id}"
+        ),
+        "audits": list(get_audit().find_audits(name="demo")),
+    }

{ddeutil_workflow-0.0.36 → ddeutil_workflow-0.0.38}/src/ddeutil/workflow/api/routes/schedules.py

@@ -15,7 +15,7 @@ from fastapi.responses import UJSONResponse
 from ...conf import config, get_logger
 from ...scheduler import Schedule
 
-logger = get_logger("ddeutil.workflow")
+logger = get_logger("uvicorn.error")
 
 schedule_route = APIRouter(
     prefix="/schedules",
@@ -24,7 +24,7 @@ schedule_route = APIRouter(
 )
 
 
-@schedule_route.get(path="/{name}")
+@schedule_route.get(path="/{name}", status_code=st.HTTP_200_OK)
 async def get_schedules(name: str):
     """Get schedule object."""
     try:
@@ -42,13 +42,13 @@ async def get_schedules(name: str):
     )
 
 
-@schedule_route.get(path="/deploy/")
+@schedule_route.get(path="/deploy/", status_code=st.HTTP_200_OK)
 async def get_deploy_schedulers(request: Request):
     snapshot = copy.deepcopy(request.state.scheduler)
     return {"schedule": snapshot}
 
 
-@schedule_route.get(path="/deploy/{name}")
+@schedule_route.get(path="/deploy/{name}", status_code=st.HTTP_200_OK)
 async def get_deploy_scheduler(request: Request, name: str):
     if name in request.state.scheduler:
         schedule = Schedule.from_loader(name)
@@ -76,7 +76,7 @@ async def get_deploy_scheduler(request: Request, name: str):
     )
 
 
-@schedule_route.post(path="/deploy/{name}")
+@schedule_route.post(path="/deploy/{name}", status_code=st.HTTP_202_ACCEPTED)
 async def add_deploy_scheduler(request: Request, name: str):
     """Adding schedule name to application state store."""
     if name in request.state.scheduler:
@@ -116,7 +116,7 @@ async def add_deploy_scheduler(request: Request, name: str):
     }
 
 
-@schedule_route.delete(path="/deploy/{name}")
+@schedule_route.delete(path="/deploy/{name}", status_code=st.HTTP_202_ACCEPTED)
 async def del_deploy_scheduler(request: Request, name: str):
     """Delete workflow task on the schedule listener."""
     if name in request.state.scheduler:

{ddeutil_workflow-0.0.36 → ddeutil_workflow-0.0.38}/src/ddeutil/workflow/api/routes/workflows.py

@@ -20,7 +20,7 @@ from ...conf import Loader, get_logger
 from ...result import Result
 from ...workflow import Workflow
 
-logger = get_logger("ddeutil.workflow")
+logger = get_logger("uvicorn.error")
 
 workflow_route = APIRouter(
     prefix="/workflows",
@@ -29,7 +29,7 @@ workflow_route = APIRouter(
 )
 
 
-@workflow_route.get(path="/")
+@workflow_route.get(path="/", status_code=st.HTTP_200_OK)
 async def get_workflows() -> DictData:
     """Return all workflow workflows that exists in config path."""
     workflows: DictData = dict(Loader.finds(Workflow))
@@ -40,7 +40,7 @@ async def get_workflows() -> DictData:
     }
 
 
-@workflow_route.get(path="/{name}")
+@workflow_route.get(path="/{name}", status_code=st.HTTP_200_OK)
 async def get_workflow_by_name(name: str) -> DictData:
     """Return model of workflow that passing an input workflow name."""
     try:
@@ -66,7 +66,7 @@ class ExecutePayload(BaseModel):
 
 
 @workflow_route.post(path="/{name}/execute", status_code=st.HTTP_202_ACCEPTED)
-async def execute_workflow(name: str, payload: ExecutePayload) -> DictData:
+async def workflow_execute(name: str, payload: ExecutePayload) -> DictData:
     """Return model of workflow that passing an input workflow name."""
     try:
         workflow: Workflow = Workflow.from_loader(name=name, externals={})
@@ -90,7 +90,7 @@ async def execute_workflow(name: str, payload: ExecutePayload) -> DictData:
     return asdict(result)
 
 
-@workflow_route.get(path="/{name}/audits")
+@workflow_route.get(path="/{name}/audits", status_code=st.HTTP_200_OK)
 async def get_workflow_audits(name: str):
     try:
         return {
@@ -112,11 +112,13 @@ async def get_workflow_audits(name: str):
         ) from None
 
 
-@workflow_route.get(path="/{name}/audits/{release}")
+@workflow_route.get(path="/{name}/audits/{release}", status_code=st.HTTP_200_OK)
 async def get_workflow_release_audit(name: str, release: str):
+    """Get Workflow audit log with an input release value."""
     try:
         audit: Audit = get_audit().find_audit_with_release(
-            name=name, release=datetime.strptime(release, "%Y%m%d%H%M%S")
+            name=name,
+            release=datetime.strptime(release, "%Y%m%d%H%M%S"),
         )
     except FileNotFoundError:
         raise HTTPException(

{ddeutil_workflow-0.0.36 → ddeutil_workflow-0.0.38}/src/ddeutil/workflow/caller.py

@@ -26,6 +26,7 @@ T = TypeVar("T")
 P = ParamSpec("P")
 
 logger = logging.getLogger("ddeutil.workflow")
+logging.getLogger("asyncio").setLevel(logging.INFO)
 
 
 class TagFunc(Protocol):
@@ -60,10 +61,13 @@ def tag(
 
         @wraps(func)
         def wrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
-            # NOTE: Able to do anything before calling the call function.
             return func(*args, **kwargs)
 
-        return wrapped
+        @wraps(func)
+        async def awrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
+            return await func(*args, **kwargs)
+
+        return awrapped if inspect.iscoroutinefunction(func) else wrapped
 
     return func_internal
 
@@ -91,7 +95,9 @@ def make_registry(submodule: str) -> dict[str, Registry]:
         for fstr, func in inspect.getmembers(importer, inspect.isfunction):
             # NOTE: check function attribute that already set tag by
             #   ``utils.tag`` decorator.
-            if not (hasattr(func, "tag") and hasattr(func, "name")):
+            if not (
+                hasattr(func, "tag") and hasattr(func, "name")
+            ):  # pragma: no cov
                 continue
 
             # NOTE: Define type of the func value.

{ddeutil_workflow-0.0.36 → ddeutil_workflow-0.0.38}/src/ddeutil/workflow/conf.py

@@ -31,7 +31,6 @@ def glob_files(path: Path) -> Iterator[Path]:  # pragma: no cov
 
 
 __all__: TupleStr = (
-    "LOGGING_CONFIG",
     "env",
     "get_logger",
     "Config",
@@ -422,62 +421,3 @@ def get_logger(name: str):
 
     logger.setLevel(logging.DEBUG if config.debug else logging.INFO)
     return logger
-
-
-LOGGING_CONFIG = {  # pragma: no cov
-    "version": 1,
-    "disable_existing_loggers": False,
-    "formatters": {
-        "standard": {
-            "format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
-        },
-        "custom_formatter": {
-            "format": config.log_format,
-            "datefmt": config.log_datetime_format,
-        },
-    },
-    "root": {
-        "level": "DEBUG" if config.debug else "INFO",
-    },
-    "handlers": {
-        "default": {
-            "formatter": "standard",
-            "class": "logging.StreamHandler",
-            "stream": "ext://sys.stderr",
-        },
-        "stream_handler": {
-            "formatter": "custom_formatter",
-            "class": "logging.StreamHandler",
-            "stream": "ext://sys.stdout",
-        },
-        "file_handler": {
-            "formatter": "custom_formatter",
-            "class": "logging.handlers.RotatingFileHandler",
-            "filename": "logs/app.log",
-            "maxBytes": 1024 * 1024 * 1,
-            "backupCount": 3,
-        },
-    },
-    "loggers": {
-        "uvicorn": {
-            "handlers": ["default", "file_handler"],
-            "level": "DEBUG" if config.debug else "INFO",
-            "propagate": False,
-        },
-        "uvicorn.access": {
-            "handlers": ["stream_handler", "file_handler"],
-            "level": "DEBUG" if config.debug else "INFO",
-            "propagate": False,
-        },
-        "uvicorn.error": {
-            "handlers": ["stream_handler", "file_handler"],
-            "level": "DEBUG" if config.debug else "INFO",
-            "propagate": False,
-        },
-        # "uvicorn.asgi": {
-        #     "handlers": ["stream_handler", "file_handler"],
-        #     "level": "TRACE",
-        #     "propagate": False,
-        # },
-    },
-}