ddeutil-workflow 0.0.35__py3-none-any.whl → 0.0.37__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.35"
+__version__: str = "0.0.37"

ddeutil/workflow/__init__.py

@@ -37,9 +37,11 @@ from .exceptions import (
 )
 from .job import (
     Job,
+    RunsOn,
     Strategy,
 )
 from .logs import (
+    TraceData,
     TraceLog,
     get_dt_tznow,
     get_trace,

ddeutil/workflow/api/api.py

@@ -11,7 +11,11 @@ from datetime import datetime, timedelta
 from typing import TypedDict
 
 from dotenv import load_dotenv
-from fastapi import FastAPI
+from fastapi import FastAPI, Request
+from fastapi import status as st
+from fastapi.encoders import jsonable_encoder
+from fastapi.exceptions import RequestValidationError
+from fastapi.middleware.cors import CORSMiddleware
 from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import UJSONResponse
 
@@ -20,10 +24,10 @@ from ..conf import config, get_logger
 from ..scheduler import ReleaseThread, ReleaseThreads
 from ..workflow import ReleaseQueue, WorkflowTask
 from .repeat import repeat_at
-from .routes import log
+from .routes import job, log
 
 load_dotenv()
-logger = get_logger("ddeutil.workflow")
+logger = get_logger("uvicorn.error")
 
 
 class State(TypedDict):
@@ -61,24 +65,38 @@ async def lifespan(a: FastAPI) -> AsyncIterator[State]:
 
 
 app = FastAPI(
-    titile="Workflow API",
+    titile="Workflow",
     description=(
-        "This is workflow FastAPI web application that use to manage manual "
-        "execute or schedule workflow via RestAPI."
+        "This is a workflow FastAPI application that use to manage manual "
+        "execute, logging, and schedule workflow via RestAPI."
     ),
     version=__version__,
     lifespan=lifespan,
     default_response_class=UJSONResponse,
 )
 app.add_middleware(GZipMiddleware, minimum_size=1000)
+origins: list[str] = [
+    "http://localhost",
+    "http://localhost:88",
+    "http://localhost:80",
+]
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
 
 
 @app.get("/")
 async def health():
-    return {"message": "Workflow API already start up"}
+    """Index view that not return any template without json status."""
+    return {"message": "Workflow already start up with healthy status."}
 
 
-# NOTE Add the logs route by default.
+# NOTE Add the jobs and logs routes by default.
+app.include_router(job, prefix=config.prefix_path)
 app.include_router(log, prefix=config.prefix_path)
 
 
@@ -111,12 +129,13 @@ if config.enable_route_schedule:
                 stop=datetime.now(config.tz) + timedelta(minutes=1),
                 queue=app.state.workflow_queue,
                 threads=app.state.workflow_threads,
-                log=get_audit(),
+                audit=get_audit(),
             )
 
     @schedule.on_event("startup")
     @repeat_at(cron="*/5 * * * *", delay=10)
     def monitoring():
+        """Monitoring workflow thread that running in the background."""
         logger.debug("[MONITOR]: Start monitoring threading.")
         snapshot_threads: list[str] = list(app.state.workflow_threads.keys())
         for t_name in snapshot_threads:
@@ -126,3 +145,25 @@ if config.enable_route_schedule:
             # NOTE: remove the thread that running success.
             if not thread_release["thread"].is_alive():
                 app.state.workflow_threads.pop(t_name)
+
+
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(
+    request: Request, exc: RequestValidationError
+):
+    _ = request
+    return UJSONResponse(
+        status_code=st.HTTP_422_UNPROCESSABLE_ENTITY,
+        content=jsonable_encoder({"detail": exc.errors(), "body": exc.body}),
+    )
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(
+        app,
+        host="0.0.0.0",
+        port=80,
+        log_level="DEBUG",
+    )

ddeutil/workflow/api/log.py

@@ -0,0 +1,59 @@
+from ..conf import config
+
+LOGGING_CONFIG = {  # pragma: no cov
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "standard": {
+            "format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
+        },
+        "custom_formatter": {
+            "format": config.log_format,
+            "datefmt": config.log_datetime_format,
+        },
+    },
+    "root": {
+        "level": "DEBUG" if config.debug else "INFO",
+    },
+    "handlers": {
+        "default": {
+            "formatter": "standard",
+            "class": "logging.StreamHandler",
+            "stream": "ext://sys.stderr",
+        },
+        "stream_handler": {
+            "formatter": "custom_formatter",
+            "class": "logging.StreamHandler",
+            "stream": "ext://sys.stdout",
+        },
+        "file_handler": {
+            "formatter": "custom_formatter",
+            "class": "logging.handlers.RotatingFileHandler",
+            "filename": "logs/app.log",
+            "maxBytes": 1024 * 1024 * 1,
+            "backupCount": 3,
+        },
+    },
+    "loggers": {
+        "uvicorn": {
+            "handlers": ["default", "file_handler"],
+            "level": "DEBUG" if config.debug else "INFO",
+            "propagate": False,
+        },
+        "uvicorn.access": {
+            "handlers": ["stream_handler", "file_handler"],
+            "level": "DEBUG" if config.debug else "INFO",
+            "propagate": False,
+        },
+        "uvicorn.error": {
+            "handlers": ["stream_handler", "file_handler"],
+            "level": "DEBUG" if config.debug else "INFO",
+            "propagate": False,
+        },
+        # "uvicorn.asgi": {
+        #     "handlers": ["stream_handler", "file_handler"],
+        #     "level": "TRACE",
+        #     "propagate": False,
+        # },
+    },
+}

ddeutil/workflow/api/repeat.py

@@ -15,23 +15,32 @@ from starlette.concurrency import run_in_threadpool
 from ..__cron import CronJob
 from ..conf import config, get_logger
 
-logger = get_logger("ddeutil.workflow")
+logger = get_logger("uvicorn.error")
 
 
 def get_cronjob_delta(cron: str) -> float:
     """This function returns the time delta between now and the next cron
     execution time.
+
+    :rtype: float
     """
     now: datetime = datetime.now(tz=config.tz)
     cron = CronJob(cron)
     return (cron.schedule(now).next - now).total_seconds()
 
 
-def cron_valid(cron: str):
+def cron_valid(cron: str, raise_error: bool = True) -> bool:
+    """Check this crontab string value is valid with its cron syntax.
+
+    :rtype: bool
+    """
     try:
         CronJob(cron)
+        return True
     except Exception as err:
-        raise ValueError(f"Crontab value does not valid, {cron}") from err
+        if raise_error:
+            raise ValueError(f"Crontab value does not valid, {cron}") from err
+        return False
 
 
 async def run_func(
@@ -41,6 +50,7 @@ async def run_func(
     raise_exceptions: bool = False,
     **kwargs,
 ):
+    """Run function inside the repeat decorator functions."""
     try:
         if is_coroutine:
             await func(*args, **kwargs)
@@ -62,11 +72,11 @@ def repeat_at(
     """This function returns a decorator that makes a function execute
     periodically as per the cron expression provided.
 
-    :param cron: str
-        Cron-style string for periodic execution, eg. '0 0 * * *' every midnight
-    :param delay:
-    :param raise_exceptions: bool (default False)
-        Whether to raise exceptions or log them
+    :param cron: (str) A Cron-style string for periodic execution, e.g.
+        '0 0 * * *' every midnight
+    :param delay: (float) A delay seconds value.
+    :param raise_exceptions: (bool) A raise exception flag. Whether to raise
+        exceptions or log them if raise was set be false.
     :param max_repetitions: int (default None)
         Maximum number of times to repeat the function. If None, repeat
         indefinitely.
@@ -81,12 +91,12 @@ def repeat_at(
 
         @wraps(func)
         def wrapper(*_args, **_kwargs):
-            repititions: int = 0
+            repetitions: int = 0
             cron_valid(cron)
 
             async def loop(*args, **kwargs):
-                nonlocal repititions
-                while max_repetitions is None or repititions < max_repetitions:
+                nonlocal repetitions
+                while max_repetitions is None or repetitions < max_repetitions:
                     sleep_time = get_cronjob_delta(cron) + delay
                     await asyncio.sleep(sleep_time)
                     await run_func(
@@ -96,7 +106,7 @@ def repeat_at(
                         raise_exceptions=raise_exceptions,
                         **kwargs,
                     )
-                    repititions += 1
+                    repetitions += 1
 
             ensure_future(loop(*_args, **_kwargs))
 

ddeutil/workflow/api/routes/__init__.py

@@ -3,6 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+from .job import job_route as job
 from .logs import log_route as log
 from .schedules import schedule_route as schedule
 from .workflows import workflow_route as workflow

ddeutil/workflow/api/routes/job.py

@@ -0,0 +1,73 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from fastapi import APIRouter
+from fastapi.responses import UJSONResponse
+from pydantic import BaseModel
+
+from ...__types import DictData
+from ...conf import get_logger
+from ...exceptions import JobException
+from ...job import Job
+from ...result import Result
+
+logger = get_logger("uvicorn.error")
+
+
+job_route = APIRouter(
+    prefix="/job",
+    tags=["job"],
+    default_response_class=UJSONResponse,
+)
+
+
+class ResultPost(BaseModel):
+    context: DictData
+    run_id: str
+    parent_run_id: Optional[str] = None
+
+
+@job_route.post(path="/execute/")
+async def job_execute(
+    result: ResultPost,
+    job: Job,
+    params: dict[str, Any],
+):
+    """Execute job via API."""
+    rs: Result = Result(
+        context=result.context,
+        run_id=result.run_id,
+        parent_run_id=result.parent_run_id,
+    )
+    try:
+        job.set_outputs(
+            job.execute(
+                params=params,
+                run_id=rs.run_id,
+                parent_run_id=rs.parent_run_id,
+            ).context,
+            to=params,
+        )
+    except JobException as err:
+        rs.trace.error(f"[WORKFLOW]: {err.__class__.__name__}: {err}")
+
+    return {
+        "message": "Start execute job via API.",
+        "result": {
+            "run_id": rs.run_id,
+            "parent_run_id": rs.parent_run_id,
+        },
+        "job": job.model_dump(
+            by_alias=True,
+            exclude_none=True,
+            exclude_unset=True,
+            exclude_defaults=True,
+        ),
+        "params": params,
+    }

ddeutil/workflow/api/routes/logs.py

@@ -3,19 +3,16 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+"""This route include audit and trace log paths."""
 from __future__ import annotations
 
 from fastapi import APIRouter
+from fastapi import status as st
 from fastapi.responses import UJSONResponse
 
-from ...conf import get_logger
+from ...audit import get_audit
 from ...logs import get_trace_obj
 
-logger = get_logger("ddeutil.workflow")
-
-
-# NOTE: Start create the schedule routes.
-#
 log_route = APIRouter(
     prefix="/logs",
     tags=["logs"],
@@ -23,14 +20,146 @@ log_route = APIRouter(
 )
 
 
-@log_route.get(path="/")
-async def get_logs():
+@log_route.get(
+    path="/traces/",
+    response_class=UJSONResponse,
+    status_code=st.HTTP_200_OK,
+    summary="Read all trace logs.",
+    tags=["trace"],
+)
+async def get_traces():
+    """Return all trace logs from the current trace log path that config with
+    `WORKFLOW_LOG_PATH` environment variable name.
+    """
+    return {
+        "message": "Getting trace logs",
+        "traces": [
+            trace.model_dump(
+                by_alias=True,
+                exclude_none=True,
+                exclude_unset=True,
+                exclude_defaults=True,
+            )
+            for trace in get_trace_obj().find_logs()
+        ],
+    }
+
+
+@log_route.get(
+    path="/traces/{run_id}",
+    response_class=UJSONResponse,
+    status_code=st.HTTP_200_OK,
+    summary="Read trace log with specific running ID.",
+    tags=["trace"],
+)
+async def get_trace_with_id(run_id: str):
+    """Return trace log with specific running ID from the current trace log path
+    that config with `WORKFLOW_LOG_PATH` environment variable name.
+
+    - **run_id**: A running ID that want to search a trace log from the log
+        path.
+    """
+    return {
+        "message": f"Getting trace log with specific running ID: {run_id}",
+        "trace": (
+            get_trace_obj()
+            .find_log_with_id(run_id)
+            .model_dump(
+                by_alias=True,
+                exclude_none=True,
+                exclude_unset=True,
+                exclude_defaults=True,
+            )
+        ),
+    }
+
+
+@log_route.get(
+    path="/audits/",
+    response_class=UJSONResponse,
+    status_code=st.HTTP_200_OK,
+    summary="Read all audit logs.",
+    tags=["audit"],
+)
+async def get_audits():
+    """Return all audit logs from the current audit log path that config with
+    `WORKFLOW_AUDIT_PATH` environment variable name.
+    """
+    return {
+        "message": "Getting audit logs",
+        "audits": list(get_audit().find_audits(name="demo")),
+    }
+
+
+@log_route.get(
+    path="/audits/{workflow}/",
+    response_class=UJSONResponse,
+    status_code=st.HTTP_200_OK,
+    summary="Read all audit logs with specific workflow name.",
+    tags=["audit"],
+)
+async def get_audit_with_workflow(workflow: str):
+    """Return all audit logs with specific workflow name from the current audit
+    log path that config with `WORKFLOW_AUDIT_PATH` environment variable name.
+
+    - **workflow**: A specific workflow name that want to find audit logs.
+    """
     return {
-        "message": "Getting logs",
-        "audits": list(get_trace_obj().find_logs()),
+        "message": f"Getting audit logs with workflow name {workflow}",
+        "audits": list(get_audit().find_audits(name="demo")),
     }
 
 
-@log_route.get(path="/{run_id}")
-async def get_log_with_run_id(run_id: str):
-    return get_trace_obj().find_log_with_id(run_id)
+@log_route.get(
+    path="/audits/{workflow}/{release}",
+    response_class=UJSONResponse,
+    status_code=st.HTTP_200_OK,
+    summary="Read all audit logs with specific workflow name and release date.",
+    tags=["audit"],
+)
+async def get_audit_with_workflow_release(workflow: str, release: str):
+    """Return all audit logs with specific workflow name and release date from
+    the current audit log path that config with `WORKFLOW_AUDIT_PATH`
+    environment variable name.
+
+    - **workflow**: A specific workflow name that want to find audit logs.
+    - **release**: A release date with a string format `%Y%m%d%H%M%S`.
+    """
+    return {
+        "message": (
+            f"Getting audit logs with workflow name {workflow} and release "
+            f"{release}"
+        ),
+        "audits": list(get_audit().find_audits(name="demo")),
+    }
+
+
+@log_route.get(
+    path="/audits/{workflow}/{release}/{run_id}",
+    response_class=UJSONResponse,
+    status_code=st.HTTP_200_OK,
+    summary=(
+        "Read all audit logs with specific workflow name, release date "
+        "and running ID."
+    ),
+    tags=["audit"],
+)
+async def get_audit_with_workflow_release_run_id(
+    workflow: str, release: str, run_id: str
+):
+    """Return all audit logs with specific workflow name and release date from
+    the current audit log path that config with `WORKFLOW_AUDIT_PATH`
+    environment variable name.
+
+    - **workflow**: A specific workflow name that want to find audit logs.
+    - **release**: A release date with a string format `%Y%m%d%H%M%S`.
+    - **run_id**: A running ID that want to search audit log from this release
+        date.
+    """
+    return {
+        "message": (
+            f"Getting audit logs with workflow name {workflow}, release "
+            f"{release}, and running ID {run_id}"
+        ),
+        "audits": list(get_audit().find_audits(name="demo")),
+    }

ddeutil/workflow/api/routes/schedules.py

@@ -15,7 +15,7 @@ from fastapi.responses import UJSONResponse
 from ...conf import config, get_logger
 from ...scheduler import Schedule
 
-logger = get_logger("ddeutil.workflow")
+logger = get_logger("uvicorn.error")
 
 schedule_route = APIRouter(
     prefix="/schedules",
@@ -24,8 +24,9 @@ schedule_route = APIRouter(
 )
 
 
-@schedule_route.get(path="/{name}")
+@schedule_route.get(path="/{name}", status_code=st.HTTP_200_OK)
 async def get_schedules(name: str):
+    """Get schedule object."""
     try:
         schedule: Schedule = Schedule.from_loader(name=name, externals={})
     except ValueError:
@@ -41,13 +42,13 @@ async def get_schedules(name: str):
     )
 
 
-@schedule_route.get(path="/deploy/")
+@schedule_route.get(path="/deploy/", status_code=st.HTTP_200_OK)
 async def get_deploy_schedulers(request: Request):
     snapshot = copy.deepcopy(request.state.scheduler)
     return {"schedule": snapshot}
 
 
-@schedule_route.get(path="/deploy/{name}")
+@schedule_route.get(path="/deploy/{name}", status_code=st.HTTP_200_OK)
 async def get_deploy_scheduler(request: Request, name: str):
     if name in request.state.scheduler:
         schedule = Schedule.from_loader(name)
@@ -75,7 +76,7 @@ async def get_deploy_scheduler(request: Request, name: str):
     )
 
 
-@schedule_route.post(path="/deploy/{name}")
+@schedule_route.post(path="/deploy/{name}", status_code=st.HTTP_202_ACCEPTED)
 async def add_deploy_scheduler(request: Request, name: str):
     """Adding schedule name to application state store."""
     if name in request.state.scheduler:
@@ -115,7 +116,7 @@ async def add_deploy_scheduler(request: Request, name: str):
     }
 
 
-@schedule_route.delete(path="/deploy/{name}")
+@schedule_route.delete(path="/deploy/{name}", status_code=st.HTTP_202_ACCEPTED)
 async def del_deploy_scheduler(request: Request, name: str):
     """Delete workflow task on the schedule listener."""
     if name in request.state.scheduler:

ddeutil/workflow/api/routes/workflows.py

@@ -20,7 +20,7 @@ from ...conf import Loader, get_logger
 from ...result import Result
 from ...workflow import Workflow
 
-logger = get_logger("ddeutil.workflow")
+logger = get_logger("uvicorn.error")
 
 workflow_route = APIRouter(
     prefix="/workflows",
@@ -29,7 +29,7 @@ workflow_route = APIRouter(
 )
 
 
-@workflow_route.get(path="/")
+@workflow_route.get(path="/", status_code=st.HTTP_200_OK)
 async def get_workflows() -> DictData:
     """Return all workflow workflows that exists in config path."""
     workflows: DictData = dict(Loader.finds(Workflow))
@@ -40,7 +40,7 @@ async def get_workflows() -> DictData:
     }
 
 
-@workflow_route.get(path="/{name}")
+@workflow_route.get(path="/{name}", status_code=st.HTTP_200_OK)
 async def get_workflow_by_name(name: str) -> DictData:
     """Return model of workflow that passing an input workflow name."""
     try:
@@ -66,7 +66,7 @@ class ExecutePayload(BaseModel):
 
 
 @workflow_route.post(path="/{name}/execute", status_code=st.HTTP_202_ACCEPTED)
-async def execute_workflow(name: str, payload: ExecutePayload) -> DictData:
+async def workflow_execute(name: str, payload: ExecutePayload) -> DictData:
     """Return model of workflow that passing an input workflow name."""
     try:
         workflow: Workflow = Workflow.from_loader(name=name, externals={})
@@ -90,7 +90,7 @@ async def execute_workflow(name: str, payload: ExecutePayload) -> DictData:
     return asdict(result)
 
 
-@workflow_route.get(path="/{name}/audits")
+@workflow_route.get(path="/{name}/audits", status_code=st.HTTP_200_OK)
 async def get_workflow_audits(name: str):
     try:
         return {
@@ -112,11 +112,13 @@ async def get_workflow_audits(name: str):
         ) from None
 
 
-@workflow_route.get(path="/{name}/audits/{release}")
+@workflow_route.get(path="/{name}/audits/{release}", status_code=st.HTTP_200_OK)
 async def get_workflow_release_audit(name: str, release: str):
+    """Get Workflow audit log with an input release value."""
     try:
         audit: Audit = get_audit().find_audit_with_release(
-            name=name, release=datetime.strptime(release, "%Y%m%d%H%M%S")
+            name=name,
+            release=datetime.strptime(release, "%Y%m%d%H%M%S"),
         )
     except FileNotFoundError:
         raise HTTPException(

ddeutil/workflow/audit.py

@@ -112,7 +112,8 @@ class FileAudit(BaseAudit):
         :param release: A release datetime that want to search log.
 
         :raise FileNotFoundError:
-        :raise NotImplementedError:
+        :raise NotImplementedError: If an input release does not pass to this
+            method. Because this method does not implement latest log.
 
         :rtype: Self
         """
@@ -181,7 +182,9 @@ class FileAudit(BaseAudit):
             trace.debug("[LOG]: Skip writing log cause config was set")
             return self
 
-        log_file: Path = self.pointer() / f"{self.run_id}.log"
+        log_file: Path = (
+            self.pointer() / f"{self.parent_run_id or self.run_id}.log"
+        )
         log_file.write_text(
             json.dumps(
                 self.model_dump(exclude=excluded),
@@ -196,7 +199,7 @@ class FileAudit(BaseAudit):
 class SQLiteAudit(BaseAudit):  # pragma: no cov
     """SQLite Audit Pydantic Model."""
 
-    table_name: ClassVar[str] = "workflow_log"
+    table_name: ClassVar[str] = "audits"
     schemas: ClassVar[
         str
     ] = """

ddeutil/workflow/caller.py

@@ -91,7 +91,9 @@ def make_registry(submodule: str) -> dict[str, Registry]:
         for fstr, func in inspect.getmembers(importer, inspect.isfunction):
             # NOTE: check function attribute that already set tag by
             #   ``utils.tag`` decorator.
-            if not (hasattr(func, "tag") and hasattr(func, "name")):
+            if not (
+                hasattr(func, "tag") and hasattr(func, "name")
+            ):  # pragma: no cov
                 continue
 
             # NOTE: Define type of the func value.