ddeutil-workflow 0.0.63__py3-none-any.whl → 0.0.65__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.63"
+__version__: str = "0.0.65"

ddeutil/workflow/__init__.py

@@ -6,8 +6,8 @@
 from .__cron import CronJob, CronRunner
 from .__types import DictData, DictStr, Matrix, Re, TupleStr
 from .conf import *
+from .errors import *
 from .event import *
-from .exceptions import *
 from .job import *
 from .logs import (
     Audit,
@@ -33,13 +33,6 @@ from .result import (
     Status,
 )
 from .reusables import *
-from .scheduler import (
-    Schedule,
-    ScheduleWorkflow,
-    schedule_control,
-    schedule_runner,
-    schedule_task,
-)
 from .stages import *
 from .utils import *
 from .workflow import *

ddeutil/workflow/api/__init__.py

@@ -7,8 +7,6 @@ from __future__ import annotations
 
 import contextlib
 from collections.abc import AsyncIterator
-from datetime import datetime, timedelta
-from typing import TypedDict
 
 from dotenv import load_dotenv
 from fastapi import FastAPI, Request
@@ -20,49 +18,18 @@ from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import UJSONResponse
 
 from ..__about__ import __version__
-from ..conf import api_config, config
+from ..conf import api_config
 from ..logs import get_logger
-from ..scheduler import ReleaseThread, ReleaseThreads
-from ..workflow import ReleaseQueue, WorkflowTask
-from .routes import job, log
-from .utils import repeat_at
+from .routes import job, log, workflow
 
 load_dotenv()
 logger = get_logger("uvicorn.error")
 
 
-class State(TypedDict):
-    """TypeDict for State of FastAPI application."""
-
-    scheduler: list[str]
-    workflow_threads: ReleaseThreads
-    workflow_tasks: list[WorkflowTask]
-    workflow_queue: dict[str, ReleaseQueue]
-
-
 @contextlib.asynccontextmanager
-async def lifespan(a: FastAPI) -> AsyncIterator[State]:
+async def lifespan(_: FastAPI) -> AsyncIterator[dict[str, list]]:
     """Lifespan function for the FastAPI application."""
-    a.state.scheduler = []
-    a.state.workflow_threads = {}
-    a.state.workflow_tasks = []
-    a.state.workflow_queue = {}
-
-    yield {
-        # NOTE: Scheduler value should be contained a key of workflow and
-        #   list of datetime of queue and running.
-        #
-        #   ... {
-        #   ...     '<workflow-name>': (
-        #   ...         [<running-datetime>, ...], [<queue-datetime>, ...]
-        #   ...     )
-        #   ... }
-        #
-        "scheduler": a.state.scheduler,
-        "workflow_queue": a.state.workflow_queue,
-        "workflow_threads": a.state.workflow_threads,
-        "workflow_tasks": a.state.workflow_tasks,
-    }
+    yield {}
 
 
 app = FastAPI(
@@ -99,53 +66,7 @@ async def health():
 # NOTE Add the jobs and logs routes by default.
 app.include_router(job, prefix=api_config.prefix_path)
 app.include_router(log, prefix=api_config.prefix_path)
-
-
-# NOTE: Enable the workflows route.
-if api_config.enable_route_workflow:
-    from .routes import workflow
-
-    app.include_router(workflow, prefix=api_config.prefix_path)
-
-
-# NOTE: Enable the schedules route.
-if api_config.enable_route_schedule:
-    from ..logs import get_audit
-    from ..scheduler import schedule_task
-    from .routes import schedule
-
-    app.include_router(schedule, prefix=api_config.prefix_path)
-
-    @schedule.on_event("startup")
-    @repeat_at(cron="* * * * *", delay=2)
-    def scheduler_listener():
-        """Schedule broker every minute at 02 second."""
-        logger.debug(
-            f"[SCHEDULER]: Start listening schedule from queue "
-            f"{app.state.scheduler}"
-        )
-        if app.state.workflow_tasks:
-            schedule_task(
-                app.state.workflow_tasks,
-                stop=datetime.now(config.tz) + timedelta(minutes=1),
-                queue=app.state.workflow_queue,
-                threads=app.state.workflow_threads,
-                audit=get_audit(),
-            )
-
-    @schedule.on_event("startup")
-    @repeat_at(cron="*/5 * * * *", delay=10)
-    def monitoring():
-        """Monitoring workflow thread that running in the background."""
-        logger.debug("[MONITOR]: Start monitoring threading.")
-        snapshot_threads: list[str] = list(app.state.workflow_threads.keys())
-        for t_name in snapshot_threads:
-
-            thread_release: ReleaseThread = app.state.workflow_threads[t_name]
-
-            # NOTE: remove the thread that running success.
-            if not thread_release["thread"].is_alive():
-                app.state.workflow_threads.pop(t_name)
+app.include_router(workflow, prefix=api_config.prefix_path)
 
 
 @app.exception_handler(RequestValidationError)

ddeutil/workflow/api/routes/__init__.py

@@ -5,5 +5,4 @@
 # ------------------------------------------------------------------------------
 from .job import job_route as job
 from .logs import log_route as log
-from .schedules import schedule_route as schedule
 from .workflows import workflow_route as workflow

ddeutil/workflow/api/routes/job.py

@@ -12,7 +12,7 @@ from fastapi.responses import UJSONResponse
 from pydantic import BaseModel, Field
 
 from ...__types import DictData
-from ...exceptions import JobException
+from ...errors import JobError
 from ...job import Job
 from ...logs import get_logger
 from ...result import Result
@@ -59,7 +59,7 @@ async def job_execute(
             ).context,
             to=context,
         )
-    except JobException as err:
+    except JobError as err:
         rs.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
 
     return {
@@ -69,7 +69,6 @@ async def job_execute(
             by_alias=True,
             exclude_none=False,
             exclude_unset=True,
-            exclude_defaults=True,
         ),
         "params": params,
         "context": context,

ddeutil/workflow/api/routes/logs.py

@@ -44,7 +44,6 @@ async def get_traces(
                 by_alias=True,
                 exclude_none=True,
                 exclude_unset=True,
-                exclude_defaults=True,
             )
             for trace in result.trace.find_traces()
         ],
@@ -73,7 +72,6 @@ async def get_trace_with_id(run_id: str):
                 by_alias=True,
                 exclude_none=True,
                 exclude_unset=True,
-                exclude_defaults=True,
             )
         ),
     }

ddeutil/workflow/api/routes/workflows.py

@@ -56,7 +56,6 @@ async def get_workflow_by_name(name: str) -> DictData:
         by_alias=True,
         exclude_none=False,
         exclude_unset=True,
-        exclude_defaults=True,
     )
 
 
@@ -99,7 +98,6 @@ async def get_workflow_audits(name: str):
                     by_alias=True,
                     exclude_none=False,
                     exclude_unset=True,
-                    exclude_defaults=True,
                 )
                 for audit in get_audit().find_audits(name=name)
             ],
@@ -133,6 +131,5 @@ async def get_workflow_release_audit(name: str, release: str):
             by_alias=True,
             exclude_none=False,
             exclude_unset=True,
-            exclude_defaults=True,
         ),
     }

ddeutil/workflow/conf.py

@@ -6,11 +6,9 @@
 from __future__ import annotations
 
 import copy
-import json
 import os
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
-from datetime import timedelta
 from functools import cached_property
 from inspect import isclass
 from pathlib import Path
@@ -20,7 +18,7 @@ from zoneinfo import ZoneInfo
 from ddeutil.core import str2bool
 from ddeutil.io import YamlFlResolve, search_env_replace
 from ddeutil.io.paths import glob_files, is_ignored, read_ignore
-from pydantic import SecretStr
+from pydantic import SecretStr, TypeAdapter
 
 from .__types import DictData
 
@@ -143,10 +141,6 @@ class Config:  # pragma: no cov
     def log_datetime_format(self) -> str:
         return env("LOG_DATETIME_FORMAT", "%Y-%m-%d %H:%M:%S")
 
-    @property
-    def stage_raise_error(self) -> bool:
-        return str2bool(env("CORE_STAGE_RAISE_ERROR", "false"))
-
     @property
     def stage_default_id(self) -> bool:
         return str2bool(env("CORE_STAGE_DEFAULT_ID", "false"))
@@ -163,28 +157,6 @@ class Config:  # pragma: no cov
     def max_queue_complete_hist(self) -> int:
         return int(env("CORE_MAX_QUEUE_COMPLETE_HIST", "16"))
 
-    # NOTE: App
-    @property
-    def max_schedule_process(self) -> int:
-        return int(env("APP_MAX_PROCESS", "2"))
-
-    @property
-    def max_schedule_per_process(self) -> int:
-        return int(env("APP_MAX_SCHEDULE_PER_PROCESS", "100"))
-
-    @property
-    def stop_boundary_delta(self) -> timedelta:
-        stop_boundary_delta_str: str = env(
-            "APP_STOP_BOUNDARY_DELTA", '{"minutes": 5, "seconds": 20}'
-        )
-        try:
-            return timedelta(**json.loads(stop_boundary_delta_str))
-        except Exception as err:
-            raise ValueError(
-                "Config `WORKFLOW_APP_STOP_BOUNDARY_DELTA` can not parsing to"
-                f"timedelta with {stop_boundary_delta_str}."
-            ) from err
-
 
 class APIConfig:
     """API Config object."""
@@ -193,14 +165,6 @@ class APIConfig:
     def prefix_path(self) -> str:
         return env("API_PREFIX_PATH", "/api/v1")
 
-    @property
-    def enable_route_workflow(self) -> bool:
-        return str2bool(env("API_ENABLE_ROUTE_WORKFLOW", "true"))
-
-    @property
-    def enable_route_schedule(self) -> bool:
-        return str2bool(env("API_ENABLE_ROUTE_SCHEDULE", "true"))
-
 
 class BaseLoad(ABC):  # pragma: no cov
     """Base Load object is the abstraction object for any Load object that
@@ -496,7 +460,7 @@ def pass_env(value: T) -> T:  # pragma: no cov
     return None if rs == "null" else rs
 
 
-class WorkflowSecret(SecretStr):  # pragma: no cov
+class CallerSecret(SecretStr):  # pragma: no cov
     """Workflow Secret String model."""
 
     def get_secret_value(self) -> str:
@@ -506,3 +470,7 @@ class WorkflowSecret(SecretStr):  # pragma: no cov
         :rtype: str
         """
         return pass_env(super().get_secret_value())
+
+
+# NOTE: Define the caller secret type for use it directly in the caller func.
+CallerSecretType = TypeAdapter(CallerSecret)

ddeutil/workflow/{exceptions.py → errors.py}

@@ -11,6 +11,8 @@ from __future__ import annotations
 
 from typing import Literal, Optional, TypedDict, Union, overload
 
+from .__types import DictData, StrOrInt
+
 
 class ErrorData(TypedDict):
     """Error data type dict for typing necessary keys of return of to_dict func
@@ -21,7 +23,7 @@ class ErrorData(TypedDict):
     message: str
 
 
-def to_dict(exception: Exception) -> ErrorData:  # pragma: no cov
+def to_dict(exception: Exception, **kwargs) -> ErrorData:  # pragma: no cov
     """Create dict data from exception instance.
 
     :param exception: An exception object.
@@ -31,17 +33,27 @@ def to_dict(exception: Exception) -> ErrorData:  # pragma: no cov
     return {
         "name": exception.__class__.__name__,
         "message": str(exception),
+        **kwargs,
     }
 
 
-class BaseWorkflowException(Exception):
+class BaseError(Exception):
     """Base Workflow exception class will implement the `refs` argument for
     making an error context to the result context.
     """
 
-    def __init__(self, message: str, *, refs: Optional[str] = None):
+    def __init__(
+        self,
+        message: str,
+        *,
+        refs: Optional[StrOrInt] = None,
+        context: Optional[DictData] = None,
+        params: Optional[DictData] = None,
+    ) -> None:
         super().__init__(message)
         self.refs: Optional[str] = refs
+        self.context: DictData = context or {}
+        self.params: DictData = params or {}
 
     @overload
     def to_dict(
@@ -54,7 +66,9 @@ class BaseWorkflowException(Exception):
     ) -> ErrorData: ...  # pragma: no cov
 
     def to_dict(
-        self, with_refs: bool = False
+        self,
+        with_refs: bool = False,
+        **kwargs,
     ) -> Union[ErrorData, dict[str, ErrorData]]:
         """Return ErrorData data from the current exception object. If with_refs
         flag was set, it will return mapping of refs and itself data.
@@ -64,25 +78,46 @@ class BaseWorkflowException(Exception):
         data: ErrorData = to_dict(self)
         if with_refs and (self.refs is not None and self.refs != "EMPTY"):
             return {self.refs: data}
-        return data
+        return data | kwargs
+
+
+class UtilError(BaseError): ...
+
+
+class ResultError(UtilError): ...
+
+
+class StageError(BaseError): ...
+
+
+class StageRetryError(StageError): ...
+
+
+class StageCancelError(StageError): ...
+
+
+class StageSkipError(StageError): ...
+
+
+class JobError(BaseError): ...
 
 
-class UtilException(BaseWorkflowException): ...
+class JobCancelError(JobError): ...
 
 
-class ResultException(UtilException): ...
+class JobSkipError(JobError): ...
 
 
-class StageException(BaseWorkflowException): ...
+class WorkflowError(BaseError): ...
 
 
-class JobException(BaseWorkflowException): ...
+class WorkflowCancelError(WorkflowError): ...
 
 
-class WorkflowException(BaseWorkflowException): ...
+class WorkflowSkipError(WorkflowError): ...
 
 
-class ParamValueException(WorkflowException): ...
+class WorkflowTimeoutError(WorkflowError): ...
 
 
-class ScheduleException(BaseWorkflowException): ...
+class ParamError(WorkflowError): ...