ddeutil-workflow 0.0.41__py3-none-any.whl → 0.0.43__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.41"
+__version__: str = "0.0.43"

ddeutil/workflow/__init__.py

@@ -4,7 +4,7 @@
 # license information.
 # ------------------------------------------------------------------------------
 from .__cron import CronJob, CronRunner
-from .__types import Re
+from .__types import DictData, DictStr, Matrix, Re, TupleStr
 from .conf import (
     Config,
     Loader,
@@ -47,6 +47,10 @@ from .params import (
     StrParam,
 )
 from .result import (
+    FAILED,
+    SKIP,
+    SUCCESS,
+    WAIT,
     Result,
     Status,
 )

ddeutil/workflow/api/api.py

@@ -20,7 +20,7 @@ from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import UJSONResponse
 
 from ..__about__ import __version__
-from ..conf import config, get_logger
+from ..conf import api_config, config, get_logger
 from ..scheduler import ReleaseThread, ReleaseThreads
 from ..workflow import ReleaseQueue, WorkflowTask
 from .repeat import repeat_at
@@ -96,24 +96,24 @@ async def health():
 
 
 # NOTE Add the jobs and logs routes by default.
-app.include_router(job, prefix=config.prefix_path)
-app.include_router(log, prefix=config.prefix_path)
+app.include_router(job, prefix=api_config.prefix_path)
+app.include_router(log, prefix=api_config.prefix_path)
 
 
 # NOTE: Enable the workflows route.
-if config.enable_route_workflow:
+if api_config.enable_route_workflow:
     from .routes import workflow
 
-    app.include_router(workflow, prefix=config.prefix_path)
+    app.include_router(workflow, prefix=api_config.prefix_path)
 
 
 # NOTE: Enable the schedules route.
-if config.enable_route_schedule:
+if api_config.enable_route_schedule:
     from ..logs import get_audit
     from ..scheduler import schedule_task
     from .routes import schedule
 
-    app.include_router(schedule, prefix=config.prefix_path)
+    app.include_router(schedule, prefix=api_config.prefix_path)
 
     @schedule.on_event("startup")
     @repeat_at(cron="* * * * *", delay=2)

ddeutil/workflow/api/routes/schedules.py

@@ -28,7 +28,7 @@ schedule_route = APIRouter(
 async def get_schedules(name: str):
     """Get schedule object."""
     try:
-        schedule: Schedule = Schedule.from_loader(name=name, externals={})
+        schedule: Schedule = Schedule.from_conf(name=name, extras={})
     except ValueError:
         raise HTTPException(
             status_code=st.HTTP_404_NOT_FOUND,
@@ -51,7 +51,7 @@ async def get_deploy_schedulers(request: Request):
 @schedule_route.get(path="/deploy/{name}", status_code=st.HTTP_200_OK)
 async def get_deploy_scheduler(request: Request, name: str):
     if name in request.state.scheduler:
-        schedule = Schedule.from_loader(name)
+        schedule = Schedule.from_conf(name)
         getter: list[dict[str, dict[str, list[datetime]]]] = []
         for workflow in schedule.workflows:
             getter.append(
@@ -94,7 +94,7 @@ async def add_deploy_scheduler(request: Request, name: str):
 
     # NOTE: Create a pair of workflow and on from schedule model.
     try:
-        schedule: Schedule = Schedule.from_loader(name)
+        schedule: Schedule = Schedule.from_conf(name)
     except ValueError as err:
         request.state.scheduler.remove(name)
         logger.exception(err)
@@ -107,7 +107,7 @@ async def add_deploy_scheduler(request: Request, name: str):
         schedule.tasks(
             start_date_waiting,
             queue=request.state.workflow_queue,
-            externals={},
+            extras={},
         ),
     )
     return {
@@ -124,7 +124,7 @@ async def del_deploy_scheduler(request: Request, name: str):
         # NOTE: Remove current schedule name from the state.
         request.state.scheduler.remove(name)
 
-        schedule: Schedule = Schedule.from_loader(name)
+        schedule: Schedule = Schedule.from_conf(name)
 
         for task in schedule.tasks(datetime.now(tz=config.tz), queue={}):
             if task in request.state.workflow_tasks:

ddeutil/workflow/api/routes/workflows.py

@@ -44,7 +44,7 @@ async def get_workflows() -> DictData:
 async def get_workflow_by_name(name: str) -> DictData:
     """Return model of workflow that passing an input workflow name."""
     try:
-        workflow: Workflow = Workflow.from_loader(name=name, externals={})
+        workflow: Workflow = Workflow.from_conf(name=name, extras={})
     except ValueError as err:
         logger.exception(err)
         raise HTTPException(
@@ -69,7 +69,7 @@ class ExecutePayload(BaseModel):
 async def workflow_execute(name: str, payload: ExecutePayload) -> DictData:
     """Return model of workflow that passing an input workflow name."""
     try:
-        workflow: Workflow = Workflow.from_loader(name=name, externals={})
+        workflow: Workflow = Workflow.from_conf(name=name, extras={})
     except ValueError:
         raise HTTPException(
             status_code=st.HTTP_404_NOT_FOUND,

ddeutil/workflow/conf.py

@@ -12,6 +12,7 @@ from collections.abc import Iterator
 from datetime import timedelta
 from functools import cached_property, lru_cache
 from pathlib import Path
+from typing import Optional, TypeVar
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
@@ -20,6 +21,7 @@ from ddeutil.io.paths import glob_files, is_ignored, read_ignore
 
 from .__types import DictData, TupleStr
 
+T = TypeVar("T")
 PREFIX: str = "WORKFLOW"
 
 
@@ -29,12 +31,14 @@ def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
 
 
 __all__: TupleStr = (
+    "api_config",
     "env",
     "get_logger",
     "Config",
     "SimLoad",
     "Loader",
     "config",
+    "dynamic",
 )
 
 
@@ -99,7 +103,7 @@ class Config(BaseConfig):  # pragma: no cov
 
         :rtype: list[str]
         """
-        regis_call_str: str = env("CORE_REGISTRY", ".")
+        regis_call_str: str = env("CORE_REGISTRY_CALLER", ".")
         return [r.strip() for r in regis_call_str.split(",")]
 
     @property
@@ -116,7 +120,7 @@ class Config(BaseConfig):  # pragma: no cov
     # NOTE: Log
     @property
     def log_path(self) -> Path:
-        return Path(env("LOG_PATH", "./logs"))
+        return Path(env("LOG_TRACE_PATH", "./logs"))
 
     @property
     def debug(self) -> bool:
@@ -145,16 +149,15 @@ class Config(BaseConfig):  # pragma: no cov
 
     @property
     def enable_write_log(self) -> bool:
-        return str2bool(env("LOG_ENABLE_WRITE", "false"))
+        return str2bool(env("LOG_TRACE_ENABLE_WRITE", "false"))
 
-    # NOTE: Audit Log
     @property
     def audit_path(self) -> Path:
-        return Path(env("AUDIT_PATH", "./audits"))
+        return Path(env("LOG_AUDIT_PATH", "./audits"))
 
     @property
     def enable_write_audit(self) -> bool:
-        return str2bool(env("AUDIT_ENABLE_WRITE", "false"))
+        return str2bool(env("LOG_AUDIT_ENABLE_WRITE", "false"))
 
     @property
     def log_datetime_format(self) -> str:
@@ -178,29 +181,12 @@ class Config(BaseConfig):  # pragma: no cov
     def job_default_id(self) -> bool:
         return str2bool(env("CORE_JOB_DEFAULT_ID", "false"))
 
-    # NOTE: Workflow
-    @property
-    def max_job_parallel(self) -> int:
-        max_job_parallel = int(env("CORE_MAX_JOB_PARALLEL", "2"))
-
-        # VALIDATE: the MAX_JOB_PARALLEL value should not less than 0.
-        if max_job_parallel < 0:
-            raise ValueError(
-                f"``WORKFLOW_MAX_JOB_PARALLEL`` should more than 0 but got "
-                f"{max_job_parallel}."
-            )
-        return max_job_parallel
-
-    @property
-    def max_job_exec_timeout(self) -> int:
-        return int(env("CORE_MAX_JOB_EXEC_TIMEOUT", "600"))
-
-    @property
-    def max_poking_pool_worker(self) -> int:
-        return int(env("CORE_MAX_NUM_POKING", "4"))
-
     @property
     def max_on_per_workflow(self) -> int:
+        """The maximum on value that store in workflow model.
+
+        :rtype: int
+        """
         return int(env("CORE_MAX_CRON_PER_WORKFLOW", "5"))
 
     @property
@@ -229,7 +215,9 @@ class Config(BaseConfig):  # pragma: no cov
                 f"timedelta with {stop_boundary_delta_str}."
             ) from err
 
-    # NOTE: API
+
+class APIConfig:
+
     @property
     def prefix_path(self) -> str:
         return env("API_PREFIX_PATH", "/api/v1")
@@ -389,6 +377,29 @@ class Loader(SimLoad):
 
 
 config: Config = Config()
+api_config: APIConfig = APIConfig()
+
+
+def dynamic(
+    key: Optional[str] = None,
+    *,
+    f: Optional[T] = None,
+    extras: Optional[DictData] = None,
+) -> Optional[T]:
+    """Dynamic get config if extra value was passed at run-time.
+
+    :param key: (str) A config key that get from Config object.
+    :param f: An inner config function scope.
+    :param extras: An extra values that pass at run-time.
+    """
+    rsx: Optional[T] = extras[key] if extras and key in extras else None
+    rs: Optional[T] = f or getattr(config, key, None)
+    if rsx is not None and not isinstance(rsx, type(rs)):
+        raise TypeError(
+            f"Type of config {key!r} from extras: {rsx!r} does not valid "
+            f"as config {type(rs)}."
+        )
+    return rsx or rs
 
 
 @lru_cache

ddeutil/workflow/cron.py

@@ -72,17 +72,16 @@ class On(BaseModel):
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
-    # NOTE: This is fields of the base schedule.
     extras: Annotated[
         DictData,
         Field(
             default_factory=dict,
-            description="An extras mapping parameters",
+            description="An extras mapping parameters.",
         ),
     ]
     cronjob: Annotated[
         CronJob,
-        Field(description="Cron job of this schedule"),
+        Field(description="A Cronjob object of this schedule."),
     ]
     tz: Annotated[
         str,
@@ -93,12 +92,12 @@ class On(BaseModel):
     ] = "Etc/UTC"
 
     @classmethod
-    def from_value(cls, value: DictStr, externals: DictData) -> Self:
+    def from_value(cls, value: DictStr, extras: DictData) -> Self:
         """Constructor from values that will generate crontab by function.
 
         :param value: A mapping value that will generate crontab before create
             schedule model.
-        :param externals: An extras external parameter that will keep in extras.
+        :param extras: An extras parameter that will keep in extras.
         """
         passing: DictStr = {}
         if "timezone" in value:
@@ -106,22 +105,22 @@ class On(BaseModel):
         passing["cronjob"] = interval2crontab(
             **{v: value[v] for v in value if v in ("interval", "day", "time")}
         )
-        return cls(extras=externals | passing.pop("extras", {}), **passing)
+        return cls(extras=extras | passing.pop("extras", {}), **passing)
 
     @classmethod
-    def from_loader(
+    def from_conf(
         cls,
         name: str,
-        externals: DictData | None = None,
+        extras: DictData | None = None,
     ) -> Self:
         """Constructor from the name of config that will use loader object for
         getting the data.
 
         :param name: A name of config that will get from loader.
-        :param externals: An extras external parameter that will keep in extras.
+        :param extras: An extra parameter that will keep in extras.
         """
-        externals: DictData = externals or {}
-        loader: Loader = Loader(name, externals=externals)
+        extras: DictData = extras or {}
+        loader: Loader = Loader(name, externals=extras)
 
         # NOTE: Validate the config type match with current connection model
         if loader.type != cls.__name__:
@@ -138,7 +137,7 @@ class On(BaseModel):
                             if v in ("interval", "day", "time")
                         }
                     ),
-                    extras=externals | loader_data.pop("extras", {}),
+                    extras=extras | loader_data.pop("extras", {}),
                     **loader_data,
                 )
             )
@@ -149,7 +148,7 @@ class On(BaseModel):
         return cls.model_validate(
             obj=dict(
                 cronjob=loader_data.pop("cronjob"),
-                extras=externals | loader_data.pop("extras", {}),
+                extras=extras | loader_data.pop("extras", {}),
                 **loader_data,
             )
         )

ddeutil/workflow/exceptions.py

@@ -9,10 +9,20 @@ annotate for handle error only.
 """
 from __future__ import annotations
 
-from typing import Any
+from typing import TypedDict
 
+ErrorData = TypedDict(
+    "ErrorData",
+    {
+        "class": Exception,
+        "name": str,
+        "message": str,
+    },
+)
 
-def to_dict(exception: Exception) -> dict[str, Any]:  # pragma: no cov
+
+def to_dict(exception: Exception) -> ErrorData:  # pragma: no cov
+    """Create dict data from exception instance."""
     return {
         "class": exception,
         "name": exception.__class__.__name__,
@@ -22,7 +32,7 @@ def to_dict(exception: Exception) -> dict[str, Any]:  # pragma: no cov
 
 class BaseWorkflowException(Exception):
 
-    def to_dict(self) -> dict[str, Any]:
+    def to_dict(self) -> ErrorData:
         return to_dict(self)
 
 

ddeutil/workflow/job.py

@@ -3,6 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+# [x] Use dynamic config
 """Job Model that use for keeping stages and node that running its stages.
 The job handle the lineage of stages and location of execution of stages that
 mean the job model able to define `runs-on` key that allow you to run this
@@ -32,13 +33,13 @@ from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, Matrix, TupleStr
-from .conf import config
+from .conf import dynamic
 from .exceptions import (
     JobException,
     StageException,
     UtilException,
 )
-from .result import Result, Status
+from .result import FAILED, SKIP, SUCCESS, WAIT, Result, Status
 from .reusables import has_template, param2template
 from .stages import Stage
 from .utils import cross_product, filter_func, gen_id
@@ -50,7 +51,6 @@ __all__: TupleStr = (
     "Strategy",
     "Job",
     "TriggerRules",
-    "TriggerState",
     "RunsOn",
     "RunsOnLocal",
     "RunsOnSelfHosted",
@@ -205,16 +205,6 @@ class TriggerRules(str, Enum):
     none_skipped: str = "none_skipped"
 
 
-class TriggerState(str, Enum):
-    waiting: str = "waiting"
-    passed: str = "passed"
-    skipped: str = "skipped"
-    failed: str = "failed"
-
-    def is_waiting(self):
-        return self.value == "waiting"
-
-
 class RunsOnType(str, Enum):
     """Runs-On enum object."""
 
@@ -346,7 +336,7 @@ class Job(BaseModel):
     )
     extras: DictData = Field(
         default_factory=dict,
-        description="An extra override values.",
+        description="An extra override config values.",
     )
 
     @field_validator("desc", mode="after")
@@ -404,29 +394,33 @@ class Job(BaseModel):
         raise ValueError(f"Stage ID {stage_id} does not exists")
 
     def check_needs(
-        self, jobs: dict[str, Any]
-    ) -> TriggerState:  # pragma: no cov
-        """Return True if job's need exists in an input list of job's ID.
+        self,
+        jobs: dict[str, Any],
+    ) -> Status:  # pragma: no cov
+        """Return Status enum for checking job's need trigger logic in an
+        input list of job's ID.
+
+        :param jobs: A mapping of job ID and result context.
 
-        :param jobs: A mapping of job model and its ID.
+        :raise NotImplementedError: If the job trigger rule out of scope.
 
-        :rtype: TriggerState
+        :rtype: Status
         """
         if not self.needs:
-            return TriggerState.passed
+            return SUCCESS
 
-        def make_return(result: bool) -> TriggerState:
-            return TriggerState.passed if result else TriggerState.failed
+        def make_return(result: bool) -> Status:
+            return SUCCESS if result else FAILED
 
         need_exist: dict[str, Any] = {
             need: jobs[need] for need in self.needs if need in jobs
         }
         if len(need_exist) != len(self.needs):
-            return TriggerState.waiting
+            return WAIT
         elif all("skipped" in need_exist[job] for job in need_exist):
-            return TriggerState.skipped
+            return SKIP
         elif self.trigger_rule == TriggerRules.all_done:
-            return TriggerState.passed
+            return SUCCESS
         elif self.trigger_rule == TriggerRules.all_success:
             rs = all(
                 k not in need_exist[job]
@@ -477,7 +471,9 @@ class Job(BaseModel):
             #   should use the `re` module to validate eval-string before
             #   running.
             rs: bool = eval(
-                param2template(self.condition, params), globals() | params, {}
+                param2template(self.condition, params, extras=self.extras),
+                globals() | params,
+                {},
             )
             if not isinstance(rs, bool):
                 raise TypeError("Return type of condition does not be boolean")
@@ -519,7 +515,9 @@ class Job(BaseModel):
         if "jobs" not in to:
             to["jobs"] = {}
 
-        if self.id is None and not config.job_default_id:
+        if self.id is None and not dynamic(
+            "job_default_id", extras=self.extras
+        ):
             raise JobException(
                 "This job do not set the ID before setting execution output."
             )
@@ -666,7 +664,7 @@ def local_execute_strategy(
                 "strategy execution."
             )
             return result.catch(
-                status=Status.FAILED,
+                status=FAILED,
                 context={
                     strategy_id: {
                         "matrix": strategy,
@@ -706,8 +704,8 @@ def local_execute_strategy(
             )
         except (StageException, UtilException) as err:
             result.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
-            do_raise: bool = (
-                config.job_raise_error if raise_error is None else raise_error
+            do_raise: bool = dynamic(
+                "job_raise_error", f=raise_error, extras=job.extras
             )
             if do_raise:
                 raise JobException(
@@ -716,7 +714,7 @@ def local_execute_strategy(
                 ) from None
 
             return result.catch(
-                status=Status.FAILED,
+                status=FAILED,
                 context={
                     strategy_id: {
                         "matrix": strategy,
@@ -727,7 +725,7 @@ def local_execute_strategy(
             )
 
     return result.catch(
-        status=Status.SUCCESS,
+        status=SUCCESS,
         context={
             strategy_id: {
                 "matrix": strategy,
@@ -782,7 +780,7 @@ def local_execute(
 
             if event and event.is_set():  # pragma: no cov
                 return result.catch(
-                    status=Status.FAILED,
+                    status=FAILED,
                     context={
                         "errors": JobException(
                             "Job strategy was canceled from event that had set "
@@ -800,7 +798,7 @@ def local_execute(
                 raise_error=raise_error,
             )
 
-        return result.catch(status=Status.SUCCESS)
+        return result.catch(status=SUCCESS)
 
     fail_fast_flag: bool = job.strategy.fail_fast
     ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
@@ -811,7 +809,7 @@ def local_execute(
 
     if event and event.is_set():  # pragma: no cov
         return result.catch(
-            status=Status.FAILED,
+            status=FAILED,
             context={
                 "errors": JobException(
                     "Job strategy was canceled from event that had set "
@@ -841,7 +839,7 @@ def local_execute(
         ]
 
         context: DictData = {}
-        status: Status = Status.SUCCESS
+        status: Status = SUCCESS
 
         if not fail_fast_flag:
             done = as_completed(futures, timeout=1800)
@@ -867,7 +865,7 @@ def local_execute(
             try:
                 future.result()
             except JobException as err:
-                status = Status.FAILED
+                status = FAILED
                 result.trace.error(
                     f"[JOB]: {ls} Catch:\n\t{err.__class__.__name__}:"
                     f"\n\t{err}"
@@ -895,7 +893,7 @@ def self_hosted_execute(
     )
 
     if event and event.is_set():
-        return result.catch(status=Status.FAILED)
+        return result.catch(status=FAILED)
 
     import requests
 
@@ -905,8 +903,8 @@ def self_hosted_execute(
     )
 
     if resp.status_code != 200:
-        do_raise: bool = (
-            config.job_raise_error if raise_error is None else raise_error
+        do_raise: bool = dynamic(
+            "job_raise_error", f=raise_error, extras=job.extras
         )
         if do_raise:
             raise JobException(
@@ -914,5 +912,5 @@ def self_hosted_execute(
                 f"{job.runs_on.args.host!r}"
             )
 
-        return result.catch(status=Status.FAILED)
-    return result.catch(status=Status.SUCCESS)
+        return result.catch(status=FAILED)
+    return result.catch(status=SUCCESS)