ddeutil-workflow 0.0.40__py3-none-any.whl → 0.0.42__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.40"
+__version__: str = "0.0.42"

ddeutil/workflow/__init__.py

@@ -5,17 +5,6 @@
 # ------------------------------------------------------------------------------
 from .__cron import CronJob, CronRunner
 from .__types import Re
-from .audit import (
-    Audit,
-    get_audit,
-)
-from .caller import (
-    ReturnTagFunc,
-    TagFunc,
-    extract_call,
-    make_registry,
-    tag,
-)
 from .conf import (
     Config,
     Loader,
@@ -43,8 +32,10 @@ from .job import (
     local_execute_strategy,
 )
 from .logs import (
+    Audit,
     TraceData,
     TraceLog,
+    get_audit,
     get_dt_tznow,
     get_trace,
 )
@@ -58,7 +49,24 @@ from .params import (
 from .result import (
     Result,
     Status,
-    default_gen_id,
+)
+from .reusables import (
+    FILTERS,
+    FilterFunc,
+    FilterRegistry,
+    ReturnTagFunc,
+    TagFunc,
+    custom_filter,
+    extract_call,
+    get_args_const,
+    has_template,
+    make_filter_registry,
+    make_registry,
+    map_post_filter,
+    not_in_template,
+    param2template,
+    str2template,
+    tag,
 )
 from .scheduler import (
     Schedule,
@@ -77,22 +85,10 @@ from .stages import (
     Stage,
     TriggerStage,
 )
-from .templates import (
-    FILTERS,
-    FilterFunc,
-    FilterRegistry,
-    custom_filter,
-    get_args_const,
-    has_template,
-    make_filter_registry,
-    map_post_filter,
-    not_in_template,
-    param2template,
-    str2template,
-)
 from .utils import (
     batch,
     cross_product,
+    default_gen_id,
     delay,
     filter_func,
     gen_id,

ddeutil/workflow/api/api.py

@@ -20,7 +20,7 @@ from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import UJSONResponse
 
 from ..__about__ import __version__
-from ..conf import config, get_logger
+from ..conf import api_config, config, get_logger
 from ..scheduler import ReleaseThread, ReleaseThreads
 from ..workflow import ReleaseQueue, WorkflowTask
 from .repeat import repeat_at
@@ -96,24 +96,24 @@ async def health():
 
 
 # NOTE Add the jobs and logs routes by default.
-app.include_router(job, prefix=config.prefix_path)
-app.include_router(log, prefix=config.prefix_path)
+app.include_router(job, prefix=api_config.prefix_path)
+app.include_router(log, prefix=api_config.prefix_path)
 
 
 # NOTE: Enable the workflows route.
-if config.enable_route_workflow:
+if api_config.enable_route_workflow:
     from .routes import workflow
 
-    app.include_router(workflow, prefix=config.prefix_path)
+    app.include_router(workflow, prefix=api_config.prefix_path)
 
 
 # NOTE: Enable the schedules route.
-if config.enable_route_schedule:
-    from ..audit import get_audit
+if api_config.enable_route_schedule:
+    from ..logs import get_audit
     from ..scheduler import schedule_task
     from .routes import schedule
 
-    app.include_router(schedule, prefix=config.prefix_path)
+    app.include_router(schedule, prefix=api_config.prefix_path)
 
     @schedule.on_event("startup")
     @repeat_at(cron="* * * * *", delay=2)

ddeutil/workflow/api/routes/logs.py

@@ -10,8 +10,7 @@ from fastapi import APIRouter, Path, Query
 from fastapi import status as st
 from fastapi.responses import UJSONResponse
 
-from ...audit import get_audit
-from ...logs import get_trace_obj
+from ...logs import get_audit, get_trace_obj
 
 log_route = APIRouter(
     prefix="/logs",

ddeutil/workflow/api/routes/schedules.py

@@ -28,7 +28,7 @@ schedule_route = APIRouter(
 async def get_schedules(name: str):
     """Get schedule object."""
     try:
-        schedule: Schedule = Schedule.from_loader(name=name, externals={})
+        schedule: Schedule = Schedule.from_conf(name=name, extras={})
     except ValueError:
         raise HTTPException(
             status_code=st.HTTP_404_NOT_FOUND,
@@ -51,7 +51,7 @@ async def get_deploy_schedulers(request: Request):
 @schedule_route.get(path="/deploy/{name}", status_code=st.HTTP_200_OK)
 async def get_deploy_scheduler(request: Request, name: str):
     if name in request.state.scheduler:
-        schedule = Schedule.from_loader(name)
+        schedule = Schedule.from_conf(name)
         getter: list[dict[str, dict[str, list[datetime]]]] = []
         for workflow in schedule.workflows:
             getter.append(
@@ -94,7 +94,7 @@ async def add_deploy_scheduler(request: Request, name: str):
 
     # NOTE: Create a pair of workflow and on from schedule model.
     try:
-        schedule: Schedule = Schedule.from_loader(name)
+        schedule: Schedule = Schedule.from_conf(name)
     except ValueError as err:
         request.state.scheduler.remove(name)
         logger.exception(err)
@@ -107,7 +107,7 @@ async def add_deploy_scheduler(request: Request, name: str):
         schedule.tasks(
             start_date_waiting,
             queue=request.state.workflow_queue,
-            externals={},
+            extras={},
         ),
     )
     return {
@@ -124,7 +124,7 @@ async def del_deploy_scheduler(request: Request, name: str):
         # NOTE: Remove current schedule name from the state.
         request.state.scheduler.remove(name)
 
-        schedule: Schedule = Schedule.from_loader(name)
+        schedule: Schedule = Schedule.from_conf(name)
 
         for task in schedule.tasks(datetime.now(tz=config.tz), queue={}):
             if task in request.state.workflow_tasks:

ddeutil/workflow/api/routes/workflows.py

@@ -15,8 +15,8 @@ from fastapi.responses import UJSONResponse
 from pydantic import BaseModel
 
 from ...__types import DictData
-from ...audit import Audit, get_audit
 from ...conf import Loader, get_logger
+from ...logs import Audit, get_audit
 from ...result import Result
 from ...workflow import Workflow
 
@@ -44,7 +44,7 @@ async def get_workflows() -> DictData:
 async def get_workflow_by_name(name: str) -> DictData:
     """Return model of workflow that passing an input workflow name."""
     try:
-        workflow: Workflow = Workflow.from_loader(name=name, externals={})
+        workflow: Workflow = Workflow.from_conf(name=name, extras={})
     except ValueError as err:
         logger.exception(err)
         raise HTTPException(
@@ -69,7 +69,7 @@ class ExecutePayload(BaseModel):
 async def workflow_execute(name: str, payload: ExecutePayload) -> DictData:
     """Return model of workflow that passing an input workflow name."""
     try:
-        workflow: Workflow = Workflow.from_loader(name=name, externals={})
+        workflow: Workflow = Workflow.from_conf(name=name, extras={})
     except ValueError:
         raise HTTPException(
             status_code=st.HTTP_404_NOT_FOUND,

ddeutil/workflow/conf.py

@@ -12,6 +12,7 @@ from collections.abc import Iterator
 from datetime import timedelta
 from functools import cached_property, lru_cache
 from pathlib import Path
+from typing import Optional, TypeVar
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
@@ -20,20 +21,24 @@ from ddeutil.io.paths import glob_files, is_ignored, read_ignore
 
 from .__types import DictData, TupleStr
 
+T = TypeVar("T")
 PREFIX: str = "WORKFLOW"
 
 
 def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
+    """Get environment variable with uppercase and adding prefix string."""
     return os.getenv(f"{PREFIX}_{var.upper().replace(' ', '_')}", default)
 
 
 __all__: TupleStr = (
+    "api_config",
     "env",
     "get_logger",
     "Config",
     "SimLoad",
     "Loader",
     "config",
+    "dynamic",
 )
 
 
@@ -94,11 +99,19 @@ class Config(BaseConfig):  # pragma: no cov
     # NOTE: Register
     @property
     def regis_call(self) -> list[str]:
-        regis_call_str: str = env("CORE_REGISTRY", ".")
+        """Register Caller module importer str.
+
+        :rtype: list[str]
+        """
+        regis_call_str: str = env("CORE_REGISTRY_CALLER", ".")
         return [r.strip() for r in regis_call_str.split(",")]
 
     @property
     def regis_filter(self) -> list[str]:
+        """Register Filter module.
+
+        :rtype: list[str]
+        """
         regis_filter_str: str = env(
             "CORE_REGISTRY_FILTER", "ddeutil.workflow.templates"
         )
@@ -107,7 +120,7 @@ class Config(BaseConfig):  # pragma: no cov
     # NOTE: Log
     @property
     def log_path(self) -> Path:
-        return Path(env("LOG_PATH", "./logs"))
+        return Path(env("LOG_TRACE_PATH", "./logs"))
 
     @property
     def debug(self) -> bool:
@@ -136,16 +149,15 @@ class Config(BaseConfig):  # pragma: no cov
 
     @property
     def enable_write_log(self) -> bool:
-        return str2bool(env("LOG_ENABLE_WRITE", "false"))
+        return str2bool(env("LOG_TRACE_ENABLE_WRITE", "false"))
 
-    # NOTE: Audit Log
     @property
     def audit_path(self) -> Path:
-        return Path(env("AUDIT_PATH", "./audits"))
+        return Path(env("LOG_AUDIT_PATH", "./audits"))
 
     @property
     def enable_write_audit(self) -> bool:
-        return str2bool(env("AUDIT_ENABLE_WRITE", "false"))
+        return str2bool(env("LOG_AUDIT_ENABLE_WRITE", "false"))
 
     @property
     def log_datetime_format(self) -> str:
@@ -169,29 +181,12 @@ class Config(BaseConfig):  # pragma: no cov
     def job_default_id(self) -> bool:
         return str2bool(env("CORE_JOB_DEFAULT_ID", "false"))
 
-    # NOTE: Workflow
-    @property
-    def max_job_parallel(self) -> int:
-        max_job_parallel = int(env("CORE_MAX_JOB_PARALLEL", "2"))
-
-        # VALIDATE: the MAX_JOB_PARALLEL value should not less than 0.
-        if max_job_parallel < 0:
-            raise ValueError(
-                f"``WORKFLOW_MAX_JOB_PARALLEL`` should more than 0 but got "
-                f"{max_job_parallel}."
-            )
-        return max_job_parallel
-
-    @property
-    def max_job_exec_timeout(self) -> int:
-        return int(env("CORE_MAX_JOB_EXEC_TIMEOUT", "600"))
-
-    @property
-    def max_poking_pool_worker(self) -> int:
-        return int(env("CORE_MAX_NUM_POKING", "4"))
-
     @property
     def max_on_per_workflow(self) -> int:
+        """The maximum on value that store in workflow model.
+
+        :rtype: int
+        """
         return int(env("CORE_MAX_CRON_PER_WORKFLOW", "5"))
 
     @property
@@ -220,7 +215,9 @@ class Config(BaseConfig):  # pragma: no cov
                 f"timedelta with {stop_boundary_delta_str}."
             ) from err
 
-    # NOTE: API
+
+class APIConfig:
+
     @property
     def prefix_path(self) -> str:
         return env("API_PREFIX_PATH", "/api/v1")
@@ -326,7 +323,8 @@ class SimLoad:
     def filter_yaml(cls, file: Path, name: str | None = None) -> DictData:
         if any(file.suffix.endswith(s) for s in (".yml", ".yaml")):
             values: DictData = YamlFlResolve(file).read()
-            return values.get(name, {}) if name else values
+            if values is not None:
+                return values.get(name, {}) if name else values
         return {}
 
     @cached_property
@@ -379,6 +377,29 @@ class Loader(SimLoad):
 
 
 config: Config = Config()
+api_config: APIConfig = APIConfig()
+
+
+def dynamic(
+    key: Optional[str] = None,
+    *,
+    f: Optional[T] = None,
+    extras: Optional[DictData] = None,
+) -> Optional[T]:
+    """Dynamic get config if extra value was passed at run-time.
+
+    :param key: (str) A config key that get from Config object.
+    :param f: An inner config function scope.
+    :param extras: An extra values that pass at run-time.
+    """
+    rsx: Optional[T] = extras[key] if extras and key in extras else None
+    rs: Optional[T] = f or getattr(config, key, None)
+    if rsx is not None and not isinstance(rsx, type(rs)):
+        raise TypeError(
+            f"Type of config {key!r} from extras: {rsx!r} does not valid "
+            f"as config {type(rs)}."
+        )
+    return rsx or rs
 
 
 @lru_cache

ddeutil/workflow/cron.py

@@ -72,17 +72,16 @@ class On(BaseModel):
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
-    # NOTE: This is fields of the base schedule.
     extras: Annotated[
         DictData,
         Field(
             default_factory=dict,
-            description="An extras mapping parameters",
+            description="An extras mapping parameters.",
         ),
     ]
     cronjob: Annotated[
         CronJob,
-        Field(description="Cron job of this schedule"),
+        Field(description="A Cronjob object of this schedule."),
     ]
     tz: Annotated[
         str,
@@ -93,12 +92,12 @@ class On(BaseModel):
     ] = "Etc/UTC"
 
     @classmethod
-    def from_value(cls, value: DictStr, externals: DictData) -> Self:
+    def from_value(cls, value: DictStr, extras: DictData) -> Self:
         """Constructor from values that will generate crontab by function.
 
         :param value: A mapping value that will generate crontab before create
             schedule model.
-        :param externals: An extras external parameter that will keep in extras.
+        :param extras: An extras parameter that will keep in extras.
         """
         passing: DictStr = {}
         if "timezone" in value:
@@ -106,22 +105,22 @@ class On(BaseModel):
         passing["cronjob"] = interval2crontab(
             **{v: value[v] for v in value if v in ("interval", "day", "time")}
         )
-        return cls(extras=externals | passing.pop("extras", {}), **passing)
+        return cls(extras=extras | passing.pop("extras", {}), **passing)
 
     @classmethod
-    def from_loader(
+    def from_conf(
         cls,
         name: str,
-        externals: DictData | None = None,
+        extras: DictData | None = None,
     ) -> Self:
         """Constructor from the name of config that will use loader object for
         getting the data.
 
         :param name: A name of config that will get from loader.
-        :param externals: An extras external parameter that will keep in extras.
+        :param extras: An extra parameter that will keep in extras.
         """
-        externals: DictData = externals or {}
-        loader: Loader = Loader(name, externals=externals)
+        extras: DictData = extras or {}
+        loader: Loader = Loader(name, externals=extras)
 
         # NOTE: Validate the config type match with current connection model
         if loader.type != cls.__name__:
@@ -138,7 +137,7 @@ class On(BaseModel):
                             if v in ("interval", "day", "time")
                         }
                     ),
-                    extras=externals | loader_data.pop("extras", {}),
+                    extras=extras | loader_data.pop("extras", {}),
                     **loader_data,
                 )
             )
@@ -149,7 +148,7 @@ class On(BaseModel):
         return cls.model_validate(
             obj=dict(
                 cronjob=loader_data.pop("cronjob"),
-                extras=externals | loader_data.pop("extras", {}),
+                extras=extras | loader_data.pop("extras", {}),
                 **loader_data,
             )
         )

ddeutil/workflow/job.py

@@ -3,6 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+# [x] Use dynamic config
 """Job Model that use for keeping stages and node that running its stages.
 The job handle the lineage of stages and location of execution of stages that
 mean the job model able to define `runs-on` key that allow you to run this
@@ -32,15 +33,15 @@ from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, Matrix, TupleStr
-from .conf import config
+from .conf import dynamic
 from .exceptions import (
     JobException,
     StageException,
     UtilException,
 )
 from .result import Result, Status
+from .reusables import has_template, param2template
 from .stages import Stage
-from .templates import has_template, param2template
 from .utils import cross_product, filter_func, gen_id
 
 MatrixFilter = list[dict[str, Union[str, int]]]
@@ -344,6 +345,10 @@ class Job(BaseModel):
         default_factory=Strategy,
         description="A strategy matrix that want to generate.",
     )
+    extras: DictData = Field(
+        default_factory=dict,
+        description="An extra override config values.",
+    )
 
     @field_validator("desc", mode="after")
     def ___prepare_desc__(cls, value: str) -> str:
@@ -394,15 +399,20 @@ class Job(BaseModel):
         """
         for stage in self.stages:
             if stage_id == (stage.id or ""):
+                if self.extras:
+                    stage.extras = self.extras
                 return stage
         raise ValueError(f"Stage ID {stage_id} does not exists")
 
     def check_needs(
-        self, jobs: dict[str, Any]
+        self,
+        jobs: dict[str, Any],
     ) -> TriggerState:  # pragma: no cov
         """Return True if job's need exists in an input list of job's ID.
 
-        :param jobs: A mapping of job model and its ID.
+        :param jobs: A mapping of job ID and result context.
+
+        :raise NotImplementedError: If the job trigger rule out of scope.
 
         :rtype: TriggerState
         """
@@ -471,7 +481,9 @@ class Job(BaseModel):
             #   should use the `re` module to validate eval-string before
             #   running.
             rs: bool = eval(
-                param2template(self.condition, params), globals() | params, {}
+                param2template(self.condition, params, extras=self.extras),
+                globals() | params,
+                {},
             )
             if not isinstance(rs, bool):
                 raise TypeError("Return type of condition does not be boolean")
@@ -513,7 +525,9 @@ class Job(BaseModel):
         if "jobs" not in to:
             to["jobs"] = {}
 
-        if self.id is None and not config.job_default_id:
+        if self.id is None and not dynamic(
+            "job_default_id", extras=self.extras
+        ):
             raise JobException(
                 "This job do not set the ID before setting execution output."
             )
@@ -596,7 +610,7 @@ def local_execute_strategy(
     *,
     result: Result | None = None,
     event: Event | None = None,
-    raise_error: bool = False,
+    raise_error: bool | None = None,
 ) -> Result:
     """Local job strategy execution with passing dynamic parameters from the
     workflow execution to strategy matrix.
@@ -694,12 +708,16 @@ def local_execute_strategy(
                     params=context,
                     run_id=result.run_id,
                     parent_run_id=result.parent_run_id,
+                    event=event,
                 ).context,
                 to=context,
             )
         except (StageException, UtilException) as err:
             result.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
-            if raise_error or config.job_raise_error:
+            do_raise: bool = dynamic(
+                "job_raise_error", f=raise_error, extras=job.extras
+            )
+            if do_raise:
                 raise JobException(
                     f"Stage execution error: {err.__class__.__name__}: "
                     f"{err}"
@@ -735,7 +753,7 @@ def local_execute(
     parent_run_id: str | None = None,
     result: Result | None = None,
     event: Event | None = None,
-    raise_error: bool = False,
+    raise_error: bool | None = None,
 ) -> Result:
     """Local job execution with passing dynamic parameters from the workflow
     execution. It will generate matrix values at the first step and run
@@ -865,3 +883,44 @@ def local_execute(
                 context.update({"errors": err.to_dict()})
 
     return result.catch(status=status, context=context)
+
+
+def self_hosted_execute(
+    job: Job,
+    params: DictData,
+    *,
+    run_id: str | None = None,
+    parent_run_id: str | None = None,
+    result: Result | None = None,
+    event: Event | None = None,
+    raise_error: bool | None = None,
+) -> Result:  # pragma: no cov
+    result: Result = Result.construct_with_rs_or_id(
+        result,
+        run_id=run_id,
+        parent_run_id=parent_run_id,
+        id_logic=(job.id or "not-set"),
+    )
+
+    if event and event.is_set():
+        return result.catch(status=Status.FAILED)
+
+    import requests
+
+    resp = requests.post(
+        job.runs_on.args.host,
+        data={"job": job.model_dump(), "params": params},
+    )
+
+    if resp.status_code != 200:
+        do_raise: bool = dynamic(
+            "job_raise_error", f=raise_error, extras=job.extras
+        )
+        if do_raise:
+            raise JobException(
+                f"Job execution error from request to self-hosted: "
+                f"{job.runs_on.args.host!r}"
+            )
+
+        return result.catch(status=Status.FAILED)
+    return result.catch(status=Status.SUCCESS)