ddeutil-workflow 0.0.14__py3-none-any.whl → 0.0.16__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.14"
+__version__: str = "0.0.16"

ddeutil/workflow/__types.py

@@ -6,21 +6,50 @@
 from __future__ import annotations
 
 import re
+from collections.abc import Iterator
+from dataclasses import dataclass
 from re import (
     IGNORECASE,
     MULTILINE,
     UNICODE,
     VERBOSE,
+    Match,
     Pattern,
 )
-from typing import Any, Union
+from typing import Any, Optional, TypedDict, Union
+
+from typing_extensions import Self
 
 TupleStr = tuple[str, ...]
 DictData = dict[str, Any]
 DictStr = dict[str, str]
 Matrix = dict[str, Union[list[str], list[int]]]
-MatrixInclude = list[dict[str, Union[str, int]]]
-MatrixExclude = list[dict[str, Union[str, int]]]
+
+
+class Context(TypedDict):
+    params: dict[str, Any]
+    jobs: dict[str, Any]
+
+
+@dataclass(frozen=True)
+class CallerRe:
+    """Caller dataclass that catching result from the matching regex with the
+    Re.RE_CALLER value.
+    """
+
+    full: str
+    caller: str
+    caller_prefix: Optional[str]
+    caller_last: str
+    post_filters: str
+
+    @classmethod
+    def from_regex(cls, match: Match[str]) -> Self:
+        """Class construct from matching result.
+
+        :rtype: Self
+        """
+        return cls(full=match.group(0), **match.groupdict())
 
 
 class Re:
@@ -30,28 +59,33 @@ class Re:
     #   Regular expression:
     #       - Version 1:
     #         \${{\s*(?P<caller>[a-zA-Z0-9_.\s'\"\[\]\(\)\-\{}]+?)\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
-    #       - Version 2 (2024-09-30):
-    #         \${{\s*(?P<caller>(?P<caller_prefix>[a-zA-Z_-]+\.)*(?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+))\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
+    #       - Version 2: (2024-09-30):
+    #         \${{\s*(?P<caller>(?P<caller_prefix>(?:[a-zA-Z_-]+\.)*)(?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+))\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
+    #       - Version 3: (2024-10-05):
+    #         \${{\s*(?P<caller>(?P<caller_prefix>(?:[a-zA-Z_-]+\??\.)*)(?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+\??))\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
     #
     #   Examples:
-    #       - ${{ params.asat_dt }}
+    #       - ${{ params.data_dt }}
     #       - ${{ params.source.table }}
+    #       - ${{ params.datetime | fmt('%Y-%m-%d') }}
+    #       - ${{ params.source?.schema }}
     #
     __re_caller: str = r"""
         \$
         {{
             \s*
             (?P<caller>
-                (?P<caller_prefix>[a-zA-Z_-]+\.)*
-                (?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+)
+                (?P<caller_prefix>(?:[a-zA-Z_-]+\??\.)*)
+                (?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+\??)
             )
             \s*
             (?P<post_filters>
                 (?:
-                    \|
-                    \s*
-                    (?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]*)
-                    \s*
+                    \|\s*
+                    (?:
+                        [a-zA-Z0-9_]{3,}
+                        [a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]*
+                    )\s*
                 )*
             )
         }}
@@ -62,7 +96,8 @@ class Re:
 
     # NOTE:
     #   Regular expression:
-    #   ^(?P<path>[^/@]+)/(?P<func>[^@]+)@(?P<tag>.+)$
+    #       - Version 1:
+    #         ^(?P<path>[^/@]+)/(?P<func>[^@]+)@(?P<tag>.+)$
     #
     #   Examples:
     #       - tasks/function@dummy
@@ -78,3 +113,13 @@ class Re:
     RE_TASK_FMT: Pattern = re.compile(
         __re_task_fmt, MULTILINE | IGNORECASE | UNICODE | VERBOSE
     )
+
+    @classmethod
+    def finditer_caller(cls, value) -> Iterator[CallerRe]:
+        """Generate CallerRe object that create from matching object that
+        extract with re.finditer function.
+
+        :rtype: Iterator[CallerRe]
+        """
+        for found in cls.RE_CALLER.finditer(value):
+            yield CallerRe.from_regex(found)

ddeutil/workflow/api.py

@@ -7,7 +7,6 @@ from __future__ import annotations
 
 import asyncio
 import contextlib
-import os
 import uuid
 from collections.abc import AsyncIterator
 from datetime import datetime, timedelta
@@ -15,7 +14,6 @@ from queue import Empty, Queue
 from threading import Thread
 from typing import TypedDict
 
-from ddeutil.core import str2bool
 from dotenv import load_dotenv
 from fastapi import FastAPI
 from fastapi.middleware.gzip import GZipMiddleware
@@ -23,6 +21,7 @@ from fastapi.responses import UJSONResponse
 from pydantic import BaseModel
 
 from .__about__ import __version__
+from .conf import config
 from .log import get_logger
 from .repeat import repeat_at, repeat_every
 from .scheduler import WorkflowTaskData
@@ -131,12 +130,12 @@ async def message_upper(payload: Payload):
     return await get_result(request_id)
 
 
-if str2bool(os.getenv("WORKFLOW_API_ENABLE_ROUTE_WORKFLOW", "true")):
+if config.enable_route_workflow:
     from .route import workflow
 
     app.include_router(workflow)
 
-if str2bool(os.getenv("WORKFLOW_API_ENABLE_ROUTE_SCHEDULE", "true")):
+if config.enable_route_schedule:
     from .route import schedule
     from .scheduler import workflow_task
 

ddeutil/workflow/cli.py

@@ -6,15 +6,14 @@
 from __future__ import annotations
 
 import json
-import os
 from datetime import datetime
 from enum import Enum
 from typing import Annotated, Optional
-from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2list
 from typer import Argument, Option, Typer
 
+from .conf import config
 from .log import get_logger
 
 logger = get_logger("ddeutil.workflow")
@@ -73,9 +72,7 @@ def schedule(
     excluded: list[str] = str2list(excluded) if excluded else []
     externals: str = externals or "{}"
     if stop:
-        stop: datetime = stop.astimezone(
-            tz=ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
-        )
+        stop: datetime = stop.astimezone(tz=config.tz)
 
     from .scheduler import workflow_runner
 

ddeutil/workflow/conf.py

@@ -5,37 +5,314 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
+import json
 import os
+from collections.abc import Iterator
+from datetime import timedelta
+from functools import cached_property
+from pathlib import Path
+from typing import Any, TypeVar
 from zoneinfo import ZoneInfo
 
-from ddeutil.core import str2bool
+from ddeutil.core import import_string, str2bool
+from ddeutil.io import Paths, PathSearch, YamlFlResolve
 from dotenv import load_dotenv
+from pydantic import BaseModel, Field
+from pydantic.functional_validators import model_validator
 
 load_dotenv()
 env = os.getenv
+DictData = dict[str, Any]
+AnyModel = TypeVar("AnyModel", bound=BaseModel)
+AnyModelType = type[AnyModel]
 
 
 class Config:
+    """Config object for keeping application configuration on current session
+    without changing when if the application still running.
+    """
+
     # NOTE: Core
+    root_path: Path = Path(os.getenv("WORKFLOW_ROOT_PATH", "."))
     tz: ZoneInfo = ZoneInfo(env("WORKFLOW_CORE_TIMEZONE", "UTC"))
+    workflow_id_simple_mode: bool = str2bool(
+        os.getenv("WORKFLOW_CORE_WORKFLOW_ID_SIMPLE_MODE", "true")
+    )
+
+    # NOTE: Logging
+    debug: bool = str2bool(os.getenv("WORKFLOW_LOG_DEBUG_MODE", "true"))
+    enable_write_log: bool = str2bool(
+        os.getenv("WORKFLOW_LOG_ENABLE_WRITE", "false")
+    )
 
     # NOTE: Stage
     stage_raise_error: bool = str2bool(
-        env("WORKFLOW_CORE_STAGE_RAISE_ERROR", "true")
+        env("WORKFLOW_CORE_STAGE_RAISE_ERROR", "false")
     )
     stage_default_id: bool = str2bool(
         env("WORKFLOW_CORE_STAGE_DEFAULT_ID", "false")
     )
 
+    # NOTE: Job
+    job_default_id: bool = str2bool(
+        env("WORKFLOW_CORE_JOB_DEFAULT_ID", "false")
+    )
+
     # NOTE: Workflow
     max_job_parallel: int = int(env("WORKFLOW_CORE_MAX_JOB_PARALLEL", "2"))
+    max_poking_pool_worker: int = int(
+        os.getenv("WORKFLOW_CORE_MAX_NUM_POKING", "4")
+    )
+
+    # NOTE: Schedule App
+    max_schedule_process: int = int(env("WORKFLOW_APP_MAX_PROCESS", "2"))
+    max_schedule_per_process: int = int(
+        env("WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS", "100")
+    )
+    __stop_boundary_delta: str = env(
+        "WORKFLOW_APP_STOP_BOUNDARY_DELTA", '{"minutes": 5, "seconds": 20}'
+    )
+
+    # NOTE: API
+    enable_route_workflow: bool = str2bool(
+        os.getenv("WORKFLOW_API_ENABLE_ROUTE_WORKFLOW", "true")
+    )
+    enable_route_schedule: bool = str2bool(
+        os.getenv("WORKFLOW_API_ENABLE_ROUTE_SCHEDULE", "true")
+    )
 
     def __init__(self):
         if self.max_job_parallel < 0:
             raise ValueError(
-                f"MAX_JOB_PARALLEL should more than 0 but got "
+                f"``MAX_JOB_PARALLEL`` should more than 0 but got "
                 f"{self.max_job_parallel}."
             )
+        try:
+            self.stop_boundary_delta: timedelta = timedelta(
+                **json.loads(self.__stop_boundary_delta)
+            )
+        except Exception as err:
+            raise ValueError(
+                "Config ``WORKFLOW_APP_STOP_BOUNDARY_DELTA`` can not parsing to"
+                f"timedelta with {self.__stop_boundary_delta}."
+            ) from err
+
+    def refresh_dotenv(self):
+        """Reload environment variables from the current stage."""
+        self.tz: ZoneInfo = ZoneInfo(env("WORKFLOW_CORE_TIMEZONE", "UTC"))
+        self.stage_raise_error: bool = str2bool(
+            env("WORKFLOW_CORE_STAGE_RAISE_ERROR", "false")
+        )
+
+
+class Engine(BaseModel):
+    """Engine Pydantic Model for keeping application path."""
+
+    paths: Paths = Field(default_factory=Paths)
+    registry: list[str] = Field(
+        default_factory=lambda: ["ddeutil.workflow"],  # pragma: no cover
+    )
+    registry_filter: list[str] = Field(
+        default_factory=lambda: ["ddeutil.workflow.utils"],  # pragma: no cover
+    )
+
+    @model_validator(mode="before")
+    def __prepare_registry(cls, values: DictData) -> DictData:
+        """Prepare registry value that passing with string type. It convert the
+        string type to list of string.
+        """
+        if (_regis := values.get("registry")) and isinstance(_regis, str):
+            values["registry"] = [_regis]
+        if (_regis_filter := values.get("registry_filter")) and isinstance(
+            _regis_filter, str
+        ):
+            values["registry_filter"] = [_regis_filter]
+        return values
+
+
+class ConfParams(BaseModel):
+    """Params Model"""
+
+    engine: Engine = Field(
+        default_factory=Engine,
+        description="A engine mapping values.",
+    )
+
+
+def load_config() -> ConfParams:
+    """Load Config data from ``workflows-conf.yaml`` file.
+
+    Configuration Docs:
+    ---
+    :var engine.registry:
+    :var engine.registry_filter:
+    :var paths.root:
+    :var paths.conf:
+    """
+    root_path: str = config.root_path
+
+    regis: list[str] = ["ddeutil.workflow"]
+    if regis_env := os.getenv("WORKFLOW_CORE_REGISTRY"):
+        regis = [r.strip() for r in regis_env.split(",")]
+
+    regis_filter: list[str] = ["ddeutil.workflow.utils"]
+    if regis_filter_env := os.getenv("WORKFLOW_CORE_REGISTRY_FILTER"):
+        regis_filter = [r.strip() for r in regis_filter_env.split(",")]
+
+    conf_path: str = (
+        f"{root_path}/{conf_env}"
+        if (conf_env := os.getenv("WORKFLOW_CORE_PATH_CONF"))
+        else None
+    )
+    return ConfParams.model_validate(
+        obj={
+            "engine": {
+                "registry": regis,
+                "registry_filter": regis_filter,
+                "paths": {
+                    "root": root_path,
+                    "conf": conf_path,
+                },
+            },
+        }
+    )
+
+
+class SimLoad:
+    """Simple Load Object that will search config data by given some identity
+    value like name of workflow or on.
+
+    :param name: A name of config data that will read by Yaml Loader object.
+    :param params: A Params model object.
+    :param externals: An external parameters
+
+    Noted:
+
+        The config data should have ``type`` key for modeling validation that
+    make this loader know what is config should to do pass to.
+
+        ... <identity-key>:
+        ...     type: <importable-object>
+        ...     <key-data>: <value-data>
+        ...     ...
+
+    """
+
+    def __init__(
+        self,
+        name: str,
+        params: ConfParams,
+        externals: DictData | None = None,
+    ) -> None:
+        self.data: DictData = {}
+        for file in PathSearch(params.engine.paths.conf).files:
+            if any(file.suffix.endswith(s) for s in (".yml", ".yaml")) and (
+                data := YamlFlResolve(file).read().get(name, {})
+            ):
+                self.data = data
+
+        # VALIDATE: check the data that reading should not empty.
+        if not self.data:
+            raise ValueError(f"Config {name!r} does not found on conf path")
+
+        self.conf_params: ConfParams = params
+        self.externals: DictData = externals or {}
+        self.data.update(self.externals)
+
+    @classmethod
+    def finds(
+        cls,
+        obj: object,
+        params: ConfParams,
+        *,
+        include: list[str] | None = None,
+        exclude: list[str] | None = None,
+    ) -> Iterator[tuple[str, DictData]]:
+        """Find all data that match with object type in config path. This class
+        method can use include and exclude list of identity name for filter and
+        adds-on.
+
+        :param obj: A object that want to validate matching before return.
+        :param params:
+        :param include:
+        :param exclude:
+        :rtype: Iterator[tuple[str, DictData]]
+        """
+        exclude: list[str] = exclude or []
+        for file in PathSearch(params.engine.paths.conf).files:
+            if any(file.suffix.endswith(s) for s in (".yml", ".yaml")) and (
+                values := YamlFlResolve(file).read()
+            ):
+                for key, data in values.items():
+                    if key in exclude:
+                        continue
+                    if issubclass(get_type(data["type"], params), obj) and (
+                        include is None or all(i in data for i in include)
+                    ):
+                        yield key, data
+
+    @cached_property
+    def type(self) -> AnyModelType:
+        """Return object of string type which implement on any registry. The
+        object type.
+
+        :rtype: AnyModelType
+        """
+        if not (_typ := self.data.get("type")):
+            raise ValueError(
+                f"the 'type' value: {_typ} does not exists in config data."
+            )
+        return get_type(_typ, self.conf_params)
+
+
+class Loader(SimLoad):
+    """Loader Object that get the config `yaml` file from current path.
+
+    :param name: A name of config data that will read by Yaml Loader object.
+    :param externals: An external parameters
+    """
+
+    @classmethod
+    def finds(
+        cls,
+        obj: object,
+        *,
+        include: list[str] | None = None,
+        exclude: list[str] | None = None,
+        **kwargs,
+    ) -> DictData:
+        """Override the find class method from the Simple Loader object.
+
+        :param obj: A object that want to validate matching before return.
+        :param include:
+        :param exclude:
+        """
+        return super().finds(
+            obj=obj, params=load_config(), include=include, exclude=exclude
+        )
+
+    def __init__(self, name: str, externals: DictData) -> None:
+        super().__init__(name, load_config(), externals)
+
+
+def get_type(t: str, params: ConfParams) -> AnyModelType:
+    """Return import type from string importable value in the type key.
+
+    :param t: A importable type string.
+    :param params: A config parameters that use registry to search this
+        type.
+    :rtype: AnyModelType
+    """
+    try:
+        # NOTE: Auto adding module prefix if it does not set
+        return import_string(f"ddeutil.workflow.{t}")
+    except ModuleNotFoundError:
+        for registry in params.engine.registry:
+            try:
+                return import_string(f"{registry}.{t}")
+            except ModuleNotFoundError:
+                continue
+        return import_string(f"{t}")
 
 
 config = Config()

ddeutil/workflow/job.py

@@ -4,6 +4,9 @@
 # license information.
 # ------------------------------------------------------------------------------
 """Job Model that use for keeping stages and node that running its stages.
+The job handle the lineage of stages and location of execution of stages that
+mean the job model able to define ``runs-on`` key that allow you to run this
+job.
 """
 from __future__ import annotations
 
@@ -19,21 +22,15 @@ from concurrent.futures import (
 from functools import lru_cache
 from textwrap import dedent
 from threading import Event
-from typing import Optional
+from typing import Optional, Union
 
 from ddeutil.core import freeze_args
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
-from .__types import (
-    DictData,
-    DictStr,
-    Matrix,
-    MatrixExclude,
-    MatrixInclude,
-    TupleStr,
-)
+from .__types import DictData, DictStr, Matrix, TupleStr
+from .conf import config
 from .exceptions import (
     JobException,
     StageException,
@@ -51,6 +48,8 @@ from .utils import (
 )
 
 logger = get_logger("ddeutil.workflow")
+MatrixInclude = list[dict[str, Union[str, int]]]
+MatrixExclude = list[dict[str, Union[str, int]]]
 
 
 __all__: TupleStr = (
@@ -262,7 +261,7 @@ class Job(BaseModel):
     )
 
     @model_validator(mode="before")
-    def __prepare_keys(cls, values: DictData) -> DictData:
+    def __prepare_keys__(cls, values: DictData) -> DictData:
         """Rename key that use dash to underscore because Python does not
         support this character exist in any variable name.
 
@@ -273,12 +272,12 @@ class Job(BaseModel):
         return values
 
     @field_validator("desc", mode="after")
-    def ___prepare_desc(cls, value: str) -> str:
+    def ___prepare_desc__(cls, value: str) -> str:
         """Prepare description string that was created on a template."""
         return dedent(value)
 
     @model_validator(mode="after")
-    def __prepare_running_id(self) -> Self:
+    def __prepare_running_id__(self) -> Self:
         """Prepare the job running ID.
 
         :rtype: Self
@@ -319,31 +318,43 @@ class Job(BaseModel):
             For example of setting output method, If you receive execute output
         and want to set on the `to` like;
 
-            ... (i)   output: {'strategy01': bar, 'strategy02': bar}
-            ... (ii)  to: {}
+            ... (i)   output: {'strategy-01': bar, 'strategy-02': bar}
+            ... (ii)  to: {'jobs': {}}
 
         The result of the `to` variable will be;
 
             ... (iii) to: {
-                            'strategies': {
-                                'strategy01': bar, 'strategy02': bar
+                        'jobs': {
+                            '<job-id>': {
+                                'strategies': {
+                                    'strategy-01': bar,
+                                    'strategy-02': bar
+                                }
                             }
                         }
+                    }
 
         :param output: An output context.
         :param to: A context data that want to add output result.
         :rtype: DictData
         """
-        if self.id is None:
+        if self.id is None and not config.job_default_id:
             raise JobException(
-                "This job do not set the ID before setting output."
+                "This job do not set the ID before setting execution output."
             )
 
-        to[self.id] = (
+        # NOTE: Create jobs key to receive an output from the job execution.
+        if "jobs" not in to:
+            to["jobs"] = {}
+
+        # NOTE: If the job ID did not set, it will use index of jobs key
+        #   instead.
+        _id: str = self.id or str(len(to["jobs"]) + 1)
+
+        logger.debug(f"({self.run_id}) [JOB]: Set outputs on: {_id}")
+        to["jobs"][_id] = (
             {"strategies": output}
             if self.strategy.is_set()
-            # NOTE:
-            #   This is the best way to get single key from dict.
             else output[next(iter(output))]
         )
         return to
@@ -382,6 +393,7 @@ class Job(BaseModel):
         #       "params": { ... },      <== Current input params
         #       "jobs": { ... },        <== Current input params
         #       "matrix": { ... }       <== Current strategy value
+        #       "stages": { ... }       <== Catching stage outputs
         #   }
         #
         context: DictData = copy.deepcopy(params)
@@ -489,15 +501,18 @@ class Job(BaseModel):
         :param params: An input parameters that use on job execution.
         :rtype: Result
         """
-        context: DictData = {}
+
+        # NOTE: I use this condition because this method allow passing empty
+        #   params and I do not want to create new dict object.
         params: DictData = {} if params is None else params
+        context: DictData = {}
 
         # NOTE: Normal Job execution without parallel strategy.
         if (not self.strategy.is_set()) or self.strategy.max_parallel == 1:
             for strategy in self.strategy.make():
                 rs: Result = self.execute_strategy(
                     strategy=strategy,
-                    params=copy.deepcopy(params),
+                    params=params,
                 )
                 context.update(rs.context)
             return Result(
@@ -505,11 +520,15 @@ class Job(BaseModel):
                 context=context,
             )
 
-        # NOTE: Create event for cancel executor stop running.
+        # NOTE: Create event for cancel executor by trigger stop running event.
         event: Event = Event()
 
+        # IMPORTANT: Start running strategy execution by multithreading because
+        #   it will running by strategy values without waiting previous
+        #   execution.
         with ThreadPoolExecutor(
-            max_workers=self.strategy.max_parallel
+            max_workers=self.strategy.max_parallel,
+            thread_name_prefix="job_strategy_exec_",
         ) as executor:
             futures: list[Future] = [
                 executor.submit(