ddeutil-workflow 0.0.27__py3-none-any.whl → 0.0.29__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.27"
+__version__: str = "0.0.29"

ddeutil/workflow/__init__.py

@@ -4,6 +4,7 @@
 # license information.
 # ------------------------------------------------------------------------------
 from .__cron import CronJob, CronRunner
+from .__types import Re
 from .conf import (
     Config,
     Loader,

ddeutil/workflow/__types.py

@@ -27,6 +27,8 @@ Matrix = dict[str, Union[list[str], list[int]]]
 
 
 class Context(TypedDict):
+    """TypeDict support the Context."""
+
     params: dict[str, Any]
     jobs: dict[str, Any]
 
@@ -71,14 +73,14 @@ class Re:
     #       - ${{ params.source?.schema }}
     #
     __re_caller: str = r"""
-        \$
-        {{
-            \s*
+        \$                                                      # start with $
+        {{                                                      # value open with {{
+            \s*                                                 # whitespace or not
             (?P<caller>
                 (?P<caller_prefix>(?:[a-zA-Z_-]+\??\.)*)
                 (?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+\??)
             )
-            \s*
+            \s*                                                 # whitespace or not
             (?P<post_filters>
                 (?:
                     \|\s*
@@ -88,7 +90,7 @@ class Re:
                     )\s*
                 )*
             )
-        }}
+        }}                                                      # value close with }}
     """
     RE_CALLER: Pattern = re.compile(
         __re_caller, MULTILINE | IGNORECASE | UNICODE | VERBOSE
@@ -103,13 +105,13 @@ class Re:
     #       - tasks/function@dummy
     #
     __re_task_fmt: str = r"""
-        ^
+        ^                               # start task format
             (?P<path>[^/@]+)
-            /
+            /                           # start get function with /
             (?P<func>[^@]+)
-            @
+            @                           # start tag with @
             (?P<tag>.+)
-        $
+        $                               # end task format
     """
     RE_TASK_FMT: Pattern = re.compile(
         __re_task_fmt, MULTILINE | IGNORECASE | UNICODE | VERBOSE

ddeutil/workflow/api/api.py

@@ -5,25 +5,21 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
-import asyncio
 import contextlib
-import uuid
 from collections.abc import AsyncIterator
 from datetime import datetime, timedelta
-from queue import Empty, Queue
 from typing import TypedDict
 
 from dotenv import load_dotenv
 from fastapi import FastAPI
 from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import UJSONResponse
-from pydantic import BaseModel
 
 from ..__about__ import __version__
 from ..conf import config, get_logger
 from ..scheduler import ReleaseThread, ReleaseThreads
 from ..workflow import WorkflowQueue, WorkflowTask
-from .repeat import repeat_at, repeat_every
+from .repeat import repeat_at
 
 load_dotenv()
 logger = get_logger("ddeutil.workflow")
@@ -32,11 +28,6 @@ logger = get_logger("ddeutil.workflow")
 class State(TypedDict):
     """TypeDict for State of FastAPI application."""
 
-    # NOTE: For upper queue route.
-    upper_queue: Queue
-    upper_result: dict[str, str]
-
-    # NOTE: For schedule listener.
     scheduler: list[str]
     workflow_threads: ReleaseThreads
     workflow_tasks: list[WorkflowTask]
@@ -46,15 +37,11 @@ class State(TypedDict):
 @contextlib.asynccontextmanager
 async def lifespan(a: FastAPI) -> AsyncIterator[State]:
     """Lifespan function for the FastAPI application."""
-    a.state.upper_queue = Queue()
-    a.state.upper_result = {}
     a.state.scheduler = []
     a.state.workflow_threads = {}
     a.state.workflow_tasks = []
     a.state.workflow_queue = {}
 
-    await asyncio.create_task(broker_upper_messages())
-
     yield {
         "upper_queue": a.state.upper_queue,
         "upper_result": a.state.upper_result,
@@ -87,50 +74,11 @@ app = FastAPI(
 app.add_middleware(GZipMiddleware, minimum_size=1000)
 
 
-@repeat_every(seconds=10)
-async def broker_upper_messages():
-    """Broker for receive message from the `/upper` path and change it to upper
-    case. This broker use interval running in background every 10 seconds.
-    """
-    for _ in range(10):
-        try:
-            obj = app.state.upper_queue.get_nowait()
-            app.state.upper_result[obj["request_id"]] = obj["text"].upper()
-            logger.info(f"Upper message: {app.state.upper_result}")
-        except Empty:
-            pass
-    await asyncio.sleep(0.0001)
-
-
-class Payload(BaseModel):
-    text: str
-
-
-async def get_result(request_id: str) -> dict[str, str]:
-    """Get data from output dict that global."""
-    while True:
-        if request_id in app.state.upper_result:
-            result: str = app.state.upper_result[request_id]
-            del app.state.upper_result[request_id]
-            return {"message": result}
-        await asyncio.sleep(0.0025)
-
-
 @app.get("/")
 async def health():
     return {"message": "Workflow API already start up"}
 
 
-@app.post(f"{config.prefix_path}/upper")
-async def message_upper(payload: Payload):
-    """Convert message from any case to the upper case."""
-    request_id: str = str(uuid.uuid4())
-    app.state.upper_queue.put(
-        {"text": payload.text, "request_id": request_id},
-    )
-    return await get_result(request_id)
-
-
 # NOTE: Enable the workflow route.
 if config.enable_route_workflow:
     from .route import workflow_route

ddeutil/workflow/conf.py

@@ -13,7 +13,7 @@ from collections.abc import Iterator
 from datetime import datetime, timedelta
 from functools import cached_property, lru_cache
 from pathlib import Path
-from typing import ClassVar, Optional, Union
+from typing import ClassVar, Optional, TypeVar, Union
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
@@ -24,9 +24,11 @@ from typing_extensions import Self
 
 from .__types import DictData, TupleStr
 
+PREFIX: str = "WORKFLOW"
+
 
 def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
-    return os.getenv(f"WORKFLOW_{var}", default)
+    return os.getenv(f"{PREFIX}_{var.upper().replace(' ', '_')}", default)
 
 
 def glob_files(path: Path) -> Iterator[Path]:  # pragma: no cov
@@ -37,6 +39,7 @@ __all__: TupleStr = (
     "env",
     "get_logger",
     "get_log",
+    "C",
     "Config",
     "SimLoad",
     "Loader",
@@ -79,15 +82,43 @@ def get_logger(name: str):
     return lg
 
 
-class Config:  # pragma: no cov
-    """Config object for keeping application configuration on current session
+class BaseConfig:  # pragma: no cov
+    """BaseConfig object inheritable."""
+
+    __slots__ = ()
+
+    @property
+    def root_path(self) -> Path:
+        """Root path or the project path.
+
+        :rtype: Path
+        """
+        return Path(os.getenv("ROOT_PATH", "."))
+
+    @property
+    def conf_path(self) -> Path:
+        """Config path that use root_path class argument for this construction.
+
+        :rtype: Path
+        """
+        return self.root_path / os.getenv("CONF_PATH", "conf")
+
+
+class Config(BaseConfig):  # pragma: no cov
+    """Config object for keeping core configurations on the current session
     without changing when if the application still running.
+
+        The config value can change when you call that config property again.
     """
 
     # NOTE: Core
     @property
     def root_path(self) -> Path:
-        return Path(env("ROOT_PATH", "."))
+        """Root path or the project path.
+
+        :rtype: Path
+        """
+        return Path(env("CORE_ROOT_PATH", "."))
 
     @property
     def conf_path(self) -> Path:
@@ -95,7 +126,7 @@ class Config:  # pragma: no cov
 
         :rtype: Path
         """
-        return self.root_path / env("CORE_PATH_CONF", "conf")
+        return self.root_path / env("CORE_CONF_PATH", "conf")
 
     @property
     def tz(self) -> ZoneInfo:
@@ -108,7 +139,7 @@ class Config:  # pragma: no cov
     # NOTE: Register
     @property
     def regis_hook(self) -> list[str]:
-        regis_hook_str: str = env("CORE_REGISTRY", "src")
+        regis_hook_str: str = env("CORE_REGISTRY", ".")
         return [r.strip() for r in regis_hook_str.split(",")]
 
     @property
@@ -214,6 +245,9 @@ class Config:  # pragma: no cov
         return str2bool(env("API_ENABLE_ROUTE_SCHEDULE", "true"))
 
 
+C = TypeVar("C", bound=BaseConfig)
+
+
 class SimLoad:
     """Simple Load Object that will search config data by given some identity
     value like name of workflow or on.
@@ -237,7 +271,7 @@ class SimLoad:
     def __init__(
         self,
         name: str,
-        conf: Config,
+        conf: C,
         externals: DictData | None = None,
     ) -> None:
         self.data: DictData = {}
@@ -250,7 +284,7 @@ class SimLoad:
         if not self.data:
             raise ValueError(f"Config {name!r} does not found on conf path")
 
-        self.conf: Config = conf
+        self.conf: C = conf
         self.externals: DictData = externals or {}
         self.data.update(self.externals)
 
@@ -258,7 +292,7 @@ class SimLoad:
     def finds(
         cls,
         obj: object,
-        conf: Config,
+        conf: C,
         *,
         included: list[str] | None = None,
         excluded: list[str] | None = None,
@@ -267,7 +301,7 @@ class SimLoad:
         method can use include and exclude list of identity name for filter and
         adds-on.
 
-        :param obj: A object that want to validate matching before return.
+        :param obj: An object that want to validate matching before return.
         :param conf: A config object.
         :param included:
         :param excluded:
@@ -332,7 +366,7 @@ class Loader(SimLoad):
     ) -> Iterator[tuple[str, DictData]]:
         """Override the find class method from the Simple Loader object.
 
-        :param obj: A object that want to validate matching before return.
+        :param obj: An object that want to validate matching before return.
         :param included:
         :param excluded:
 
@@ -349,7 +383,7 @@ class Loader(SimLoad):
 class BaseLog(BaseModel, ABC):
     """Base Log Pydantic Model with abstraction class property that implement
     only model fields. This model should to use with inherit to logging
-    sub-class like file, sqlite, etc.
+    subclass like file, sqlite, etc.
     """
 
     name: str = Field(description="A workflow name.")
@@ -357,9 +391,7 @@ class BaseLog(BaseModel, ABC):
     type: str = Field(description="A running type before logging.")
     context: DictData = Field(
         default_factory=dict,
-        description=(
-            "A context data that receive from a workflow execution result.",
-        ),
+        description="A context that receive from a workflow execution result.",
     )
     parent_run_id: Optional[str] = Field(default=None)
     run_id: str
@@ -386,7 +418,7 @@ class BaseLog(BaseModel, ABC):
 
 class FileLog(BaseLog):
     """File Log Pydantic Model that use to saving log data from result of
-    workflow execution. It inherit from BaseLog model that implement the
+    workflow execution. It inherits from BaseLog model that implement the
     ``self.save`` method for file.
     """
 
@@ -526,7 +558,7 @@ class SQLiteLog(BaseLog):  # pragma: no cov
         primary key ( run_id )
         """
 
-    def save(self, excluded: list[str] | None) -> None:
+    def save(self, excluded: list[str] | None) -> SQLiteLog:
         """Save logging data that receive a context data from a workflow
         execution result.
         """
@@ -549,7 +581,7 @@ Log = Union[
 ]
 
 
-def get_log() -> Log:  # pragma: no cov
+def get_log() -> type[Log]:  # pragma: no cov
     if config.log_path.is_file():
         return SQLiteLog
     return FileLog

ddeutil/workflow/cron.py

@@ -32,10 +32,10 @@ def interval2crontab(
 ) -> str:
     """Return the crontab string that was generated from specific values.
 
-    :param interval: A interval value that is one of 'daily', 'weekly', or
+    :param interval: An interval value that is one of 'daily', 'weekly', or
         'monthly'.
     :param day: A day value that will be day of week. The default value is
-        monday if it be weekly interval.
+        monday if it is weekly interval.
     :param time: A time value that passing with format '%H:%M'.
 
     Examples:
@@ -50,9 +50,9 @@ def interval2crontab(
     """
     d: str = "*"
     if interval == "weekly":
-        d = WEEKDAYS[(day or "monday")[:3].title()]
+        d = str(WEEKDAYS[(day or "monday")[:3].title()])
     elif interval == "monthly" and day:
-        d = WEEKDAYS[day[:3].title()]
+        d = str(WEEKDAYS[day[:3].title()])
 
     h, m = tuple(
         i.lstrip("0") if i != "00" else "0" for i in time.split(":", maxsplit=1)
@@ -95,7 +95,7 @@ class On(BaseModel):
 
         :param value: A mapping value that will generate crontab before create
             schedule model.
-        :param externals: A extras external parameter that will keep in extras.
+        :param externals: An extras external parameter that will keep in extras.
         """
         passing: DictStr = {}
         if "timezone" in value:
@@ -114,8 +114,8 @@ class On(BaseModel):
         """Constructor from the name of config that will use loader object for
         getting the data.
 
-        :param name: A name of config that will getting from loader.
-        :param externals: A extras external parameter that will keep in extras.
+        :param name: A name of config that will get from loader.
+        :param externals: An extras external parameter that will keep in extras.
         """
         externals: DictData = externals or {}
         loader: Loader = Loader(name, externals=externals)

ddeutil/workflow/exceptions.py

@@ -4,7 +4,7 @@
 # license information.
 # ------------------------------------------------------------------------------
 """Exception objects for this package do not do anything because I want to
-create the lightweight workflow package. So, this module do just a exception
+create the lightweight workflow package. So, this module do just an exception
 annotate for handle error only.
 """
 from __future__ import annotations

ddeutil/workflow/hook.py

@@ -50,6 +50,7 @@ def tag(
     :param: name: A tag name for make different use-case of a function.
     :param: alias: A alias function name that keeping in registries. If this
         value does not supply, it will use original function name from __name__.
+
     :rtype: Callable[P, TagFunc]
     """
 
@@ -58,7 +59,7 @@ def tag(
         func.name = alias or func.__name__.replace("_", "-")
 
         @wraps(func)
-        def wrapped(*args, **kwargs):
+        def wrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
             # NOTE: Able to do anything before calling hook function.
             return func(*args, **kwargs)
 
@@ -74,10 +75,11 @@ def make_registry(submodule: str) -> dict[str, Registry]:
     """Return registries of all functions that able to called with task.
 
     :param submodule: A module prefix that want to import registry.
+
     :rtype: dict[str, Registry]
     """
     rs: dict[str, Registry] = {}
-    for module in config.regis_hook:
+    for module in config.regis_hook | ["ddeutil.vendors"]:
         # NOTE: try to sequential import task functions
         try:
             importer = import_module(f"{module}.{submodule}")
@@ -87,9 +89,12 @@ def make_registry(submodule: str) -> dict[str, Registry]:
         for fstr, func in inspect.getmembers(importer, inspect.isfunction):
             # NOTE: check function attribute that already set tag by
             #   ``utils.tag`` decorator.
-            if not hasattr(func, "tag"):
+            if not (hasattr(func, "tag") and hasattr(func, "name")):
                 continue
 
+            # NOTE: Define type of the func value.
+            func: TagFunc
+
             # NOTE: Create new register name if it not exists
             if func.name not in rs:
                 rs[func.name] = {func.tag: lazy(f"{module}.{submodule}.{fstr}")}
@@ -124,9 +129,21 @@ def extract_hook(hook: str) -> Callable[[], TagFunc]:
     :raise NotImplementedError: When the searching hook's function result does
         not exist in the registry.
     :raise NotImplementedError: When the searching hook's tag result does not
-        exists in the registry with its function key.
+        exist in the registry with its function key.
 
     :param hook: A hook value that able to match with Task regex.
+
+        The format of hook value should contain 3 regular expression groups
+    which match with the below config format:
+
+        >>> "^(?P<path>[^/@]+)/(?P<func>[^@]+)@(?P<tag>.+)$"
+
+    Examples:
+        >>> extract_hook("tasks/el-postgres-to-delta@polars")
+        ...
+        >>> extract_hook("tasks/return-type-not-valid@raise")
+        ...
+
     :rtype: Callable[[], TagFunc]
     """
     if not (found := Re.RE_TASK_FMT.search(hook)):

ddeutil/workflow/job.py

@@ -83,7 +83,7 @@ def make(
     if len(matrix) == 0:
         return [{}]
 
-    # NOTE: Remove matrix that exists on the exclude.
+    # NOTE: Remove matrix that exists on the excluded.
     final: list[DictStr] = []
     for r in cross_product(matrix=matrix):
         if any(
@@ -101,7 +101,7 @@ def make(
     add: list[DictStr] = []
     for inc in include:
         # VALIDATE:
-        #   Validate any key in include list should be a subset of some one
+        #   Validate any key in include list should be a subset of someone
         #   in matrix.
         if all(not (set(inc.keys()) <= set(m.keys())) for m in final):
             raise ValueError(
@@ -128,9 +128,9 @@ class Strategy(BaseModel):
     special job with combination of matrix data.
 
         This model does not be the part of job only because you can use it to
-    any model object. The propose of this model is generate metrix result that
-    comming from combination logic with any matrix values for running it with
-    parallelism.
+    any model object. The objective of this model is generating metrix result
+    that comming from combination logic with any matrix values for running it
+    with parallelism.
 
         [1, 2, 3] x [a, b] --> [1a], [1b], [2a], [2b], [3a], [3b]
 
@@ -180,7 +180,7 @@ class Strategy(BaseModel):
         """Rename key that use dash to underscore because Python does not
         support this character exist in any variable name.
 
-        :param values: A parsing values to this models
+        :param values: A parsing values to these models
         :rtype: DictData
         """
         dash2underscore("max-parallel", values)
@@ -226,7 +226,7 @@ class Job(BaseModel):
     """Job Pydantic model object (short descripte: a group of stages).
 
         This job model allow you to use for-loop that call matrix strategy. If
-    you pass matrix mapping and it able to generate, you will see it running
+    you pass matrix mapping, and it is able to generate, you will see it running
     with loop of matrix values.
 
     Data Validate:
@@ -355,7 +355,7 @@ class Job(BaseModel):
         return all(need in jobs for need in self.needs)
 
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
-        """Set an outputs from execution process to the receive context. The
+        """Set an outputs from execution process to the received context. The
         result from execution will pass to value of ``strategies`` key.
 
             For example of setting output method, If you receive execute output
@@ -420,7 +420,7 @@ class Job(BaseModel):
         strategy and return with context of this strategy data.
 
             The result of this execution will return result with strategy ID
-        that generated from the `gen_id` function with a input strategy value.
+        that generated from the `gen_id` function with an input strategy value.
 
         :raise JobException: If it has any error from ``StageException`` or
             ``UtilException``.
@@ -429,7 +429,7 @@ class Job(BaseModel):
             This value will pass to the `matrix` key for templating.
         :param params: A dynamic parameters that will deepcopy to the context.
         :param run_id: A job running ID for this strategy execution.
-        :param event: An manger event that pass to the PoolThreadExecutor.
+        :param event: An event manager that pass to the PoolThreadExecutor.
 
         :rtype: Result
         """
@@ -496,7 +496,7 @@ class Job(BaseModel):
             # PARAGRAPH:
             #
             #       I do not use below syntax because `params` dict be the
-            #   reference memory pointer and it was changed when I action
+            #   reference memory pointer, and it was changed when I action
             #   anything like update or re-construct this.
             #
             #       ... params |= stage.execute(params=params)
@@ -513,7 +513,9 @@ class Job(BaseModel):
             #
             try:
                 stage.set_outputs(
-                    stage.execute(params=context, run_id=run_id).context,
+                    stage.handler_execute(
+                        params=context, run_id=run_id
+                    ).context,
                     to=context,
                 )
             except (StageException, UtilException) as err:
@@ -566,7 +568,7 @@ class Job(BaseModel):
         run_id: str = run_id or gen_id(self.id or "", unique=True)
         context: DictData = {}
 
-        # NOTE: Normal Job execution without parallel strategy matrix. It use
+        # NOTE: Normal Job execution without parallel strategy matrix. It uses
         #   for-loop to control strategy execution sequentially.
         if (not self.strategy.is_set()) or self.strategy.max_parallel == 1:
             for strategy in self.strategy.make():
@@ -585,8 +587,7 @@ class Job(BaseModel):
         event: Event = Event()
 
         # IMPORTANT: Start running strategy execution by multithreading because
-        #   it will running by strategy values without waiting previous
-        #   execution.
+        #   it will run by strategy values without waiting previous execution.
         with ThreadPoolExecutor(
             max_workers=self.strategy.max_parallel,
             thread_name_prefix="job_strategy_exec_",
@@ -618,7 +619,7 @@ class Job(BaseModel):
         timeout: int = 1800,
     ) -> Result:
         """Job parallel pool futures catching with fail-fast mode. That will
-        stop and set event on all not done futures if it receive the first
+        stop and set event on all not done futures if it receives the first
         exception from all running futures.
 
         :param event: An event manager instance that able to set stopper on the

ddeutil/workflow/params.py

@@ -75,7 +75,7 @@ class DatetimeParam(DefaultParam):
     default: datetime = Field(default_factory=get_dt_now)
 
     def receive(self, value: str | datetime | date | None = None) -> datetime:
-        """Receive value that match with datetime. If a input value pass with
+        """Receive value that match with datetime. If an input value pass with
         None, it will use default value instead.
 
         :param value: A value that want to validate with datetime parameter
@@ -98,7 +98,7 @@ class DatetimeParam(DefaultParam):
             return datetime.fromisoformat(value)
         except ValueError:
             raise ParamValueException(
-                f"Invalid isoformat string: {value!r}"
+                f"Invalid the ISO format string: {value!r}"
             ) from None
 
 
@@ -158,7 +158,7 @@ class ChoiceParam(BaseParam):
         :rtype: str
         """
         # NOTE:
-        #   Return the first value in options if does not pass any input value
+        #   Return the first value in options if it does not pass any input value
         if value is None:
             return self.options[0]
         if value not in self.options:

ddeutil/workflow/result.py

@@ -37,8 +37,8 @@ class Result:
 
     @model_validator(mode="after")
     def __prepare_run_id(self) -> Self:
-        """Prepare running ID which use default ID if it initialize at the first
-        time
+        """Prepare running ID which use default ID if it initializes at the
+        first time.
 
         :rtype: Self
         """
@@ -84,7 +84,7 @@ class Result:
 
     def receive_jobs(self, result: Result) -> Self:
         """Receive context from another result object that use on the workflow
-        execution which create a ``jobs`` keys on the context if it do not
+        execution which create a ``jobs`` keys on the context if it does not
         exist.
 
         :rtype: Self