ddeutil-workflow 0.0.27__py3-none-any.whl → 0.0.28__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.27"
+__version__: str = "0.0.28"

ddeutil/workflow/api/api.py

@@ -5,25 +5,21 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
-import asyncio
 import contextlib
-import uuid
 from collections.abc import AsyncIterator
 from datetime import datetime, timedelta
-from queue import Empty, Queue
 from typing import TypedDict
 
 from dotenv import load_dotenv
 from fastapi import FastAPI
 from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import UJSONResponse
-from pydantic import BaseModel
 
 from ..__about__ import __version__
 from ..conf import config, get_logger
 from ..scheduler import ReleaseThread, ReleaseThreads
 from ..workflow import WorkflowQueue, WorkflowTask
-from .repeat import repeat_at, repeat_every
+from .repeat import repeat_at
 
 load_dotenv()
 logger = get_logger("ddeutil.workflow")
@@ -32,11 +28,6 @@ logger = get_logger("ddeutil.workflow")
 class State(TypedDict):
     """TypeDict for State of FastAPI application."""
 
-    # NOTE: For upper queue route.
-    upper_queue: Queue
-    upper_result: dict[str, str]
-
-    # NOTE: For schedule listener.
     scheduler: list[str]
     workflow_threads: ReleaseThreads
     workflow_tasks: list[WorkflowTask]
@@ -46,15 +37,11 @@ class State(TypedDict):
 @contextlib.asynccontextmanager
 async def lifespan(a: FastAPI) -> AsyncIterator[State]:
     """Lifespan function for the FastAPI application."""
-    a.state.upper_queue = Queue()
-    a.state.upper_result = {}
     a.state.scheduler = []
     a.state.workflow_threads = {}
     a.state.workflow_tasks = []
     a.state.workflow_queue = {}
 
-    await asyncio.create_task(broker_upper_messages())
-
     yield {
         "upper_queue": a.state.upper_queue,
         "upper_result": a.state.upper_result,
@@ -87,50 +74,11 @@ app = FastAPI(
 app.add_middleware(GZipMiddleware, minimum_size=1000)
 
 
-@repeat_every(seconds=10)
-async def broker_upper_messages():
-    """Broker for receive message from the `/upper` path and change it to upper
-    case. This broker use interval running in background every 10 seconds.
-    """
-    for _ in range(10):
-        try:
-            obj = app.state.upper_queue.get_nowait()
-            app.state.upper_result[obj["request_id"]] = obj["text"].upper()
-            logger.info(f"Upper message: {app.state.upper_result}")
-        except Empty:
-            pass
-    await asyncio.sleep(0.0001)
-
-
-class Payload(BaseModel):
-    text: str
-
-
-async def get_result(request_id: str) -> dict[str, str]:
-    """Get data from output dict that global."""
-    while True:
-        if request_id in app.state.upper_result:
-            result: str = app.state.upper_result[request_id]
-            del app.state.upper_result[request_id]
-            return {"message": result}
-        await asyncio.sleep(0.0025)
-
-
 @app.get("/")
 async def health():
     return {"message": "Workflow API already start up"}
 
 
-@app.post(f"{config.prefix_path}/upper")
-async def message_upper(payload: Payload):
-    """Convert message from any case to the upper case."""
-    request_id: str = str(uuid.uuid4())
-    app.state.upper_queue.put(
-        {"text": payload.text, "request_id": request_id},
-    )
-    return await get_result(request_id)
-
-
 # NOTE: Enable the workflow route.
 if config.enable_route_workflow:
     from .route import workflow_route

ddeutil/workflow/conf.py

@@ -24,9 +24,11 @@ from typing_extensions import Self
 
 from .__types import DictData, TupleStr
 
+PREFIX: str = "WORKFLOW"
+
 
 def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
-    return os.getenv(f"WORKFLOW_{var}", default)
+    return os.getenv(f"{PREFIX}_{var.upper().replace(' ', '_')}", default)
 
 
 def glob_files(path: Path) -> Iterator[Path]:  # pragma: no cov
@@ -80,14 +82,18 @@ def get_logger(name: str):
 
 
 class Config:  # pragma: no cov
-    """Config object for keeping application configuration on current session
+    """Config object for keeping core configurations on the current session
     without changing when if the application still running.
+
+        The config value can change when you call that config property again.
     """
 
+    __slots__ = ()
+
     # NOTE: Core
     @property
     def root_path(self) -> Path:
-        return Path(env("ROOT_PATH", "."))
+        return Path(env("CORE_ROOT_PATH", "."))
 
     @property
     def conf_path(self) -> Path:
@@ -95,7 +101,7 @@ class Config:  # pragma: no cov
 
         :rtype: Path
         """
-        return self.root_path / env("CORE_PATH_CONF", "conf")
+        return self.root_path / env("CORE_CONF_PATH", "conf")
 
     @property
     def tz(self) -> ZoneInfo:
@@ -349,7 +355,7 @@ class Loader(SimLoad):
 class BaseLog(BaseModel, ABC):
     """Base Log Pydantic Model with abstraction class property that implement
     only model fields. This model should to use with inherit to logging
-    sub-class like file, sqlite, etc.
+    subclass like file, sqlite, etc.
     """
 
     name: str = Field(description="A workflow name.")
@@ -357,9 +363,7 @@ class BaseLog(BaseModel, ABC):
     type: str = Field(description="A running type before logging.")
     context: DictData = Field(
         default_factory=dict,
-        description=(
-            "A context data that receive from a workflow execution result.",
-        ),
+        description="A context that receive from a workflow execution result.",
     )
     parent_run_id: Optional[str] = Field(default=None)
     run_id: str
@@ -386,7 +390,7 @@ class BaseLog(BaseModel, ABC):
 
 class FileLog(BaseLog):
     """File Log Pydantic Model that use to saving log data from result of
-    workflow execution. It inherit from BaseLog model that implement the
+    workflow execution. It inherits from BaseLog model that implement the
     ``self.save`` method for file.
     """
 
@@ -526,7 +530,7 @@ class SQLiteLog(BaseLog):  # pragma: no cov
         primary key ( run_id )
         """
 
-    def save(self, excluded: list[str] | None) -> None:
+    def save(self, excluded: list[str] | None) -> SQLiteLog:
         """Save logging data that receive a context data from a workflow
         execution result.
         """
@@ -549,7 +553,7 @@ Log = Union[
 ]
 
 
-def get_log() -> Log:  # pragma: no cov
+def get_log() -> type[Log]:  # pragma: no cov
     if config.log_path.is_file():
         return SQLiteLog
     return FileLog

ddeutil/workflow/cron.py

@@ -32,10 +32,10 @@ def interval2crontab(
 ) -> str:
     """Return the crontab string that was generated from specific values.
 
-    :param interval: A interval value that is one of 'daily', 'weekly', or
+    :param interval: An interval value that is one of 'daily', 'weekly', or
         'monthly'.
     :param day: A day value that will be day of week. The default value is
-        monday if it be weekly interval.
+        monday if it is weekly interval.
     :param time: A time value that passing with format '%H:%M'.
 
     Examples:
@@ -50,9 +50,9 @@ def interval2crontab(
     """
     d: str = "*"
     if interval == "weekly":
-        d = WEEKDAYS[(day or "monday")[:3].title()]
+        d = str(WEEKDAYS[(day or "monday")[:3].title()])
     elif interval == "monthly" and day:
-        d = WEEKDAYS[day[:3].title()]
+        d = str(WEEKDAYS[day[:3].title()])
 
     h, m = tuple(
         i.lstrip("0") if i != "00" else "0" for i in time.split(":", maxsplit=1)
@@ -95,7 +95,7 @@ class On(BaseModel):
 
         :param value: A mapping value that will generate crontab before create
             schedule model.
-        :param externals: A extras external parameter that will keep in extras.
+        :param externals: An extras external parameter that will keep in extras.
         """
         passing: DictStr = {}
         if "timezone" in value:
@@ -114,8 +114,8 @@ class On(BaseModel):
         """Constructor from the name of config that will use loader object for
         getting the data.
 
-        :param name: A name of config that will getting from loader.
-        :param externals: A extras external parameter that will keep in extras.
+        :param name: A name of config that will get from loader.
+        :param externals: An extras external parameter that will keep in extras.
         """
         externals: DictData = externals or {}
         loader: Loader = Loader(name, externals=externals)

ddeutil/workflow/exceptions.py

@@ -4,7 +4,7 @@
 # license information.
 # ------------------------------------------------------------------------------
 """Exception objects for this package do not do anything because I want to
-create the lightweight workflow package. So, this module do just a exception
+create the lightweight workflow package. So, this module do just an exception
 annotate for handle error only.
 """
 from __future__ import annotations

ddeutil/workflow/hook.py

@@ -87,9 +87,12 @@ def make_registry(submodule: str) -> dict[str, Registry]:
         for fstr, func in inspect.getmembers(importer, inspect.isfunction):
             # NOTE: check function attribute that already set tag by
             #   ``utils.tag`` decorator.
-            if not hasattr(func, "tag"):
+            if not (hasattr(func, "tag") and hasattr(func, "name")):
                 continue
 
+            # NOTE: Define type of the func value.
+            func: TagFunc
+
             # NOTE: Create new register name if it not exists
             if func.name not in rs:
                 rs[func.name] = {func.tag: lazy(f"{module}.{submodule}.{fstr}")}
@@ -124,9 +127,21 @@ def extract_hook(hook: str) -> Callable[[], TagFunc]:
     :raise NotImplementedError: When the searching hook's function result does
         not exist in the registry.
     :raise NotImplementedError: When the searching hook's tag result does not
-        exists in the registry with its function key.
+        exist in the registry with its function key.
 
     :param hook: A hook value that able to match with Task regex.
+
+        The format of hook value should contain 3 regular expression groups
+    which match with the below config format:
+
+        >>> "^(?P<path>[^/@]+)/(?P<func>[^@]+)@(?P<tag>.+)$"
+
+    Examples:
+        >>> extract_hook("tasks/el-postgres-to-delta@polars")
+        ...
+        >>> extract_hook("tasks/return-type-not-valid@raise")
+        ...
+
     :rtype: Callable[[], TagFunc]
     """
     if not (found := Re.RE_TASK_FMT.search(hook)):

ddeutil/workflow/job.py

@@ -83,7 +83,7 @@ def make(
     if len(matrix) == 0:
         return [{}]
 
-    # NOTE: Remove matrix that exists on the exclude.
+    # NOTE: Remove matrix that exists on the excluded.
     final: list[DictStr] = []
     for r in cross_product(matrix=matrix):
         if any(
@@ -101,7 +101,7 @@ def make(
     add: list[DictStr] = []
     for inc in include:
         # VALIDATE:
-        #   Validate any key in include list should be a subset of some one
+        #   Validate any key in include list should be a subset of someone
         #   in matrix.
         if all(not (set(inc.keys()) <= set(m.keys())) for m in final):
             raise ValueError(
@@ -128,9 +128,9 @@ class Strategy(BaseModel):
     special job with combination of matrix data.
 
         This model does not be the part of job only because you can use it to
-    any model object. The propose of this model is generate metrix result that
-    comming from combination logic with any matrix values for running it with
-    parallelism.
+    any model object. The objective of this model is generating metrix result
+    that comming from combination logic with any matrix values for running it
+    with parallelism.
 
         [1, 2, 3] x [a, b] --> [1a], [1b], [2a], [2b], [3a], [3b]
 
@@ -180,7 +180,7 @@ class Strategy(BaseModel):
         """Rename key that use dash to underscore because Python does not
         support this character exist in any variable name.
 
-        :param values: A parsing values to this models
+        :param values: A parsing values to these models
         :rtype: DictData
         """
         dash2underscore("max-parallel", values)
@@ -226,7 +226,7 @@ class Job(BaseModel):
     """Job Pydantic model object (short descripte: a group of stages).
 
         This job model allow you to use for-loop that call matrix strategy. If
-    you pass matrix mapping and it able to generate, you will see it running
+    you pass matrix mapping, and it is able to generate, you will see it running
     with loop of matrix values.
 
     Data Validate:
@@ -355,7 +355,7 @@ class Job(BaseModel):
         return all(need in jobs for need in self.needs)
 
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
-        """Set an outputs from execution process to the receive context. The
+        """Set an outputs from execution process to the received context. The
         result from execution will pass to value of ``strategies`` key.
 
             For example of setting output method, If you receive execute output
@@ -420,7 +420,7 @@ class Job(BaseModel):
         strategy and return with context of this strategy data.
 
             The result of this execution will return result with strategy ID
-        that generated from the `gen_id` function with a input strategy value.
+        that generated from the `gen_id` function with an input strategy value.
 
         :raise JobException: If it has any error from ``StageException`` or
             ``UtilException``.
@@ -429,7 +429,7 @@ class Job(BaseModel):
             This value will pass to the `matrix` key for templating.
         :param params: A dynamic parameters that will deepcopy to the context.
         :param run_id: A job running ID for this strategy execution.
-        :param event: An manger event that pass to the PoolThreadExecutor.
+        :param event: An event manager that pass to the PoolThreadExecutor.
 
         :rtype: Result
         """
@@ -496,7 +496,7 @@ class Job(BaseModel):
             # PARAGRAPH:
             #
             #       I do not use below syntax because `params` dict be the
-            #   reference memory pointer and it was changed when I action
+            #   reference memory pointer, and it was changed when I action
             #   anything like update or re-construct this.
             #
             #       ... params |= stage.execute(params=params)
@@ -513,7 +513,9 @@ class Job(BaseModel):
             #
             try:
                 stage.set_outputs(
-                    stage.execute(params=context, run_id=run_id).context,
+                    stage.handler_execute(
+                        params=context, run_id=run_id
+                    ).context,
                     to=context,
                 )
             except (StageException, UtilException) as err:
@@ -566,7 +568,7 @@ class Job(BaseModel):
         run_id: str = run_id or gen_id(self.id or "", unique=True)
         context: DictData = {}
 
-        # NOTE: Normal Job execution without parallel strategy matrix. It use
+        # NOTE: Normal Job execution without parallel strategy matrix. It uses
         #   for-loop to control strategy execution sequentially.
         if (not self.strategy.is_set()) or self.strategy.max_parallel == 1:
             for strategy in self.strategy.make():
@@ -585,8 +587,7 @@ class Job(BaseModel):
         event: Event = Event()
 
         # IMPORTANT: Start running strategy execution by multithreading because
-        #   it will running by strategy values without waiting previous
-        #   execution.
+        #   it will run by strategy values without waiting previous execution.
         with ThreadPoolExecutor(
             max_workers=self.strategy.max_parallel,
             thread_name_prefix="job_strategy_exec_",
@@ -618,7 +619,7 @@ class Job(BaseModel):
         timeout: int = 1800,
     ) -> Result:
         """Job parallel pool futures catching with fail-fast mode. That will
-        stop and set event on all not done futures if it receive the first
+        stop and set event on all not done futures if it receives the first
         exception from all running futures.
 
         :param event: An event manager instance that able to set stopper on the

ddeutil/workflow/params.py

@@ -75,7 +75,7 @@ class DatetimeParam(DefaultParam):
     default: datetime = Field(default_factory=get_dt_now)
 
     def receive(self, value: str | datetime | date | None = None) -> datetime:
-        """Receive value that match with datetime. If a input value pass with
+        """Receive value that match with datetime. If an input value pass with
         None, it will use default value instead.
 
         :param value: A value that want to validate with datetime parameter
@@ -98,7 +98,7 @@ class DatetimeParam(DefaultParam):
             return datetime.fromisoformat(value)
         except ValueError:
             raise ParamValueException(
-                f"Invalid isoformat string: {value!r}"
+                f"Invalid the ISO format string: {value!r}"
             ) from None
 
 
@@ -158,7 +158,7 @@ class ChoiceParam(BaseParam):
         :rtype: str
         """
         # NOTE:
-        #   Return the first value in options if does not pass any input value
+        #   Return the first value in options if it does not pass any input value
         if value is None:
             return self.options[0]
         if value not in self.options:

ddeutil/workflow/result.py

@@ -37,8 +37,8 @@ class Result:
 
     @model_validator(mode="after")
     def __prepare_run_id(self) -> Self:
-        """Prepare running ID which use default ID if it initialize at the first
-        time
+        """Prepare running ID which use default ID if it initializes at the
+        first time.
 
         :rtype: Self
         """
@@ -84,7 +84,7 @@ class Result:
 
     def receive_jobs(self, result: Result) -> Self:
         """Receive context from another result object that use on the workflow
-        execution which create a ``jobs`` keys on the context if it do not
+        execution which create a ``jobs`` keys on the context if it does not
         exist.
 
         :rtype: Self

ddeutil/workflow/scheduler.py

@@ -149,7 +149,7 @@ class WorkflowSchedule(BaseModel):
     @field_validator("on", mode="after")
     def __on_no_dup__(cls, value: list[On]) -> list[On]:
         """Validate the on fields should not contain duplicate values and if it
-        contain every minute value, it should has only one on value.
+        contains every minute value, it should have only one on value.
 
         :rtype: list[On]
         """
@@ -195,8 +195,8 @@ class WorkflowSchedule(BaseModel):
         wf: Workflow = Workflow.from_loader(self.name, externals=extras)
         wf_queue: WorkflowQueue = queue[self.alias]
 
-        # IMPORTANT: Create the default 'on' value if it does not passing
-        #   the on field to the Schedule object.
+        # IMPORTANT: Create the default 'on' value if it does not pass the `on`
+        #   field to the Schedule object.
         ons: list[On] = self.on or wf.on.copy()
 
         for on in ons:
@@ -223,7 +223,7 @@ class WorkflowSchedule(BaseModel):
 class Schedule(BaseModel):
     """Schedule Pydantic model that use to run with any scheduler package.
 
-        It does not equal the on value in Workflow model but it use same logic
+        It does not equal the on value in Workflow model, but it uses same logic
     to running release date with crontab interval.
     """
 
@@ -368,7 +368,7 @@ def schedule_task(
     :param stop: A stop datetime object that force stop running scheduler.
     :param queue: A mapping of alias name and WorkflowQueue object.
     :param threads: A mapping of alias name and Thread object.
-    :param log: A log class that want to making log object.
+    :param log: A log class that want to make log object.
 
     :rtype: CancelJob | None
     """
@@ -449,7 +449,7 @@ def schedule_task(
 
 
 def monitor(threads: ReleaseThreads) -> None:  # pragma: no cov
-    """Monitoring function that running every five minute for track long running
+    """Monitoring function that running every five minute for track long-running
     thread instance from the schedule_control function that run every minute.
 
     :param threads: A mapping of Thread object and its name.
@@ -479,7 +479,7 @@ def schedule_control(
     """Scheduler control function that running every minute.
 
     :param schedules: A list of workflow names that want to schedule running.
-    :param stop: An datetime value that use to stop running schedule.
+    :param stop: A datetime value that use to stop running schedule.
     :param externals: An external parameters that pass to Loader.
     :param log:
 
@@ -554,7 +554,7 @@ def schedule_control(
         scheduler.run_pending()
         time.sleep(1)
 
-        # NOTE: Break the scheduler when the control job does not exists.
+        # NOTE: Break the scheduler when the control job does not exist.
         if not scheduler.get_jobs("control"):
             scheduler.clear("monitor")
 
@@ -585,7 +585,7 @@ def schedule_runner(
 
     :param stop: A stop datetime object that force stop running scheduler.
     :param externals:
-    :param excluded: A list of schedule name that want to excluded from finding.
+    :param excluded: A list of schedule name that want to exclude from finding.
 
     :rtype: list[str]