ddeutil-workflow 0.0.15__py3-none-any.whl → 0.0.17__py3-none-any.whl

ddeutil/workflow/job.py

@@ -4,6 +4,9 @@
 # license information.
 # ------------------------------------------------------------------------------
 """Job Model that use for keeping stages and node that running its stages.
+The job handle the lineage of stages and location of execution of stages that
+mean the job model able to define ``runs-on`` key that allow you to run this
+job.
 """
 from __future__ import annotations
 
@@ -19,27 +22,20 @@ from concurrent.futures import (
 from functools import lru_cache
 from textwrap import dedent
 from threading import Event
-from typing import Optional
+from typing import Optional, Union
 
 from ddeutil.core import freeze_args
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
-from .__types import (
-    DictData,
-    DictStr,
-    Matrix,
-    MatrixExclude,
-    MatrixInclude,
-    TupleStr,
-)
+from .__types import DictData, DictStr, Matrix, TupleStr
+from .conf import config, get_logger
 from .exceptions import (
     JobException,
     StageException,
     UtilException,
 )
-from .log import get_logger
 from .stage import Stage
 from .utils import (
     Result,
@@ -51,6 +47,8 @@ from .utils import (
 )
 
 logger = get_logger("ddeutil.workflow")
+MatrixInclude = list[dict[str, Union[str, int]]]
+MatrixExclude = list[dict[str, Union[str, int]]]
 
 
 __all__: TupleStr = (
@@ -112,6 +110,7 @@ def make(
             all(inc.get(k) == v for k, v in m.items()) for m in [*final, *add]
         ):
             continue
+
         add.append(inc)
 
     # NOTE: Merge all matrix together.
@@ -262,7 +261,7 @@ class Job(BaseModel):
     )
 
     @model_validator(mode="before")
-    def __prepare_keys(cls, values: DictData) -> DictData:
+    def __prepare_keys__(cls, values: DictData) -> DictData:
         """Rename key that use dash to underscore because Python does not
         support this character exist in any variable name.
 
@@ -273,12 +272,33 @@ class Job(BaseModel):
         return values
 
     @field_validator("desc", mode="after")
-    def ___prepare_desc(cls, value: str) -> str:
-        """Prepare description string that was created on a template."""
+    def ___prepare_desc__(cls, value: str) -> str:
+        """Prepare description string that was created on a template.
+
+        :rtype: str
+        """
         return dedent(value)
 
+    @field_validator("stages", mode="after")
+    def __validate_stage_id__(cls, value: list[Stage]) -> list[Stage]:
+        """Validate a stage ID of all stage in stages field should not be
+        duplicate.
+
+        :rtype: list[Stage]
+        """
+        # VALIDATE: Validate stage id should not duplicate.
+        rs: list[str] = []
+        for stage in value:
+            name: str = stage.id or stage.name
+            if name in rs:
+                raise ValueError(
+                    "Stage name in jobs object should not be duplicate."
+                )
+            rs.append(name)
+        return value
+
     @model_validator(mode="after")
-    def __prepare_running_id(self) -> Self:
+    def __prepare_running_id_and_stage_name__(self) -> Self:
         """Prepare the job running ID.
 
         :rtype: Self
@@ -319,34 +339,44 @@ class Job(BaseModel):
             For example of setting output method, If you receive execute output
         and want to set on the `to` like;
 
-            ... (i)   output: {'strategy01': bar, 'strategy02': bar}
-            ... (ii)  to: {'jobs'}
+            ... (i)   output: {'strategy-01': bar, 'strategy-02': bar}
+            ... (ii)  to: {'jobs': {}}
 
         The result of the `to` variable will be;
 
             ... (iii) to: {
-                            'jobs': {
+                        'jobs': {
+                            '<job-id>': {
                                 'strategies': {
-                                    'strategy01': bar, 'strategy02': bar
+                                    'strategy-01': bar,
+                                    'strategy-02': bar
                                 }
                             }
                         }
+                    }
 
         :param output: An output context.
         :param to: A context data that want to add output result.
         :rtype: DictData
         """
-        if self.id is None:
+        if self.id is None and not config.job_default_id:
             raise JobException(
-                "This job do not set the ID before setting output."
+                "This job do not set the ID before setting execution output."
             )
 
-        to["jobs"][self.id] = (
+        # NOTE: Create jobs key to receive an output from the job execution.
+        if "jobs" not in to:
+            to["jobs"] = {}
+
+        # NOTE: If the job ID did not set, it will use index of jobs key
+        #   instead.
+        _id: str = self.id or str(len(to["jobs"]) + 1)
+
+        logger.debug(f"({self.run_id}) [JOB]: Set outputs on: {_id}")
+        to["jobs"][_id] = (
             {"strategies": output}
             if self.strategy.is_set()
-            # NOTE:
-            #   This is the best way to get single key from dict.
-            else output[next(iter(output))]
+            else output.get(next(iter(output), "DUMMY"), {})
         )
         return to
 
@@ -356,7 +386,6 @@ class Job(BaseModel):
         params: DictData,
         *,
         event: Event | None = None,
-        raise_error: bool = True,
     ) -> Result:
         """Job Strategy execution with passing dynamic parameters from the
         workflow execution to strategy matrix.
@@ -365,25 +394,27 @@ class Job(BaseModel):
         It different with ``self.execute`` because this method run only one
         strategy and return with context of this strategy data.
 
-        :raise JobException: If it has any error from StageException or
-            UtilException.
+        :raise JobException: If it has any error from ``StageException`` or
+            ``UtilException``.
 
         :param strategy: A metrix strategy value.
         :param params: A dynamic parameters.
         :param event: An manger event that pass to the PoolThreadExecutor.
-        :param raise_error: A flag that raise error instead catching to result
-            if it get exception from stage execution.
+
         :rtype: Result
         """
         strategy_id: str = gen_id(strategy)
 
-        # NOTE: Create strategy execution context and update a matrix and copied
+        # PARAGRAPH:
+        #
+        #       Create strategy execution context and update a matrix and copied
         #   of params. So, the context value will have structure like;
         #
         #   {
         #       "params": { ... },      <== Current input params
         #       "jobs": { ... },        <== Current input params
         #       "matrix": { ... }       <== Current strategy value
+        #       "stages": { ... }       <== Catching stage outputs
         #   }
         #
         context: DictData = copy.deepcopy(params)
@@ -395,14 +426,14 @@ class Job(BaseModel):
             # IMPORTANT: Change any stage running IDs to this job running ID.
             stage: Stage = stage.get_running_id(self.run_id)
 
-            _st_name: str = stage.id or stage.name
+            name: str = stage.id or stage.name
 
             if stage.is_skipped(params=context):
-                logger.info(f"({self.run_id}) [JOB]: Skip stage: {_st_name!r}")
+                logger.info(f"({self.run_id}) [JOB]: Skip stage: {name!r}")
                 continue
 
             logger.info(
-                f"({self.run_id}) [JOB]: Start execute the stage: {_st_name!r}"
+                f"({self.run_id}) [JOB]: Start execute the stage: {name!r}"
             )
 
             # NOTE: Logging a matrix that pass on this stage execution.
@@ -422,20 +453,20 @@ class Job(BaseModel):
                             # ---
                             # "stages": filter_func(context.pop("stages", {})),
                             "stages": context.pop("stages", {}),
-                            # NOTE: Set the error keys.
                             "error": JobException(
-                                "Process Event stopped before execution"
+                                "Job strategy was canceled from trigger event "
+                                "that had stopped before execution."
+                            ),
+                            "error_message": (
+                                "Job strategy was canceled from trigger event "
+                                "that had stopped before execution."
                             ),
-                            "error_message": {
-                                "message": (
-                                    "Process Event stopped before execution"
-                                ),
-                            },
                         },
                     },
                 )
 
-            # NOTE:
+            # PARAGRAPH:
+            #
             #       I do not use below syntax because `params` dict be the
             #   reference memory pointer and it was changed when I action
             #   anything like update or re-construct this.
@@ -461,16 +492,25 @@ class Job(BaseModel):
                 logger.error(
                     f"({self.run_id}) [JOB]: {err.__class__.__name__}: {err}"
                 )
-                if raise_error:
+                if config.job_raise_error:
                     raise JobException(
                         f"Get stage execution error: {err.__class__.__name__}: "
                         f"{err}"
                     ) from None
-                else:
-                    raise NotImplementedError() from None
+                return Result(
+                    status=1,
+                    context={
+                        strategy_id: {
+                            "matrix": strategy,
+                            "stages": context.pop("stages", {}),
+                            "error": err,
+                            "error_message": f"{err.__class__.__name__}: {err}",
+                        },
+                    },
+                )
 
-            # NOTE: Remove new stage object that was created from
-            #   ``get_running_id`` method.
+            # NOTE: Remove the current stage object that was created from
+            #   ``get_running_id`` method for saving memory.
             del stage
 
         return Result(
@@ -491,15 +531,18 @@ class Job(BaseModel):
         :param params: An input parameters that use on job execution.
         :rtype: Result
         """
-        context: DictData = {}
+
+        # NOTE: I use this condition because this method allow passing empty
+        #   params and I do not want to create new dict object.
         params: DictData = {} if params is None else params
+        context: DictData = {}
 
         # NOTE: Normal Job execution without parallel strategy.
         if (not self.strategy.is_set()) or self.strategy.max_parallel == 1:
             for strategy in self.strategy.make():
                 rs: Result = self.execute_strategy(
                     strategy=strategy,
-                    params=copy.deepcopy(params),
+                    params=params,
                 )
                 context.update(rs.context)
             return Result(
@@ -507,11 +550,17 @@ class Job(BaseModel):
                 context=context,
             )
 
-        # NOTE: Create event for cancel executor stop running.
+        # NOTE: Create event for cancel executor by trigger stop running event.
         event: Event = Event()
 
+        print("Job Run Fail-Fast:", self.strategy.fail_fast)
+
+        # IMPORTANT: Start running strategy execution by multithreading because
+        #   it will running by strategy values without waiting previous
+        #   execution.
         with ThreadPoolExecutor(
-            max_workers=self.strategy.max_parallel
+            max_workers=self.strategy.max_parallel,
+            thread_name_prefix="job_strategy_exec_",
         ) as executor:
             futures: list[Future] = [
                 executor.submit(
@@ -566,30 +615,34 @@ class Job(BaseModel):
         )
         logger.debug(f"({self.run_id}) [JOB]: Strategy is set Fail Fast{nd}")
 
-        # NOTE: Stop all running tasks with setting the event manager and cancel
+        # NOTE:
+        #       Stop all running tasks with setting the event manager and cancel
         #   any scheduled tasks.
+        #
         if len(done) != len(futures):
             event.set()
-            for future in futures:
+            for future in not_done:
                 future.cancel()
 
-            del future
-
+        future: Future
         for future in done:
-            if future.exception():
-                status = 1
+            if err := future.exception():
+                status: int = 1
                 logger.error(
                     f"({self.run_id}) [JOB]: One stage failed with: "
                     f"{future.exception()}, shutting down this future."
                 )
-            elif future.cancelled():
+                context.update(
+                    {
+                        "error": err,
+                        "error_message": f"{err.__class__.__name__}: {err}",
+                    },
+                )
                 continue
 
             # NOTE: Update the result context to main job context.
             context.update(future.result(timeout=result_timeout).context)
 
-            del future
-
         return rs_final.catch(status=status, context=context)
 
     def __catch_all_completed(
@@ -614,7 +667,7 @@ class Job(BaseModel):
         for future in as_completed(futures, timeout=timeout):
             try:
                 context.update(future.result(timeout=result_timeout).context)
-            except TimeoutError:
+            except TimeoutError:  # pragma: no cov
                 status = 1
                 logger.warning(
                     f"({self.run_id}) [JOB]: Task is hanging. Attempting to "
@@ -636,6 +689,10 @@ class Job(BaseModel):
                     f"fail-fast does not set;\n{err.__class__.__name__}:\n\t"
                     f"{err}"
                 )
-            finally:
-                del future
+                context.update(
+                    {
+                        "error": err,
+                        "error_message": f"{err.__class__.__name__}: {err}",
+                    },
+                )
         return rs_final.catch(status=status, context=context)

ddeutil/workflow/on.py

@@ -14,9 +14,9 @@ from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
+from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner
 from .__types import DictData, DictStr, TupleStr
-from .cron import WEEKDAYS, CronJob, CronJobYear, CronRunner
-from .utils import Loader
+from .conf import Loader
 
 __all__: TupleStr = (
     "On",
@@ -109,7 +109,7 @@ class On(BaseModel):
     def from_loader(
         cls,
         name: str,
-        externals: DictData,
+        externals: DictData | None = None,
     ) -> Self:
         """Constructor from the name of config that will use loader object for
         getting the data.
@@ -117,6 +117,7 @@ class On(BaseModel):
         :param name: A name of config that will getting from loader.
         :param externals: A extras external parameter that will keep in extras.
         """
+        externals: DictData = externals or {}
         loader: Loader = Loader(name, externals=externals)
 
         # NOTE: Validate the config type match with current connection model
@@ -139,7 +140,9 @@ class On(BaseModel):
                 )
             )
         if "cronjob" not in loader_data:
-            raise ValueError("Config does not set ``cronjob`` key")
+            raise ValueError(
+                "Config does not set ``cronjob`` or ``interval`` keys"
+            )
         return cls.model_validate(
             obj=dict(
                 cronjob=loader_data.pop("cronjob"),
@@ -175,17 +178,17 @@ class On(BaseModel):
 
     def generate(self, start: str | datetime) -> CronRunner:
         """Return Cron runner object."""
-        if not isinstance(start, datetime):
+        if isinstance(start, str):
             start: datetime = datetime.fromisoformat(start)
+        elif not isinstance(start, datetime):
+            raise TypeError("start value should be str or datetime type.")
         return self.cronjob.schedule(date=start, tz=self.tz)
 
     def next(self, start: str | datetime) -> datetime:
         """Return a next datetime from Cron runner object that start with any
         date that given from input.
         """
-        if not isinstance(start, datetime):
-            start: datetime = datetime.fromisoformat(start)
-        return self.cronjob.schedule(date=start, tz=self.tz).next
+        return self.generate(start=start).next
 
 
 class YearOn(On):

ddeutil/workflow/repeat.py

@@ -6,16 +6,14 @@
 from __future__ import annotations
 
 import asyncio
-import os
 from asyncio import ensure_future
 from datetime import datetime
 from functools import wraps
-from zoneinfo import ZoneInfo
 
 from starlette.concurrency import run_in_threadpool
 
+from .conf import config, get_logger
 from .cron import CronJob
-from .log import get_logger
 
 logger = get_logger("ddeutil.workflow")
 
@@ -24,9 +22,7 @@ def get_cronjob_delta(cron: str) -> float:
     """This function returns the time delta between now and the next cron
     execution time.
     """
-    now: datetime = datetime.now(
-        tz=ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
-    )
+    now: datetime = datetime.now(tz=config.tz)
     cron = CronJob(cron)
     return (cron.schedule(now).next - now).total_seconds()
 

ddeutil/workflow/route.py

@@ -6,10 +6,8 @@
 from __future__ import annotations
 
 import copy
-import os
 from datetime import datetime, timedelta
 from typing import Any
-from zoneinfo import ZoneInfo
 
 from fastapi import APIRouter, HTTPException, Request
 from fastapi import status as st
@@ -18,9 +16,9 @@ from pydantic import BaseModel
 
 from . import Workflow
 from .__types import DictData
-from .log import get_logger
+from .conf import Loader, config, get_logger
 from .scheduler import Schedule
-from .utils import Loader, Result
+from .utils import Result
 
 logger = get_logger("ddeutil.workflow")
 workflow = APIRouter(
@@ -87,12 +85,7 @@ async def execute_workflow(name: str, payload: ExecutePayload) -> DictData:
     # NOTE: Start execute manually
     rs: Result = wf.execute(params=payload.params)
 
-    return rs.model_dump(
-        by_alias=True,
-        exclude_none=True,
-        exclude_unset=True,
-        exclude_defaults=True,
-    )
+    return dict(rs)
 
 
 @workflow.get("/{name}/logs")
@@ -172,8 +165,7 @@ async def add_deploy_scheduler(request: Request, name: str):
 
     request.state.scheduler.append(name)
 
-    tz: ZoneInfo = ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
-    start_date: datetime = datetime.now(tz=tz)
+    start_date: datetime = datetime.now(tz=config.tz)
     start_date_waiting: datetime = (start_date + timedelta(minutes=1)).replace(
         second=0, microsecond=0
     )