ddeutil-workflow 0.0.18__py3-none-any.whl → 0.0.20__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.18"
+__version__: str = "0.0.20"

ddeutil/workflow/__cron.py

@@ -646,12 +646,27 @@ class CronJob:
 
         :param date: An initial date that want to mark as the start point.
         :param tz: A string timezone that want to change on runner.
+
         :rtype: CronRunner
         """
         return CronRunner(self, date, tz=tz)
 
 
 class CronJobYear(CronJob):
+    """The Cron Job Converter with Year extension object that generate datetime
+    dimension of cron job schedule format,
+
+        *  *  *  *  *  *  <command to execute>
+
+        (i)     minute (0 - 59)
+        (ii)    hour (0 - 23)
+        (iii)   day of the month (1 - 31)
+        (iv)    month (1 - 12)
+        (v)     day of the week (0 - 6) (Sunday to Saturday; 7 is also Sunday
+                on some systems)
+        (vi)    year (1990 - 2100)
+    """
+
     cron_length = 6
     cron_units = CRON_UNITS_YEAR
 
@@ -704,9 +719,17 @@ class CronRunner:
         else:
             self.date: datetime = datetime.now(tz=self.tz)
 
+        # NOTE: Add one second if the microsecond value more than 0.
+        if self.date.microsecond > 0:
+            self.date: datetime = self.date.replace(microsecond=0) + timedelta(
+                seconds=1
+            )
+
         # NOTE: Add one minute if the second value more than 0.
         if self.date.second > 0:
-            self.date: datetime = self.date + timedelta(minutes=1)
+            self.date: datetime = self.date.replace(second=0) + timedelta(
+                minutes=1
+            )
 
         self.__start_date: datetime = self.date
         self.cron: CronJob | CronJobYear = cron
@@ -752,7 +775,7 @@ class CronRunner:
                 not self.__shift_date(mode, reverse)
                 for mode in ("year", "month", "day", "hour", "minute")
             ):
-                return copy.deepcopy(self.date.replace(second=0, microsecond=0))
+                return copy.deepcopy(self.date)
 
         raise RecursionError("Unable to find execution time for schedule")
 
@@ -801,6 +824,10 @@ class CronRunner:
             # NOTE: Replace date that less than it mode to zero.
             self.date: datetime = replace_date(self.date, mode, reverse=reverse)
 
+            # NOTE: Replace second and microsecond values that change from
+            #   the replace_date func with reverse flag.
+            self.date: datetime = self.date.replace(second=0, microsecond=0)
+
             if current_value != getattr(self.date, switch[mode]):
                 return mode != "month"
 

ddeutil/workflow/__init__.py

@@ -15,7 +15,10 @@ from .exceptions import (
     UtilException,
     WorkflowException,
 )
-from .job import Job, Strategy
+from .job import (
+    Job,
+    Strategy,
+)
 from .on import (
     On,
     YearOn,
@@ -24,8 +27,6 @@ from .on import (
 from .scheduler import (
     Schedule,
     ScheduleWorkflow,
-    Workflow,
-    WorkflowTaskData,
 )
 from .stage import (
     BashStage,
@@ -34,7 +35,7 @@ from .stage import (
     PyStage,
     Stage,
     TriggerStage,
-    handler_result,
+    extract_hook,
 )
 from .utils import (
     FILTERS,
@@ -70,3 +71,7 @@ from .utils import (
     str2template,
     tag,
 )
+from .workflow import (
+    Workflow,
+    WorkflowTaskData,
+)

ddeutil/workflow/conf.py

@@ -33,6 +33,29 @@ load_dotenv()
 env = os.getenv
 
 
+@lru_cache
+def get_logger(name: str):
+    """Return logger object with an input module name.
+
+    :param name: A module name that want to log.
+    """
+    lg = logging.getLogger(name)
+    formatter = logging.Formatter(
+        fmt=(
+            "%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d, "
+            "%(thread)-5d) [%(levelname)-7s] %(message)-120s "
+            "(%(filename)s:%(lineno)s)"
+        ),
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+    stream = logging.StreamHandler()
+    stream.setFormatter(formatter)
+    lg.addHandler(stream)
+
+    lg.setLevel(logging.DEBUG if config.debug else logging.INFO)
+    return lg
+
+
 class Config:
     """Config object for keeping application configuration on current session
     without changing when if the application still running.
@@ -98,12 +121,14 @@ class Config:
         os.getenv("WORKFLOW_API_ENABLE_ROUTE_SCHEDULE", "true")
     )
 
-    def __init__(self):
+    def __init__(self) -> None:
+        # VALIDATE: the MAX_JOB_PARALLEL value should not less than 0.
         if self.max_job_parallel < 0:
             raise ValueError(
                 f"``MAX_JOB_PARALLEL`` should more than 0 but got "
                 f"{self.max_job_parallel}."
             )
+
         try:
             self.stop_boundary_delta: timedelta = timedelta(
                 **json.loads(self.stop_boundary_delta_str)
@@ -195,6 +220,7 @@ class SimLoad:
         :param conf: A config object.
         :param include:
         :param exclude:
+
         :rtype: Iterator[tuple[str, DictData]]
         """
         exclude: list[str] = exclude or []
@@ -247,12 +273,14 @@ class Loader(SimLoad):
         include: list[str] | None = None,
         exclude: list[str] | None = None,
         **kwargs,
-    ) -> DictData:
+    ) -> Iterator[tuple[str, DictData]]:
         """Override the find class method from the Simple Loader object.
 
         :param obj: A object that want to validate matching before return.
         :param include:
         :param exclude:
+
+        :rtype: Iterator[tuple[str, DictData]]
         """
         return super().finds(
             obj=obj, conf=Config(), include=include, exclude=exclude
@@ -268,6 +296,7 @@ def get_type(t: str, params: Config) -> AnyModelType:
     :param t: A importable type string.
     :param params: A config parameters that use registry to search this
         type.
+
     :rtype: AnyModelType
     """
     try:
@@ -283,29 +312,7 @@ def get_type(t: str, params: Config) -> AnyModelType:
 
 
 config = Config()
-
-
-@lru_cache
-def get_logger(name: str):
-    """Return logger object with an input module name.
-
-    :param name: A module name that want to log.
-    """
-    logger = logging.getLogger(name)
-    formatter = logging.Formatter(
-        fmt=(
-            "%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d, "
-            "%(thread)-5d) [%(levelname)-7s] %(message)-120s "
-            "(%(filename)s:%(lineno)s)"
-        ),
-        datefmt="%Y-%m-%d %H:%M:%S",
-    )
-    stream = logging.StreamHandler()
-    stream.setFormatter(formatter)
-    logger.addHandler(stream)
-
-    logger.setLevel(logging.DEBUG if config.debug else logging.INFO)
-    return logger
+logger = get_logger("ddeutil.workflow")
 
 
 class BaseLog(BaseModel, ABC):
@@ -315,8 +322,8 @@ class BaseLog(BaseModel, ABC):
     """
 
     name: str = Field(description="A workflow name.")
-    on: str = Field(description="A cronjob string of this piepline schedule.")
     release: datetime = Field(description="A release datetime.")
+    type: str = Field(description="A running type before logging.")
     context: DictData = Field(
         default_factory=dict,
         description=(
@@ -366,6 +373,8 @@ class FileLog(BaseLog):
         workflow name.
 
         :param name: A workflow name that want to search release logging data.
+
+        :rtype: Iterator[Self]
         """
         pointer: Path = config.root_path / f"./logs/workflow={name}"
         if not pointer.exists():
@@ -387,6 +396,9 @@ class FileLog(BaseLog):
         workflow name and release values. If a release does not pass to an input
         argument, it will return the latest release from the current log path.
 
+        :param name:
+        :param release:
+
         :raise FileNotFoundError:
         :raise NotImplementedError:
 
@@ -411,21 +423,17 @@ class FileLog(BaseLog):
             return cls.model_validate(obj=json.load(f))
 
     @classmethod
-    def is_pointed(
-        cls,
-        name: str,
-        release: datetime,
-        *,
-        queue: list[datetime] | None = None,
-    ) -> bool:
-        """Check this log already point in the destination.
+    def is_pointed(cls, name: str, release: datetime) -> bool:
+        """Check the release log already pointed or created at the destination
+        log path.
 
         :param name: A workflow name.
         :param release: A release datetime.
-        :param queue: A list of queue of datetime that already run in the
-            future.
+
+        :rtype: bool
+        :return: Return False if the release log was not pointed or created.
         """
-        # NOTE: Check environ variable was set for real writing.
+        # NOTE: Return False if enable writing log flag does not set.
         if not config.enable_write_log:
             return False
 
@@ -434,9 +442,7 @@ class FileLog(BaseLog):
             name=name, release=release
         )
 
-        if not queue:
-            return pointer.exists()
-        return pointer.exists() or (release in queue)
+        return pointer.exists()
 
     def pointer(self) -> Path:
         """Return release directory path that was generated from model data.
@@ -459,6 +465,9 @@ class FileLog(BaseLog):
         if not config.enable_write_log:
             return self
 
+        logger.debug(
+            f"({self.run_id}) [LOG]: Start writing log: {self.name!r}."
+        )
         log_file: Path = self.pointer() / f"{self.run_id}.log"
         log_file.write_text(
             json.dumps(

ddeutil/workflow/exceptions.py

@@ -3,6 +3,10 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+"""Exception objects for this package do not do anything because I want to
+create the lightweight workflow package. So, this module do just a exception
+annotate for handle error only.
+"""
 from __future__ import annotations
 
 

ddeutil/workflow/job.py

@@ -19,6 +19,7 @@ from concurrent.futures import (
     as_completed,
     wait,
 )
+from enum import Enum
 from functools import lru_cache
 from textwrap import dedent
 from threading import Event
@@ -68,10 +69,14 @@ def make(
     """Make a list of product of matrix values that already filter with
     exclude matrix and add specific matrix with include.
 
+        This function use the `lru_cache` decorator function increase
+    performance for duplicate matrix value scenario.
+
     :param matrix: A matrix values that want to cross product to possible
         parallelism values.
     :param include: A list of additional matrix that want to adds-in.
     :param exclude: A list of exclude matrix that want to filter-out.
+
     :rtype: list[DictStr]
     """
     # NOTE: If it does not set matrix, it will return list of an empty dict.
@@ -198,6 +203,18 @@ class Strategy(BaseModel):
         return make(self.matrix, self.include, self.exclude)
 
 
+class TriggerRules(str, Enum):
+    """Trigger Rules enum object."""
+
+    all_success: str = "all_success"
+    all_failed: str = "all_failed"
+    all_done: str = "all_done"
+    one_failed: str = "one_failed"
+    one_success: str = "one_success"
+    none_failed: str = "none_failed"
+    none_skipped: str = "none_skipped"
+
+
 class Job(BaseModel):
     """Job Pydantic model object (group of stages).
 
@@ -245,6 +262,11 @@ class Job(BaseModel):
         default_factory=list,
         description="A list of Stage of this job.",
     )
+    trigger_rule: TriggerRules = Field(
+        default=TriggerRules.all_success,
+        description="A trigger rule of tracking needed jobs.",
+        serialization_alias="trigger-rule",
+    )
     needs: list[str] = Field(
         default_factory=list,
         description="A list of the job ID that want to run before this job.",
@@ -253,12 +275,6 @@ class Job(BaseModel):
         default_factory=Strategy,
         description="A strategy matrix that want to generate.",
     )
-    run_id: Optional[str] = Field(
-        default=None,
-        description="A running job ID.",
-        repr=False,
-        exclude=True,
-    )
 
     @model_validator(mode="before")
     def __prepare_keys__(cls, values: DictData) -> DictData:
@@ -269,6 +285,7 @@ class Job(BaseModel):
         :rtype: DictData
         """
         dash2underscore("runs-on", values)
+        dash2underscore("trigger-rule", values)
         return values
 
     @field_validator("desc", mode="after")
@@ -298,29 +315,17 @@ class Job(BaseModel):
         return value
 
     @model_validator(mode="after")
-    def __prepare_running_id_and_stage_name__(self) -> Self:
-        """Prepare the job running ID.
+    def __validate_job_id__(self) -> Self:
+        """Validate job id should not have templating syntax.
 
         :rtype: Self
         """
-        if self.run_id is None:
-            self.run_id = gen_id(self.id or "", unique=True)
-
         # VALIDATE: Validate job id should not dynamic with params template.
         if has_template(self.id):
             raise ValueError("Job ID should not has any template.")
 
         return self
 
-    def get_running_id(self, run_id: str) -> Self:
-        """Return Job model object that changing job running ID with an
-        input running ID.
-
-        :param run_id: A replace job running ID.
-        :rtype: Self
-        """
-        return self.model_copy(update={"run_id": run_id})
-
     def stage(self, stage_id: str) -> Stage:
         """Return stage model that match with an input stage ID.
 
@@ -371,8 +376,6 @@ class Job(BaseModel):
         # NOTE: If the job ID did not set, it will use index of jobs key
         #   instead.
         _id: str = self.id or str(len(to["jobs"]) + 1)
-
-        logger.debug(f"({self.run_id}) [JOB]: Set outputs on: {_id}")
         to["jobs"][_id] = (
             {"strategies": output}
             if self.strategy.is_set()
@@ -384,6 +387,7 @@ class Job(BaseModel):
         self,
         strategy: DictData,
         params: DictData,
+        run_id: str | None = None,
         *,
         event: Event | None = None,
     ) -> Result:
@@ -399,10 +403,12 @@ class Job(BaseModel):
 
         :param strategy: A metrix strategy value.
         :param params: A dynamic parameters.
+        :param run_id: A job running ID for this strategy execution.
         :param event: An manger event that pass to the PoolThreadExecutor.
 
         :rtype: Result
         """
+        run_id: str = run_id or gen_id(self.id or "", unique=True)
         strategy_id: str = gen_id(strategy)
 
         # PARAGRAPH:
@@ -423,22 +429,17 @@ class Job(BaseModel):
         # IMPORTANT: The stage execution only run sequentially one-by-one.
         for stage in self.stages:
 
-            # IMPORTANT: Change any stage running IDs to this job running ID.
-            stage: Stage = stage.get_running_id(self.run_id)
-
-            name: str = stage.id or stage.name
-
             if stage.is_skipped(params=context):
-                logger.info(f"({self.run_id}) [JOB]: Skip stage: {name!r}")
+                logger.info(f"({run_id}) [JOB]: Skip stage: {stage.iden!r}")
                 continue
 
             logger.info(
-                f"({self.run_id}) [JOB]: Start execute the stage: {name!r}"
+                f"({run_id}) [JOB]: Start execute the stage: {stage.iden!r}"
             )
 
             # NOTE: Logging a matrix that pass on this stage execution.
             if strategy:
-                logger.info(f"({self.run_id}) [JOB]: Matrix: {strategy}")
+                logger.info(f"({run_id}) [JOB]: Matrix: {strategy}")
 
             # NOTE: Force stop this execution if event was set from main
             #   execution.
@@ -463,6 +464,7 @@ class Job(BaseModel):
                             ),
                         },
                     },
+                    run_id=run_id,
                 )
 
             # PARAGRAPH:
@@ -485,12 +487,12 @@ class Job(BaseModel):
             #
             try:
                 stage.set_outputs(
-                    stage.execute(params=context).context,
+                    stage.execute(params=context, run_id=run_id).context,
                     to=context,
                 )
             except (StageException, UtilException) as err:
                 logger.error(
-                    f"({self.run_id}) [JOB]: {err.__class__.__name__}: {err}"
+                    f"({run_id}) [JOB]: {err.__class__.__name__}: {err}"
                 )
                 if config.job_raise_error:
                     raise JobException(
@@ -507,10 +509,10 @@ class Job(BaseModel):
                             "error_message": f"{err.__class__.__name__}: {err}",
                         },
                     },
+                    run_id=run_id,
                 )
 
-            # NOTE: Remove the current stage object that was created from
-            #   ``get_running_id`` method for saving memory.
+            # NOTE: Remove the current stage object.
             del stage
 
         return Result(
@@ -521,20 +523,23 @@ class Job(BaseModel):
                     "stages": filter_func(context.pop("stages", {})),
                 },
             },
+            run_id=run_id,
         )
 
-    def execute(self, params: DictData | None = None) -> Result:
+    def execute(self, params: DictData, run_id: str | None = None) -> Result:
         """Job execution with passing dynamic parameters from the workflow
         execution. It will generate matrix values at the first step and run
         multithread on this metrics to the ``stages`` field of this job.
 
         :param params: An input parameters that use on job execution.
+        :param run_id: A job running ID for this execution.
+
         :rtype: Result
         """
 
         # NOTE: I use this condition because this method allow passing empty
         #   params and I do not want to create new dict object.
-        params: DictData = {} if params is None else params
+        run_id: str = run_id or gen_id(self.id or "", unique=True)
         context: DictData = {}
 
         # NOTE: Normal Job execution without parallel strategy.
@@ -543,6 +548,7 @@ class Job(BaseModel):
                 rs: Result = self.execute_strategy(
                     strategy=strategy,
                     params=params,
+                    run_id=run_id,
                 )
                 context.update(rs.context)
             return Result(
@@ -565,6 +571,7 @@ class Job(BaseModel):
                     self.execute_strategy,
                     strategy=strategy,
                     params=params,
+                    run_id=run_id,
                     event=event,
                 )
                 for strategy in self.strategy.make()
@@ -572,15 +579,18 @@ class Job(BaseModel):
 
             # NOTE: Dynamic catching futures object with fail-fast flag.
             return (
-                self.__catch_fail_fast(event=event, futures=futures)
+                self.__catch_fail_fast(
+                    event=event, futures=futures, run_id=run_id
+                )
                 if self.strategy.fail_fast
-                else self.__catch_all_completed(futures=futures)
+                else self.__catch_all_completed(futures=futures, run_id=run_id)
             )
 
+    @staticmethod
     def __catch_fail_fast(
-        self,
         event: Event,
         futures: list[Future],
+        run_id: str,
         *,
         timeout: int = 1800,
         result_timeout: int = 60,
@@ -592,6 +602,7 @@ class Job(BaseModel):
         :param event: An event manager instance that able to set stopper on the
             observing thread/process.
         :param futures: A list of futures.
+        :param run_id: A job running ID from execution.
         :param timeout: A timeout to waiting all futures complete.
         :param result_timeout: A timeout of getting result from the future
             instance when it was running completely.
@@ -611,7 +622,7 @@ class Job(BaseModel):
         nd: str = (
             f", the strategies do not run is {not_done}" if not_done else ""
         )
-        logger.debug(f"({self.run_id}) [JOB]: Strategy is set Fail Fast{nd}")
+        logger.debug(f"({run_id}) [JOB]: Strategy is set Fail Fast{nd}")
 
         # NOTE:
         #       Stop all running tasks with setting the event manager and cancel
@@ -627,7 +638,7 @@ class Job(BaseModel):
             if err := future.exception():
                 status: int = 1
                 logger.error(
-                    f"({self.run_id}) [JOB]: One stage failed with: "
+                    f"({run_id}) [JOB]: One stage failed with: "
                     f"{future.exception()}, shutting down this future."
                 )
                 context.update(
@@ -643,9 +654,10 @@ class Job(BaseModel):
 
         return rs_final.catch(status=status, context=context)
 
+    @staticmethod
     def __catch_all_completed(
-        self,
         futures: list[Future],
+        run_id: str,
         *,
         timeout: int = 1800,
         result_timeout: int = 60,
@@ -654,6 +666,7 @@ class Job(BaseModel):
 
         :param futures: A list of futures that want to catch all completed
             result.
+        :param run_id: A job running ID from execution.
         :param timeout: A timeout to waiting all futures complete.
         :param result_timeout: A timeout of getting result from the future
             instance when it was running completely.
@@ -668,7 +681,7 @@ class Job(BaseModel):
             except TimeoutError:  # pragma: no cov
                 status = 1
                 logger.warning(
-                    f"({self.run_id}) [JOB]: Task is hanging. Attempting to "
+                    f"({run_id}) [JOB]: Task is hanging. Attempting to "
                     f"kill."
                 )
                 future.cancel()
@@ -679,11 +692,11 @@ class Job(BaseModel):
                     if not future.cancelled()
                     else "Task canceled successfully."
                 )
-                logger.warning(f"({self.run_id}) [JOB]: {stmt}")
+                logger.warning(f"({run_id}) [JOB]: {stmt}")
             except JobException as err:
                 status = 1
                 logger.error(
-                    f"({self.run_id}) [JOB]: Get stage exception with "
+                    f"({run_id}) [JOB]: Get stage exception with "
                     f"fail-fast does not set;\n{err.__class__.__name__}:\n\t"
                     f"{err}"
                 )

ddeutil/workflow/on.py

@@ -184,11 +184,13 @@ class On(BaseModel):
             raise TypeError("start value should be str or datetime type.")
         return self.cronjob.schedule(date=start, tz=self.tz)
 
-    def next(self, start: str | datetime) -> datetime:
+    def next(self, start: str | datetime) -> CronRunner:
         """Return a next datetime from Cron runner object that start with any
         date that given from input.
         """
-        return self.generate(start=start).next
+        runner: CronRunner = self.generate(start=start)
+        _ = runner.next
+        return runner
 
 
 class YearOn(On):