ddeutil-workflow 0.0.19__tar.gz → 0.0.20__tar.gz

{ddeutil_workflow-0.0.19/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.19
+Version: 0.0.20
 Summary: Lightweight workflow orchestration with less dependencies
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -24,9 +24,9 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: ddeutil>=0.4.3
 Requires-Dist: ddeutil-io[toml,yaml]>=0.2.3
-Requires-Dist: pydantic==2.9.2
+Requires-Dist: pydantic==2.10.2
 Requires-Dist: python-dotenv==1.0.1
-Requires-Dist: typer<1.0.0,==0.12.5
+Requires-Dist: typer==0.15.1
 Requires-Dist: schedule<2.0.0,==1.2.2
 Provides-Extra: api
 Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "api"

{ddeutil_workflow-0.0.19 → ddeutil_workflow-0.0.20}/pyproject.toml

@@ -28,9 +28,9 @@ requires-python = ">=3.9.13"
 dependencies = [
     "ddeutil>=0.4.3",
     "ddeutil-io[yaml,toml]>=0.2.3",
-    "pydantic==2.9.2",
+    "pydantic==2.10.2",
     "python-dotenv==1.0.1",
-    "typer==0.12.5,<1.0.0",
+    "typer==0.15.1",
     "schedule==1.2.2,<2.0.0",
 ]
 dynamic = ["version"]

ddeutil_workflow-0.0.20/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.20"

{ddeutil_workflow-0.0.19 → ddeutil_workflow-0.0.20}/src/ddeutil/workflow/__cron.py

@@ -653,6 +653,20 @@ class CronJob:
 
 
 class CronJobYear(CronJob):
+    """The Cron Job Converter with Year extension object that generate datetime
+    dimension of cron job schedule format,
+
+        *  *  *  *  *  *  <command to execute>
+
+        (i)     minute (0 - 59)
+        (ii)    hour (0 - 23)
+        (iii)   day of the month (1 - 31)
+        (iv)    month (1 - 12)
+        (v)     day of the week (0 - 6) (Sunday to Saturday; 7 is also Sunday
+                on some systems)
+        (vi)    year (1990 - 2100)
+    """
+
     cron_length = 6
     cron_units = CRON_UNITS_YEAR
 
@@ -705,9 +719,17 @@ class CronRunner:
         else:
             self.date: datetime = datetime.now(tz=self.tz)
 
+        # NOTE: Add one second if the microsecond value more than 0.
+        if self.date.microsecond > 0:
+            self.date: datetime = self.date.replace(microsecond=0) + timedelta(
+                seconds=1
+            )
+
         # NOTE: Add one minute if the second value more than 0.
         if self.date.second > 0:
-            self.date: datetime = self.date + timedelta(minutes=1)
+            self.date: datetime = self.date.replace(second=0) + timedelta(
+                minutes=1
+            )
 
         self.__start_date: datetime = self.date
         self.cron: CronJob | CronJobYear = cron
@@ -753,7 +775,7 @@ class CronRunner:
                 not self.__shift_date(mode, reverse)
                 for mode in ("year", "month", "day", "hour", "minute")
             ):
-                return copy.deepcopy(self.date.replace(second=0, microsecond=0))
+                return copy.deepcopy(self.date)
 
         raise RecursionError("Unable to find execution time for schedule")
 
@@ -802,6 +824,10 @@ class CronRunner:
             # NOTE: Replace date that less than it mode to zero.
             self.date: datetime = replace_date(self.date, mode, reverse=reverse)
 
+            # NOTE: Replace second and microsecond values that change from
+            #   the replace_date func with reverse flag.
+            self.date: datetime = self.date.replace(second=0, microsecond=0)
+
             if current_value != getattr(self.date, switch[mode]):
                 return mode != "month"
 

{ddeutil_workflow-0.0.19 → ddeutil_workflow-0.0.20}/src/ddeutil/workflow/__init__.py

@@ -15,7 +15,10 @@ from .exceptions import (
     UtilException,
     WorkflowException,
 )
-from .job import Job, Strategy
+from .job import (
+    Job,
+    Strategy,
+)
 from .on import (
     On,
     YearOn,
@@ -24,8 +27,6 @@ from .on import (
 from .scheduler import (
     Schedule,
     ScheduleWorkflow,
-    Workflow,
-    WorkflowTaskData,
 )
 from .stage import (
     BashStage,
@@ -34,7 +35,7 @@ from .stage import (
     PyStage,
     Stage,
     TriggerStage,
-    handler_result,
+    extract_hook,
 )
 from .utils import (
     FILTERS,
@@ -70,3 +71,7 @@ from .utils import (
     str2template,
     tag,
 )
+from .workflow import (
+    Workflow,
+    WorkflowTaskData,
+)

{ddeutil_workflow-0.0.19 → ddeutil_workflow-0.0.20}/src/ddeutil/workflow/conf.py

@@ -33,6 +33,29 @@ load_dotenv()
 env = os.getenv
 
 
+@lru_cache
+def get_logger(name: str):
+    """Return logger object with an input module name.
+
+    :param name: A module name that want to log.
+    """
+    lg = logging.getLogger(name)
+    formatter = logging.Formatter(
+        fmt=(
+            "%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d, "
+            "%(thread)-5d) [%(levelname)-7s] %(message)-120s "
+            "(%(filename)s:%(lineno)s)"
+        ),
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+    stream = logging.StreamHandler()
+    stream.setFormatter(formatter)
+    lg.addHandler(stream)
+
+    lg.setLevel(logging.DEBUG if config.debug else logging.INFO)
+    return lg
+
+
 class Config:
     """Config object for keeping application configuration on current session
     without changing when if the application still running.
@@ -98,12 +121,14 @@ class Config:
         os.getenv("WORKFLOW_API_ENABLE_ROUTE_SCHEDULE", "true")
     )
 
-    def __init__(self):
+    def __init__(self) -> None:
+        # VALIDATE: the MAX_JOB_PARALLEL value should not less than 0.
         if self.max_job_parallel < 0:
             raise ValueError(
                 f"``MAX_JOB_PARALLEL`` should more than 0 but got "
                 f"{self.max_job_parallel}."
             )
+
         try:
             self.stop_boundary_delta: timedelta = timedelta(
                 **json.loads(self.stop_boundary_delta_str)
@@ -287,29 +312,7 @@ def get_type(t: str, params: Config) -> AnyModelType:
 
 
 config = Config()
-
-
-@lru_cache
-def get_logger(name: str):
-    """Return logger object with an input module name.
-
-    :param name: A module name that want to log.
-    """
-    logger = logging.getLogger(name)
-    formatter = logging.Formatter(
-        fmt=(
-            "%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d, "
-            "%(thread)-5d) [%(levelname)-7s] %(message)-120s "
-            "(%(filename)s:%(lineno)s)"
-        ),
-        datefmt="%Y-%m-%d %H:%M:%S",
-    )
-    stream = logging.StreamHandler()
-    stream.setFormatter(formatter)
-    logger.addHandler(stream)
-
-    logger.setLevel(logging.DEBUG if config.debug else logging.INFO)
-    return logger
+logger = get_logger("ddeutil.workflow")
 
 
 class BaseLog(BaseModel, ABC):
@@ -319,8 +322,8 @@ class BaseLog(BaseModel, ABC):
     """
 
     name: str = Field(description="A workflow name.")
-    on: str = Field(description="A cronjob string of this piepline schedule.")
     release: datetime = Field(description="A release datetime.")
+    type: str = Field(description="A running type before logging.")
     context: DictData = Field(
         default_factory=dict,
         description=(
@@ -462,6 +465,9 @@ class FileLog(BaseLog):
         if not config.enable_write_log:
             return self
 
+        logger.debug(
+            f"({self.run_id}) [LOG]: Start writing log: {self.name!r}."
+        )
         log_file: Path = self.pointer() / f"{self.run_id}.log"
         log_file.write_text(
             json.dumps(

{ddeutil_workflow-0.0.19 → ddeutil_workflow-0.0.20}/src/ddeutil/workflow/exceptions.py

@@ -3,6 +3,10 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+"""Exception objects for this package do not do anything because I want to
+create the lightweight workflow package. So, this module do just a exception
+annotate for handle error only.
+"""
 from __future__ import annotations
 
 

{ddeutil_workflow-0.0.19 → ddeutil_workflow-0.0.20}/src/ddeutil/workflow/job.py

@@ -69,10 +69,14 @@ def make(
     """Make a list of product of matrix values that already filter with
     exclude matrix and add specific matrix with include.
 
+        This function use the `lru_cache` decorator function increase
+    performance for duplicate matrix value scenario.
+
     :param matrix: A matrix values that want to cross product to possible
         parallelism values.
     :param include: A list of additional matrix that want to adds-in.
     :param exclude: A list of exclude matrix that want to filter-out.
+
     :rtype: list[DictStr]
     """
     # NOTE: If it does not set matrix, it will return list of an empty dict.
@@ -200,8 +204,15 @@ class Strategy(BaseModel):
 
 
 class TriggerRules(str, Enum):
+    """Trigger Rules enum object."""
+
     all_success: str = "all_success"
     all_failed: str = "all_failed"
+    all_done: str = "all_done"
+    one_failed: str = "one_failed"
+    one_success: str = "one_success"
+    none_failed: str = "none_failed"
+    none_skipped: str = "none_skipped"
 
 
 class Job(BaseModel):
@@ -264,12 +275,6 @@ class Job(BaseModel):
         default_factory=Strategy,
         description="A strategy matrix that want to generate.",
     )
-    run_id: Optional[str] = Field(
-        default=None,
-        description="A running job ID.",
-        repr=False,
-        exclude=True,
-    )
 
     @model_validator(mode="before")
     def __prepare_keys__(cls, values: DictData) -> DictData:
@@ -310,29 +315,17 @@ class Job(BaseModel):
         return value
 
     @model_validator(mode="after")
-    def __prepare_running_id_and_stage_name__(self) -> Self:
-        """Prepare the job running ID.
+    def __validate_job_id__(self) -> Self:
+        """Validate job id should not have templating syntax.
 
         :rtype: Self
         """
-        if self.run_id is None:
-            self.run_id = gen_id(self.id or "", unique=True)
-
         # VALIDATE: Validate job id should not dynamic with params template.
         if has_template(self.id):
             raise ValueError("Job ID should not has any template.")
 
         return self
 
-    def get_running_id(self, run_id: str) -> Self:
-        """Return Job model object that changing job running ID with an
-        input running ID.
-
-        :param run_id: A replace job running ID.
-        :rtype: Self
-        """
-        return self.model_copy(update={"run_id": run_id})
-
     def stage(self, stage_id: str) -> Stage:
         """Return stage model that match with an input stage ID.
 
@@ -383,8 +376,6 @@ class Job(BaseModel):
         # NOTE: If the job ID did not set, it will use index of jobs key
         #   instead.
         _id: str = self.id or str(len(to["jobs"]) + 1)
-
-        logger.debug(f"({self.run_id}) [JOB]: Set outputs on: {_id}")
         to["jobs"][_id] = (
             {"strategies": output}
             if self.strategy.is_set()
@@ -396,6 +387,7 @@ class Job(BaseModel):
         self,
         strategy: DictData,
         params: DictData,
+        run_id: str | None = None,
         *,
         event: Event | None = None,
     ) -> Result:
@@ -411,10 +403,12 @@ class Job(BaseModel):
 
         :param strategy: A metrix strategy value.
         :param params: A dynamic parameters.
+        :param run_id: A job running ID for this strategy execution.
         :param event: An manger event that pass to the PoolThreadExecutor.
 
         :rtype: Result
         """
+        run_id: str = run_id or gen_id(self.id or "", unique=True)
         strategy_id: str = gen_id(strategy)
 
         # PARAGRAPH:
@@ -435,22 +429,17 @@ class Job(BaseModel):
         # IMPORTANT: The stage execution only run sequentially one-by-one.
         for stage in self.stages:
 
-            # IMPORTANT: Change any stage running IDs to this job running ID.
-            stage: Stage = stage.get_running_id(self.run_id)
-
-            name: str = stage.id or stage.name
-
             if stage.is_skipped(params=context):
-                logger.info(f"({self.run_id}) [JOB]: Skip stage: {name!r}")
+                logger.info(f"({run_id}) [JOB]: Skip stage: {stage.iden!r}")
                 continue
 
             logger.info(
-                f"({self.run_id}) [JOB]: Start execute the stage: {name!r}"
+                f"({run_id}) [JOB]: Start execute the stage: {stage.iden!r}"
             )
 
             # NOTE: Logging a matrix that pass on this stage execution.
             if strategy:
-                logger.info(f"({self.run_id}) [JOB]: Matrix: {strategy}")
+                logger.info(f"({run_id}) [JOB]: Matrix: {strategy}")
 
             # NOTE: Force stop this execution if event was set from main
             #   execution.
@@ -475,6 +464,7 @@ class Job(BaseModel):
                             ),
                         },
                     },
+                    run_id=run_id,
                 )
 
             # PARAGRAPH:
@@ -497,12 +487,12 @@ class Job(BaseModel):
             #
             try:
                 stage.set_outputs(
-                    stage.execute(params=context).context,
+                    stage.execute(params=context, run_id=run_id).context,
                     to=context,
                 )
             except (StageException, UtilException) as err:
                 logger.error(
-                    f"({self.run_id}) [JOB]: {err.__class__.__name__}: {err}"
+                    f"({run_id}) [JOB]: {err.__class__.__name__}: {err}"
                 )
                 if config.job_raise_error:
                     raise JobException(
@@ -519,10 +509,10 @@ class Job(BaseModel):
                             "error_message": f"{err.__class__.__name__}: {err}",
                         },
                     },
+                    run_id=run_id,
                 )
 
-            # NOTE: Remove the current stage object that was created from
-            #   ``get_running_id`` method for saving memory.
+            # NOTE: Remove the current stage object.
             del stage
 
         return Result(
@@ -533,20 +523,23 @@ class Job(BaseModel):
                     "stages": filter_func(context.pop("stages", {})),
                 },
             },
+            run_id=run_id,
         )
 
-    def execute(self, params: DictData | None = None) -> Result:
+    def execute(self, params: DictData, run_id: str | None = None) -> Result:
         """Job execution with passing dynamic parameters from the workflow
         execution. It will generate matrix values at the first step and run
         multithread on this metrics to the ``stages`` field of this job.
 
         :param params: An input parameters that use on job execution.
+        :param run_id: A job running ID for this execution.
+
         :rtype: Result
         """
 
         # NOTE: I use this condition because this method allow passing empty
         #   params and I do not want to create new dict object.
-        params: DictData = {} if params is None else params
+        run_id: str = run_id or gen_id(self.id or "", unique=True)
         context: DictData = {}
 
         # NOTE: Normal Job execution without parallel strategy.
@@ -555,6 +548,7 @@ class Job(BaseModel):
                 rs: Result = self.execute_strategy(
                     strategy=strategy,
                     params=params,
+                    run_id=run_id,
                 )
                 context.update(rs.context)
             return Result(
@@ -577,6 +571,7 @@ class Job(BaseModel):
                     self.execute_strategy,
                     strategy=strategy,
                     params=params,
+                    run_id=run_id,
                     event=event,
                 )
                 for strategy in self.strategy.make()
@@ -584,15 +579,18 @@ class Job(BaseModel):
 
             # NOTE: Dynamic catching futures object with fail-fast flag.
             return (
-                self.__catch_fail_fast(event=event, futures=futures)
+                self.__catch_fail_fast(
+                    event=event, futures=futures, run_id=run_id
+                )
                 if self.strategy.fail_fast
-                else self.__catch_all_completed(futures=futures)
+                else self.__catch_all_completed(futures=futures, run_id=run_id)
             )
 
+    @staticmethod
     def __catch_fail_fast(
-        self,
         event: Event,
         futures: list[Future],
+        run_id: str,
         *,
         timeout: int = 1800,
         result_timeout: int = 60,
@@ -604,6 +602,7 @@ class Job(BaseModel):
         :param event: An event manager instance that able to set stopper on the
             observing thread/process.
         :param futures: A list of futures.
+        :param run_id: A job running ID from execution.
         :param timeout: A timeout to waiting all futures complete.
         :param result_timeout: A timeout of getting result from the future
             instance when it was running completely.
@@ -623,7 +622,7 @@ class Job(BaseModel):
         nd: str = (
             f", the strategies do not run is {not_done}" if not_done else ""
         )
-        logger.debug(f"({self.run_id}) [JOB]: Strategy is set Fail Fast{nd}")
+        logger.debug(f"({run_id}) [JOB]: Strategy is set Fail Fast{nd}")
 
         # NOTE:
         #       Stop all running tasks with setting the event manager and cancel
@@ -639,7 +638,7 @@ class Job(BaseModel):
             if err := future.exception():
                 status: int = 1
                 logger.error(
-                    f"({self.run_id}) [JOB]: One stage failed with: "
+                    f"({run_id}) [JOB]: One stage failed with: "
                     f"{future.exception()}, shutting down this future."
                 )
                 context.update(
@@ -655,9 +654,10 @@ class Job(BaseModel):
 
         return rs_final.catch(status=status, context=context)
 
+    @staticmethod
     def __catch_all_completed(
-        self,
         futures: list[Future],
+        run_id: str,
         *,
         timeout: int = 1800,
         result_timeout: int = 60,
@@ -666,6 +666,7 @@ class Job(BaseModel):
 
         :param futures: A list of futures that want to catch all completed
             result.
+        :param run_id: A job running ID from execution.
         :param timeout: A timeout to waiting all futures complete.
         :param result_timeout: A timeout of getting result from the future
             instance when it was running completely.
@@ -680,7 +681,7 @@ class Job(BaseModel):
             except TimeoutError:  # pragma: no cov
                 status = 1
                 logger.warning(
-                    f"({self.run_id}) [JOB]: Task is hanging. Attempting to "
+                    f"({run_id}) [JOB]: Task is hanging. Attempting to "
                     f"kill."
                 )
                 future.cancel()
@@ -691,11 +692,11 @@ class Job(BaseModel):
                     if not future.cancelled()
                     else "Task canceled successfully."
                 )
-                logger.warning(f"({self.run_id}) [JOB]: {stmt}")
+                logger.warning(f"({run_id}) [JOB]: {stmt}")
             except JobException as err:
                 status = 1
                 logger.error(
-                    f"({self.run_id}) [JOB]: Get stage exception with "
+                    f"({run_id}) [JOB]: Get stage exception with "
                     f"fail-fast does not set;\n{err.__class__.__name__}:\n\t"
                     f"{err}"
                 )

{ddeutil_workflow-0.0.19 → ddeutil_workflow-0.0.20}/src/ddeutil/workflow/on.py

@@ -184,24 +184,13 @@ class On(BaseModel):
             raise TypeError("start value should be str or datetime type.")
         return self.cronjob.schedule(date=start, tz=self.tz)
 
-    def next(self, start: str | datetime) -> datetime:
+    def next(self, start: str | datetime) -> CronRunner:
         """Return a next datetime from Cron runner object that start with any
         date that given from input.
         """
-        return self.generate(start=start).next
-
-    # def pop(self, queue: list[datetime]) -> datetime:
-    #     """Pop the matching datetime value from list of datetime alias queue."""
-    #     for dt in queue:
-    #         if self.next(dt) == dt:
-    #             return dt
-    #
-    #     # NOTE: Add 1 second value to the current datetime for forcing crontab
-    #     #   runner generate the next datetime instead if current datetime be
-    #     #   valid because I already replaced second to zero before passing.
-    #     return datetime.now(tz=config.tz).replace(
-    #         second=0, microsecond=0
-    #     ) + timedelta(seconds=1)
+        runner: CronRunner = self.generate(start=start)
+        _ = runner.next
+        return runner
 
 
 class YearOn(On):