ddeutil-workflow 0.0.17__py3-none-any.whl → 0.0.18__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.17"
+__version__: str = "0.0.18"

ddeutil/workflow/__init__.py

@@ -3,7 +3,11 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-from .conf import Config, FileLog, Loader
+from .conf import (
+    Config,
+    FileLog,
+    Loader,
+)
 from .exceptions import (
     JobException,
     ParamValueException,
@@ -12,14 +16,57 @@ from .exceptions import (
     WorkflowException,
 )
 from .job import Job, Strategy
-from .on import On, interval2crontab
+from .on import (
+    On,
+    YearOn,
+    interval2crontab,
+)
 from .scheduler import (
     Schedule,
+    ScheduleWorkflow,
     Workflow,
+    WorkflowTaskData,
+)
+from .stage import (
+    BashStage,
+    EmptyStage,
+    HookStage,
+    PyStage,
+    Stage,
+    TriggerStage,
+    handler_result,
 )
-from .stage import Stage, handler_result
 from .utils import (
+    FILTERS,
+    ChoiceParam,
+    DatetimeParam,
+    DefaultParam,
+    FilterFunc,
+    FilterRegistry,
+    IntParam,
     Param,
+    Result,
+    ReturnTagFunc,
+    StrParam,
+    TagFunc,
+    batch,
+    cross_product,
+    custom_filter,
     dash2underscore,
+    delay,
+    filter_func,
+    gen_id,
+    get_args_const,
+    get_diff_sec,
+    get_dt_now,
+    has_template,
+    make_exec,
+    make_filter_registry,
+    make_registry,
+    map_post_filter,
+    not_in_template,
     param2template,
+    queue2str,
+    str2template,
+    tag,
 )

ddeutil/workflow/conf.py

@@ -41,8 +41,8 @@ class Config:
     # NOTE: Core
     root_path: Path = Path(os.getenv("WORKFLOW_ROOT_PATH", "."))
     tz: ZoneInfo = ZoneInfo(env("WORKFLOW_CORE_TIMEZONE", "UTC"))
-    workflow_id_simple_mode: bool = str2bool(
-        os.getenv("WORKFLOW_CORE_WORKFLOW_ID_SIMPLE_MODE", "true")
+    gen_id_simple_mode: bool = str2bool(
+        os.getenv("WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE", "true")
     )
 
     # NOTE: Register

ddeutil/workflow/job.py

@@ -553,8 +553,6 @@ class Job(BaseModel):
         # NOTE: Create event for cancel executor by trigger stop running event.
         event: Event = Event()
 
-        print("Job Run Fail-Fast:", self.strategy.fail_fast)
-
         # IMPORTANT: Start running strategy execution by multithreading because
         #   it will running by strategy values without waiting previous
         #   execution.

ddeutil/workflow/repeat.py

@@ -12,8 +12,8 @@ from functools import wraps
 
 from starlette.concurrency import run_in_threadpool
 
+from .__cron import CronJob
 from .conf import config, get_logger
-from .cron import CronJob
 
 logger = get_logger("ddeutil.workflow")
 

ddeutil/workflow/scheduler.py

@@ -376,7 +376,7 @@ class Workflow(BaseModel):
                 status=0,
                 context={
                     "params": params,
-                    "poking": {"skipped": [str(on.cronjob)], "run": []},
+                    "release": {"status": "skipped", "cron": [str(on.cronjob)]},
                 },
             )
 
@@ -388,7 +388,7 @@ class Workflow(BaseModel):
         # NOTE: Release when the time is nearly to schedule time.
         while (duration := get_diff_sec(next_time, tz=cron_tz)) > (
             sleep_interval + 5
-        ):
+        ):  # pragma: no cov
             logger.debug(
                 f"({self.run_id}) [CORE]: {self.name!r} : {on.cronjob} : "
                 f"Sleep until: {duration}"
@@ -439,7 +439,7 @@ class Workflow(BaseModel):
             status=0,
             context={
                 "params": params,
-                "poking": {"skipped": [], "run": [str(on.cronjob)]},
+                "release": {"status": "run", "cron": [str(on.cronjob)]},
             },
         )
 
@@ -492,7 +492,7 @@ class Workflow(BaseModel):
             for future in as_completed(futures):
                 results.append(future.result(timeout=60))
 
-        if len(queue) > 0:
+        if len(queue) > 0:  # pragma: no cov
             logger.error(
                 f"({self.run_id}) [POKING]: Log Queue does empty when poking "
                 f"process was finishing."
@@ -717,11 +717,11 @@ class Workflow(BaseModel):
             return context
 
         # NOTE: Raise timeout error.
-        logger.warning(
+        logger.warning(  # pragma: no cov
             f"({self.run_id}) [WORKFLOW]: Execution of workflow, {self.name!r} "
             f", was timeout"
         )
-        raise WorkflowException(
+        raise WorkflowException(  # pragma: no cov
             f"Execution of workflow: {self.name} was timeout"
         )
 
@@ -765,7 +765,8 @@ class Workflow(BaseModel):
                 continue
 
             # NOTE: Start workflow job execution with deep copy context data
-            #   before release.
+            #   before release. This job execution process will running until
+            #   done before checking all execution timeout or not.
             #
             #   {
             #       'params': <input-params>,
@@ -783,10 +784,10 @@ class Workflow(BaseModel):
             return context
 
         # NOTE: Raise timeout error.
-        logger.warning(
+        logger.warning(  # pragma: no cov
             f"({self.run_id}) [WORKFLOW]: Execution of workflow was timeout"
         )
-        raise WorkflowException(
+        raise WorkflowException(  # pragma: no cov
             f"Execution of workflow: {self.name} was timeout"
         )
 
@@ -832,12 +833,13 @@ class ScheduleWorkflow(BaseModel):
         if on := data.pop("on", []):
 
             if isinstance(on, str):
-                on = [on]
+                on: list[str] = [on]
 
             if any(not isinstance(n, (dict, str)) for n in on):
                 raise TypeError("The ``on`` key should be list of str or dict")
 
-            # NOTE: Pass on value to Loader and keep on model object to on field
+            # NOTE: Pass on value to Loader and keep on model object to on
+            #   field.
             data["on"] = [
                 (
                     Loader(n, externals=(externals or {})).data
@@ -902,12 +904,14 @@ class Schedule(BaseModel):
         *,
         externals: DictData | None = None,
     ) -> list[WorkflowTaskData]:
-        """Generate Task from the current datetime.
+        """Return the list of WorkflowTaskData object from the specific input
+        datetime that mapping with the on field.
 
         :param start_date: A start date that get from the workflow schedule.
         :param queue: A mapping of name and list of datetime for queue.
         :param running: A mapping of name and list of datetime for running.
         :param externals: An external parameters that pass to the Loader object.
+
         :rtype: list[WorkflowTaskData]
         """
 
@@ -922,12 +926,14 @@ class Schedule(BaseModel):
             queue[wfs.name]: list[datetime] = []
             running[wfs.name]: list[datetime] = []
 
-            # NOTE: Create default on if it does not passing on the Schedule.
+            # NOTE: Create the default on value if it does not passing on the
+            #   Schedule object.
             _ons: list[On] = wf.on.copy() if len(wfs.on) == 0 else wfs.on
 
             for on in _ons:
-                on_gen = on.generate(start_date)
+                on_gen: CronRunner = on.generate(start_date)
                 next_running_date = on_gen.next
+
                 while next_running_date in queue[wfs.name]:
                     next_running_date = on_gen.next
 
@@ -957,13 +963,14 @@ def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
 
     :param cancel_on_failure: A flag that allow to return the CancelJob or not
         it will raise.
-    :rtype: Callable[P, Optional[CancelJob]]
+
+    :rtype: DecoratorCancelJob
     """
 
     def decorator(func: ReturnCancelJob) -> ReturnCancelJob:
         try:
             # NOTE: Check the function that want to handle is method or not.
-            if inspect.ismethod(func):
+            if inspect.ismethod(func):  # pragma: no cov
 
                 @wraps(func)
                 def wrapper(self, *args, **kwargs):
@@ -977,7 +984,7 @@ def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
 
             return wrapper
 
-        except Exception as err:
+        except Exception as err:  # pragma: no cov
             logger.exception(err)
             if cancel_on_failure:
                 return CancelJob
@@ -1005,7 +1012,7 @@ class WorkflowTaskData:
         *,
         waiting_sec: int = 60,
         sleep_interval: int = 15,
-    ) -> None:
+    ) -> None:  # pragma: no cov
         """Workflow release, it will use with the same logic of
         `workflow.release` method.
 
@@ -1119,7 +1126,7 @@ class WorkflowTaskData:
             future_running_time in self.running[wf.name]
             or future_running_time in self.queue[wf.name]
             or future_running_time < finish_time
-        ):
+        ):  # pragma: no cov
             future_running_time: datetime = gen.next
 
         heappush(self.queue[wf.name], future_running_time)
@@ -1134,7 +1141,7 @@ class WorkflowTaskData:
         return NotImplemented
 
 
-@catch_exceptions(cancel_on_failure=True)
+@catch_exceptions(cancel_on_failure=True)  # pragma: no cov
 def workflow_task(
     workflow_tasks: list[WorkflowTaskData],
     stop: datetime,
@@ -1233,7 +1240,7 @@ def workflow_task(
     logger.debug(f"[WORKFLOW]: {'=' * 100}")
 
 
-def workflow_monitor(threads: dict[str, Thread]) -> None:
+def workflow_monitor(threads: dict[str, Thread]) -> None:  # pragma: no cov
     """Workflow schedule for monitoring long running thread from the schedule
     control.
 
@@ -1255,7 +1262,7 @@ def workflow_control(
     schedules: list[str],
     stop: datetime | None = None,
     externals: DictData | None = None,
-) -> list[str]:
+) -> list[str]:  # pragma: no cov
     """Workflow scheduler control.
 
     :param schedules: A list of workflow names that want to schedule running.
@@ -1343,7 +1350,7 @@ def workflow_runner(
     stop: datetime | None = None,
     externals: DictData | None = None,
     excluded: list[str] | None = None,
-) -> list[str]:
+) -> list[str]:  # pragma: no cov
     """Workflow application that running multiprocessing schedule with chunk of
     workflows that exists in config path.
 

ddeutil/workflow/stage.py

@@ -127,7 +127,6 @@ def handler_result(message: str | None = None) -> DecoratorResult:
                 logger.error(
                     f"({self.run_id}) [STAGE]: {err.__class__.__name__}: {err}"
                 )
-                print("Stage Raise error:", config.stage_raise_error)
                 if config.stage_raise_error:
                     # NOTE: If error that raise from stage execution course by
                     #   itself, it will return that error with previous

ddeutil/workflow/utils.py

@@ -100,7 +100,7 @@ def gen_id(
     if not isinstance(value, str):
         value: str = str(value)
 
-    if config.workflow_id_simple_mode:
+    if config.gen_id_simple_mode:
         return hash_str(f"{(value if sensitive else value.lower())}", n=10) + (
             f"{datetime.now(tz=config.tz):%Y%m%d%H%M%S%f}" if unique else ""
         )

{ddeutil_workflow-0.0.17.dist-info → ddeutil_workflow-0.0.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.17
+Version: 0.0.18
 Summary: Lightweight workflow orchestration with less dependencies
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -24,6 +24,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: ddeutil >=0.4.3
 Requires-Dist: ddeutil-io[toml,yaml] >=0.2.3
+Requires-Dist: pydantic ==2.9.2
 Requires-Dist: python-dotenv ==1.0.1
 Requires-Dist: typer <1.0.0,==0.12.5
 Requires-Dist: schedule <2.0.0,==1.2.2
@@ -194,7 +195,7 @@ and do not raise any error to you.
 | `WORKFLOW_CORE_JOB_RAISE_ERROR`         | Core      | true                             | A flag that all job raise JobException from job strategy execution.                                                |        |
 | `WORKFLOW_CORE_MAX_NUM_POKING`          | Core      | 4                                | .                                                                                                                  |        |
 | `WORKFLOW_CORE_MAX_JOB_PARALLEL`        | Core      | 2                                | The maximum job number that able to run parallel in workflow executor.                                             |        |
-| `WORKFLOW_CORE_WORKFLOW_ID_SIMPLE_MODE` | Core      | true                             | .                                                                                                                  |        |
+| `WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE` | Core      | true                             | A flog that enable generating ID with `md5` algorithm.                                                             |        |
 | `WORKFLOW_LOG_DEBUG_MODE`               | Log       | true                             | A flag that enable logging with debug level mode.                                                                  |        |
 | `WORKFLOW_LOG_ENABLE_WRITE`             | Log       | true                             | A flag that enable logging object saving log to its destination.                                                   |        |
 | `WORKFLOW_APP_MAX_PROCESS`              | Schedule  | 2                                | The maximum process worker number that run in scheduler app module.                                                |        |

ddeutil_workflow-0.0.18.dist-info/RECORD

@@ -0,0 +1,21 @@
+ddeutil/workflow/__about__.py,sha256=b5h9QJ6GhQ-EDPZTcMYoeJZy8blWgeG9xjpFBHrVLPg,28
+ddeutil/workflow/__cron.py,sha256=ZiuV4ASkXvAyFJYxEb9PKiAFNYnUt4AJozu_kH3pI4U,25777
+ddeutil/workflow/__init__.py,sha256=HA0tjGBXJItNPsAqvhnFUXU0fP0K6iMMfMtJ37tRwcw,1385
+ddeutil/workflow/__types.py,sha256=yizLXzjQpBt_WPaof2pIyncitJvYeksw4Q1zYJeuCLA,3707
+ddeutil/workflow/api.py,sha256=vUT2RVS9sF3hvY-IrzAEnahxwq4ZFYP0G3xfctHbNsw,4701
+ddeutil/workflow/cli.py,sha256=baHhvtI8snbHYHeThoX401Cd6SMB2boyyCbCtTrIl3E,3278
+ddeutil/workflow/conf.py,sha256=4j7m2blvCPlz_me4SBHf_exViUK3ZLLBCwldPztHJKo,15390
+ddeutil/workflow/exceptions.py,sha256=Uf1-Tn8rAzj0aiVHSqo4fBqO80W0za7UFZgKv24E-tg,706
+ddeutil/workflow/job.py,sha256=kSllDDiSnDpyFnIT9-Sum6OHQ16Pn5h2t5_-XljHbgk,23979
+ddeutil/workflow/on.py,sha256=rneZB5HyFWTBWriGef999bovA3glQIK6LTgC996q9Gc,7334
+ddeutil/workflow/repeat.py,sha256=s0azh-f5JQeow7kpxM8GKlqgAmKL7oU6St3L4Ggx4cY,4925
+ddeutil/workflow/route.py,sha256=JALwOH6xKu5rnII7DgA1Lbp_E5ehCoBbOW_eKqB_Olk,6753
+ddeutil/workflow/scheduler.py,sha256=baCYbv5f8HiQgV36fUvkkUpSiIRhrznuwKefsgKjHv4,47546
+ddeutil/workflow/stage.py,sha256=6Ng3RiCSrnQ-FUsRRcuG2ClMD6ifiQlgyBFi6tohfxI,25455
+ddeutil/workflow/utils.py,sha256=ouuQ3mqjKVzuchCcvVelo8Hh8c6UJ4_lHPqejcxNDRA,25147
+ddeutil_workflow-0.0.18.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.18.dist-info/METADATA,sha256=VLZchB_AWG5kMf7RYFTyKI3zkxWdj7k942f_XZpBxyQ,13606
+ddeutil_workflow-0.0.18.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+ddeutil_workflow-0.0.18.dist-info/entry_points.txt,sha256=0BVOgO3LdUdXVZ-CiHHDKxzEk2c8J30jEwHeKn2YCWI,62
+ddeutil_workflow-0.0.18.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.18.dist-info/RECORD,,

ddeutil_workflow-0.0.17.dist-info/RECORD

@@ -1,21 +0,0 @@
-ddeutil/workflow/__about__.py,sha256=z3f1GAF3VbZK1m4FWAXXMsWplP_jSe-X-wVlshvlDWU,28
-ddeutil/workflow/__cron.py,sha256=ZiuV4ASkXvAyFJYxEb9PKiAFNYnUt4AJozu_kH3pI4U,25777
-ddeutil/workflow/__init__.py,sha256=RNKME4FPMAjqtrBR-IBwQVEKeoY5yBAiHYcZw0k9cI4,729
-ddeutil/workflow/__types.py,sha256=yizLXzjQpBt_WPaof2pIyncitJvYeksw4Q1zYJeuCLA,3707
-ddeutil/workflow/api.py,sha256=vUT2RVS9sF3hvY-IrzAEnahxwq4ZFYP0G3xfctHbNsw,4701
-ddeutil/workflow/cli.py,sha256=baHhvtI8snbHYHeThoX401Cd6SMB2boyyCbCtTrIl3E,3278
-ddeutil/workflow/conf.py,sha256=SV4GMtjUc-Bor9BPi0yOtTIsiZ0FImsoRbuJysUIE9w,15395
-ddeutil/workflow/exceptions.py,sha256=Uf1-Tn8rAzj0aiVHSqo4fBqO80W0za7UFZgKv24E-tg,706
-ddeutil/workflow/job.py,sha256=dW9NXR_bttDGLwelVi7qXXlLd96KX-TKG8xnHejA6u0,24041
-ddeutil/workflow/on.py,sha256=rneZB5HyFWTBWriGef999bovA3glQIK6LTgC996q9Gc,7334
-ddeutil/workflow/repeat.py,sha256=9uKku5uMcQgzY5fWyaJMwJ0wPFX0oTwmu7vXKdgB_ec,4923
-ddeutil/workflow/route.py,sha256=JALwOH6xKu5rnII7DgA1Lbp_E5ehCoBbOW_eKqB_Olk,6753
-ddeutil/workflow/scheduler.py,sha256=Oa6bZpphjlGp0mXdBuLMk1m6G-dezaBNQxQX-SB3WJ0,47032
-ddeutil/workflow/stage.py,sha256=fMv_oFkoqpfoewzPUMdl3-BQcrJ8SE53cF7es8yGxfs,25525
-ddeutil/workflow/utils.py,sha256=lpnqGGd_Rw7eZo2wDbZ-NZNItBooFooPjwM4_40Csh8,25152
-ddeutil_workflow-0.0.17.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.17.dist-info/METADATA,sha256=btmCr-yjy4gzhnZppfXjANfPH-3tKUJFGon2aOMUK30,13574
-ddeutil_workflow-0.0.17.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
-ddeutil_workflow-0.0.17.dist-info/entry_points.txt,sha256=0BVOgO3LdUdXVZ-CiHHDKxzEk2c8J30jEwHeKn2YCWI,62
-ddeutil_workflow-0.0.17.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.17.dist-info/RECORD,,