ddeutil-workflow 0.0.82__py3-none-any.whl → 0.0.83__py3-none-any.whl

ddeutil/workflow/traces.py

@@ -51,6 +51,8 @@ from .utils import cut_id, get_dt_now, prepare_newline
 
 logger = logging.getLogger("ddeutil.workflow")
 Level = Literal["debug", "info", "warning", "error", "exception"]
+EMJ_ALERT: str = "🚨"
+EMJ_SKIP: str = "⏭️"
 
 
 @lru_cache
@@ -239,7 +241,7 @@ class Metadata(BaseModel):  # pragma: no cov
         default=None, description="Environment (dev, staging, prod)."
     )
 
-    # System context
+    # NOTE: System context
     hostname: Optional[str] = Field(
         default=None, description="Hostname where workflow is running."
     )
@@ -253,7 +255,7 @@ class Metadata(BaseModel):  # pragma: no cov
         default=None, description="Workflow package version."
     )
 
-    # Custom metadata
+    # NOTE: Custom metadata
     tags: Optional[list[str]] = Field(
         default_factory=list, description="Custom tags for categorization."
     )
@@ -320,6 +322,8 @@ class Metadata(BaseModel):  # pragma: no cov
         import socket
         import sys
 
+        from .__about__ import __version__
+
         frame: Optional[FrameType] = currentframe()
         if frame is None:
             raise ValueError("Cannot get current frame")
@@ -384,7 +388,7 @@ class Metadata(BaseModel):  # pragma: no cov
             hostname=hostname,
             ip_address=ip_address,
             python_version=python_version,
-            package_version=extras_data.get("package_version"),
+            package_version=__version__,
             # NOTE: Custom metadata
             tags=extras_data.get("tags", []),
             metadata=extras_data.get("metadata", {}),
@@ -2046,7 +2050,7 @@ def get_trace(
     Args:
         run_id (str): A running ID.
         parent_run_id (str | None, default None): A parent running ID.
-        handlers:
+        handlers (list):
         extras: An extra parameter that want to override the core
             config values.
         auto_pre_process (bool, default False)
@@ -2057,7 +2061,7 @@ def get_trace(
     handlers: list[DictData] = dynamic(
         "trace_handlers", f=handlers, extras=extras
     )
-    trace = Trace.model_validate(
+    trace: Trace = Trace.model_validate(
         {
             "run_id": run_id,
             "parent_run_id": parent_run_id,

ddeutil/workflow/utils.py

@@ -8,26 +8,6 @@
 This module provides essential utility functions used throughout the workflow
 system for ID generation, datetime handling, string processing, template
 operations, and other common tasks.
-
-Functions:
-    to_train: Convert camel case strings to train case format
-    prepare_newline: Format messages with multiple newlines
-    replace_sec: Replace seconds and microseconds in datetime objects
-    clear_tz: Clear timezone info from datetime objects
-    get_dt_now: Get current datetime with timezone
-    get_d_now: Get current date
-    get_diff_sec: Calculate time difference in seconds
-    reach_next_minute: Check if datetime reaches next minute
-    wait_until_next_minute: Wait until next minute
-    delay: Add random delay to execution
-    gen_id: Generate unique identifiers for workflow components
-    default_gen_id: Generate default running ID
-    make_exec: Make files executable
-    filter_func: Filter function objects from data structures
-    cross_product: Generate cross product of matrix values
-    cut_id: Cut running ID to specified length
-    dump_all: Serialize nested BaseModel objects to dictionaries
-    obj_name: Get object name or class name
 """
 from __future__ import annotations
 
@@ -253,6 +233,33 @@ def gen_id(
     ).hexdigest()
 
 
+def extract_id(
+    name: str,
+    run_id: Optional[str] = None,
+    extras: Optional[DictData] = None,
+) -> tuple[str, str]:
+    """Extract the parent ID and running ID. If the `run_id` parameter was
+    passed, it will replace the parent_run_id with this value and re-generate
+    new running ID for it instead.
+
+    Args:
+        name (str): A name for generate hashing value for the `gen_id` function.
+        run_id (str | None, default None):
+        extras:
+
+    Returns:
+        tuple[str, str]: A pair of parent running ID and running ID.
+    """
+    generated = gen_id(name, unique=True, extras=extras)
+    if run_id:
+        parent_run_id: str = run_id
+        run_id: str = generated
+    else:
+        run_id: str = generated
+        parent_run_id: str = run_id
+    return parent_run_id, run_id
+
+
 def default_gen_id() -> str:
     """Return running ID for making default ID for the Result model.
 
@@ -327,6 +334,8 @@ def cut_id(run_id: str, *, num: int = 8) -> str:
     Example:
         >>> cut_id(run_id='20240101081330000000T1354680202')
         '202401010813680202'
+        >>> cut_id(run_id='20240101081330000000T1354680202')
+        '54680202'
 
     Args:
         run_id: A running ID to cut.
@@ -394,3 +403,8 @@ def obj_name(obj: Optional[Union[str, object]] = None) -> Optional[str]:
     else:
         obj_type: str = obj.__class__.__name__
     return obj_type
+
+
+def remove_sys_extras(extras: DictData) -> DictData:
+    """Remove key that starts with `__sys_` from the extra dict parameter."""
+    return {k: extras[k] for k in extras if not k.startswith("__sys_")}

ddeutil/workflow/workflow.py

@@ -37,6 +37,7 @@ from threading import Event as ThreadEvent
 from typing import Any, Literal, Optional, Union
 
 from pydantic import BaseModel, Field
+from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
@@ -62,10 +63,10 @@ from .result import (
 from .reusables import has_template, param2template
 from .traces import Trace, get_trace
 from .utils import (
-    UTC,
+    extract_id,
     gen_id,
     get_dt_now,
-    replace_sec,
+    remove_sys_extras,
 )
 
 
@@ -243,8 +244,15 @@ class Workflow(BaseModel):
                 f"{self.name!r}."
             )
 
+        # NOTE: Force update internal extras for handler circle execution.
+        self.extras.update({"__sys_break_circle_exec": self.name})
+
         return self
 
+    @field_serializer("extras")
+    def __serialize_extras(self, extras: DictData) -> DictData:
+        return remove_sys_extras(extras)
+
     def detail(self) -> DictData:  # pragma: no cov
         """Return the detail of this workflow for generate markdown."""
         return self.model_dump(by_alias=True)
@@ -257,7 +265,8 @@ class Workflow(BaseModel):
         """
 
         def align_newline(value: str) -> str:
-            return value.rstrip("\n").replace("\n", "\n                ")
+            space: str = " " * 16
+            return value.rstrip("\n").replace("\n", f"\n{space}")
 
         info: str = (
             f"| Author: {author or 'nobody'} "
@@ -284,8 +293,7 @@ class Workflow(BaseModel):
                 {align_newline(self.desc)}\n
                 ## Parameters\n
                 | name | type | default | description |
-                | --- | --- | --- | : --- : |
-
+                | --- | --- | --- | : --- : |\n\n
                 ## Jobs\n
                 {align_newline(jobs)}
                 """.lstrip(
@@ -314,8 +322,7 @@ class Workflow(BaseModel):
                 f"{self.name!r}"
             )
         job: Job = self.jobs[name]
-        if self.extras:
-            job.extras = self.extras
+        job.extras = self.extras
         return job
 
     def parameterize(self, params: DictData) -> DictData:
@@ -334,8 +341,8 @@ class Workflow(BaseModel):
                 execute method.
 
         Returns:
-            DictData: The parameter value that validate with its parameter fields and
-                adding jobs key to this parameter.
+            DictData: The parameter value that validate with its parameter fields
+                and adding jobs key to this parameter.
 
         Raises:
             WorkflowError: If parameter value that want to validate does
@@ -366,33 +373,6 @@ class Workflow(BaseModel):
             "jobs": {},
         }
 
-    def validate_release(self, dt: datetime) -> datetime:
-        """Validate the release datetime that should was replaced second and
-        millisecond to 0 and replaced timezone to None before checking it match
-        with the set `on` field.
-
-        Args:
-            dt (datetime): A datetime object that want to validate.
-
-        Returns:
-            datetime: The validated release datetime.
-        """
-        if dt.tzinfo is None:
-            dt = dt.replace(tzinfo=UTC)
-
-        release: datetime = replace_sec(dt.astimezone(UTC))
-
-        # NOTE: Return itself if schedule event does not set.
-        if not self.on.schedule:
-            return release
-
-        for on in self.on.schedule:
-            if release == on.cronjob.schedule(release, tz=UTC).next:
-                return release
-        raise WorkflowError(
-            "Release datetime does not support for this workflow"
-        )
-
     def release(
         self,
         release: datetime,
@@ -422,11 +402,12 @@ class Workflow(BaseModel):
 
         Args:
             release (datetime): A release datetime.
-            params: A workflow parameter that pass to execute method.
-            release_type:
+            params (DictData): A workflow parameter that pass to execute method.
+            release_type (ReleaseType): A release type that want to execute.
             run_id: (str) A workflow running ID.
             runs_metadata: (DictData)
-            audit: An audit class that want to save the execution result.
+            audit (Audit): An audit model that use to manage release log of this
+                execution.
             override_log_name: (str) An override logging name that use
                 instead the workflow name.
             timeout: (int) A workflow execution time out in second unit.
@@ -441,13 +422,9 @@ class Workflow(BaseModel):
         audit: Audit = audit or get_audit(extras=self.extras)
 
         # NOTE: Generate the parent running ID with not None value.
-        if run_id:
-            parent_run_id: str = run_id
-            run_id: str = gen_id(name, unique=True)
-        else:
-            run_id: str = gen_id(name, unique=True)
-            parent_run_id: str = run_id
-
+        parent_run_id, run_id = extract_id(
+            name, run_id=run_id, extras=self.extras
+        )
         context: DictData = {"status": WAIT}
         audit_data: DictData = {
             "name": name,
@@ -460,7 +437,7 @@ class Workflow(BaseModel):
         trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
-        release: datetime = self.validate_release(dt=release)
+        release: datetime = self.on.validate_dt(dt=release)
         trace.info(f"[RELEASE]: Start {name!r} : {release:%Y-%m-%d %H:%M:%S}")
         values: DictData = param2template(
             params,
@@ -536,7 +513,7 @@ class Workflow(BaseModel):
                     **(context["errors"] if "errors" in context else {}),
                 },
             ),
-            extras=self.extras,
+            extras=remove_sys_extras(self.extras),
         )
 
     def execute_job(
@@ -684,8 +661,9 @@ class Workflow(BaseModel):
         :rtype: Result
         """
         ts: float = time.monotonic()
-        parent_run_id: Optional[str] = run_id
-        run_id: str = gen_id(self.name, unique=True, extras=self.extras)
+        parent_run_id, run_id = extract_id(
+            self.name, run_id=run_id, extras=self.extras
+        )
         trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
@@ -913,6 +891,10 @@ class Workflow(BaseModel):
     ) -> Result:  # pragma: no cov
         """Re-Execute workflow with passing the error context data.
 
+        Warnings:
+            This rerun method allow to rerun job execution level only. That mean
+        it does not support rerun only stage.
+
         Args:
             context: A context result that get the failed status.
             run_id: (Optional[str]) A workflow running ID.

{ddeutil_workflow-0.0.82.dist-info → ddeutil_workflow-0.0.83.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.82
+Version: 0.0.83
 Summary: Lightweight workflow orchestration with YAML template
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.82.dist-info → ddeutil_workflow-0.0.83.dist-info}/RECORD

@@ -1,20 +1,20 @@
-ddeutil/workflow/__about__.py,sha256=GJmjvBMMhA0y8IwyPpPJLqLsuBWa6J90Pleir68QW5I,60
-ddeutil/workflow/__cron.py,sha256=avOagaHl9xXOmizeRWm13cOrty9Tw0vRjFq-xoEgpAY,29167
-ddeutil/workflow/__init__.py,sha256=elWSX2JPbjORH-CJIH_zR_nrdd8Xw95NjLK49R4Kqdg,3434
+ddeutil/workflow/__about__.py,sha256=6n_dCde_BwTTJlP4Yu4HNgiRti7QydfShki703lt6Ro,60
+ddeutil/workflow/__cron.py,sha256=-1tqZG7GtUmusdl6NTy_Ck7nM_tGYTXYB7TB7tKeO60,29184
+ddeutil/workflow/__init__.py,sha256=Dvfjs7LpLerGCYGnbqKwznViTw7ire_6LR8obC1I4aM,3456
 ddeutil/workflow/__main__.py,sha256=Nqk5aO-HsZVKV2BmuJYeJEufJluipvCD9R1k2kMoJ3Y,8581
-ddeutil/workflow/__types.py,sha256=tA2vsr6mzTSzbWB1sb62c5GgxODlfVRz6FvgLNJtQao,4788
-ddeutil/workflow/audits.py,sha256=YPnWQYvhILPwPS6RhYKK0OG3lCeYFLuUNBxJvMlXf5w,26109
+ddeutil/workflow/__types.py,sha256=IOKuJCxTUPHh8Z2JoLu_K7a85oq0VOcKBhpabiJ6qEE,5001
+ddeutil/workflow/audits.py,sha256=H8yuMzXs_QAAKNox-HXdojk9CcilHYYQtklJYetoZv8,26955
 ddeutil/workflow/conf.py,sha256=VfPmwaBYEgOj8bu4eim13ayZwJ4Liy7I702aQf7vS8g,17644
-ddeutil/workflow/errors.py,sha256=Rqtuf1MGxA-hKGP5wMAkaeeayst-u4P2dX6Fp_pzbsA,5678
-ddeutil/workflow/event.py,sha256=qm7QHw-Pozm6oIUzAIxpDkPzzVZVtHgJIUlIle0vEfQ,13943
-ddeutil/workflow/job.py,sha256=WHWOVz0ErOUfbN_aqpDeNmvBvpbKhcFlzcvJmlCpJuI,48430
+ddeutil/workflow/errors.py,sha256=J4bEbtI7qtBX7eghod4djLf0y5i1r4mCz_uFU4roLhY,5713
+ddeutil/workflow/event.py,sha256=OumcZBlOZD0_J53GS4V2XJEqQ9HEcIl3UicQrCyL46M,14684
+ddeutil/workflow/job.py,sha256=VVTpxVR2iVEkjvP8r0O0LRtAPnrbsguYbKzHpe2TAVo,48146
 ddeutil/workflow/params.py,sha256=y9f6DEIyae1j4awbj3Kbeq75-U2UPFlKv9K57Hdo_Go,17188
-ddeutil/workflow/result.py,sha256=3Lpyv2Jn6T1Uc-lRbweDucSCoBr0ZByHjffKj14bj6s,9492
+ddeutil/workflow/result.py,sha256=0W3z5wAs3Dyr8r2vRMY5hl1MkvdsyXWJmQD4NmsDDOM,10194
 ddeutil/workflow/reusables.py,sha256=SBLJSxR8ELoWJErBfSMZS3Rr1O_93T-fFBpfn2AvxuA,25007
-ddeutil/workflow/stages.py,sha256=CCR_D6yqVo74PuMxfqhi8GeeAq8sRbyxReHob6yxrjI,123708
-ddeutil/workflow/traces.py,sha256=YN4XuRfQK523cNy8EVgz2iPh6s6WB865K9JezCNdM7E,74637
-ddeutil/workflow/utils.py,sha256=vQwFu-wPK-lDiX2L8AZIahCkKEF6I0MCrZ1LlP8xkoQ,12011
-ddeutil/workflow/workflow.py,sha256=VN8i0mVyuCUw1kk_CqLkN-8dWLON9i4vTLbHp6lyS-s,42961
+ddeutil/workflow/stages.py,sha256=lWlzvpJ6YyhDf0ks5q_fzHjm4-o6UZfhiYp9CG-ffro,129661
+ddeutil/workflow/traces.py,sha256=pq1lOg2UMgDiSDmjHxXPoTaBHnfc7uzzlo1u2TCwN2Q,74733
+ddeutil/workflow/utils.py,sha256=XsH8DkcTiMmWt1e59b4bFQofsBdo7uW1-7gC2rghuW8,12128
+ddeutil/workflow/workflow.py,sha256=uc71PJh7e-Bjb3Xg7T83wlLrKTPGvmX7Qjsn6SJ1GDI,42544
 ddeutil/workflow/api/__init__.py,sha256=5DzYL3ngceoRshh5HYCSVWChqNJSiP01E1bEd8XxPi0,4799
 ddeutil/workflow/api/log_conf.py,sha256=WfS3udDLSyrP-C80lWOvxxmhd_XWKvQPkwDqKblcH3E,1834
 ddeutil/workflow/api/routes/__init__.py,sha256=JRaJZB0D6mgR17MbZo8yLtdYDtD62AA8MdKlFqhG84M,420
@@ -27,9 +27,9 @@ ddeutil/workflow/plugins/providers/aws.py,sha256=61uIFBEWt-_D5Sui24qUPier1Hiqlw_
 ddeutil/workflow/plugins/providers/az.py,sha256=o3dh011lEtmr7-d7FPZJPgXdT0ytFzKfc5xnVxSyXGU,34867
 ddeutil/workflow/plugins/providers/container.py,sha256=DSN0RWxMjTJN5ANheeMauDaPa3X6Z2E1eGUcctYkENw,22134
 ddeutil/workflow/plugins/providers/gcs.py,sha256=KgAOdMBvdbMLTH_z_FwVriBFtZfKEYx8_34jzUOVjTY,27460
-ddeutil_workflow-0.0.82.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.82.dist-info/METADATA,sha256=lB4PBeyneqHLwG0S4Lt_Xz8eCz4Lxn2bkDDHIFcBpxE,16087
-ddeutil_workflow-0.0.82.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ddeutil_workflow-0.0.82.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
-ddeutil_workflow-0.0.82.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.82.dist-info/RECORD,,
+ddeutil_workflow-0.0.83.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.83.dist-info/METADATA,sha256=pxD6FyTSV4ra5DujoPvWQnb2Z9WWHHqi97HBADqeeAo,16087
+ddeutil_workflow-0.0.83.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ddeutil_workflow-0.0.83.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
+ddeutil_workflow-0.0.83.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.83.dist-info/RECORD,,