ddeutil-workflow 0.0.82__py3-none-any.whl → 0.0.84__py3-none-any.whl

ddeutil/workflow/traces.py

@@ -51,6 +51,8 @@ from .utils import cut_id, get_dt_now, prepare_newline
 
 logger = logging.getLogger("ddeutil.workflow")
 Level = Literal["debug", "info", "warning", "error", "exception"]
+EMJ_ALERT: str = "🚨"
+EMJ_SKIP: str = "⏭️"
 
 
 @lru_cache
@@ -239,7 +241,7 @@ class Metadata(BaseModel):  # pragma: no cov
         default=None, description="Environment (dev, staging, prod)."
     )
 
-    # System context
+    # NOTE: System context
     hostname: Optional[str] = Field(
         default=None, description="Hostname where workflow is running."
     )
@@ -253,7 +255,7 @@ class Metadata(BaseModel):  # pragma: no cov
         default=None, description="Workflow package version."
     )
 
-    # Custom metadata
+    # NOTE: Custom metadata
     tags: Optional[list[str]] = Field(
         default_factory=list, description="Custom tags for categorization."
     )
@@ -320,6 +322,8 @@ class Metadata(BaseModel):  # pragma: no cov
         import socket
         import sys
 
+        from .__about__ import __version__
+
         frame: Optional[FrameType] = currentframe()
         if frame is None:
             raise ValueError("Cannot get current frame")
@@ -384,7 +388,7 @@ class Metadata(BaseModel):  # pragma: no cov
             hostname=hostname,
             ip_address=ip_address,
             python_version=python_version,
-            package_version=extras_data.get("package_version"),
+            package_version=__version__,
             # NOTE: Custom metadata
             tags=extras_data.get("tags", []),
             metadata=extras_data.get("metadata", {}),
@@ -2046,7 +2050,7 @@ def get_trace(
     Args:
         run_id (str): A running ID.
         parent_run_id (str | None, default None): A parent running ID.
-        handlers:
+        handlers (list):
         extras: An extra parameter that want to override the core
             config values.
         auto_pre_process (bool, default False)
@@ -2057,7 +2061,7 @@ def get_trace(
     handlers: list[DictData] = dynamic(
         "trace_handlers", f=handlers, extras=extras
     )
-    trace = Trace.model_validate(
+    trace: Trace = Trace.model_validate(
         {
             "run_id": run_id,
             "parent_run_id": parent_run_id,

ddeutil/workflow/utils.py

@@ -8,26 +8,6 @@
 This module provides essential utility functions used throughout the workflow
 system for ID generation, datetime handling, string processing, template
 operations, and other common tasks.
-
-Functions:
-    to_train: Convert camel case strings to train case format
-    prepare_newline: Format messages with multiple newlines
-    replace_sec: Replace seconds and microseconds in datetime objects
-    clear_tz: Clear timezone info from datetime objects
-    get_dt_now: Get current datetime with timezone
-    get_d_now: Get current date
-    get_diff_sec: Calculate time difference in seconds
-    reach_next_minute: Check if datetime reaches next minute
-    wait_until_next_minute: Wait until next minute
-    delay: Add random delay to execution
-    gen_id: Generate unique identifiers for workflow components
-    default_gen_id: Generate default running ID
-    make_exec: Make files executable
-    filter_func: Filter function objects from data structures
-    cross_product: Generate cross product of matrix values
-    cut_id: Cut running ID to specified length
-    dump_all: Serialize nested BaseModel objects to dictionaries
-    obj_name: Get object name or class name
 """
 from __future__ import annotations
 
@@ -253,6 +233,33 @@ def gen_id(
     ).hexdigest()
 
 
+def extract_id(
+    name: str,
+    run_id: Optional[str] = None,
+    extras: Optional[DictData] = None,
+) -> tuple[str, str]:
+    """Extract the parent ID and running ID. If the `run_id` parameter was
+    passed, it will replace the parent_run_id with this value and re-generate
+    new running ID for it instead.
+
+    Args:
+        name (str): A name for generate hashing value for the `gen_id` function.
+        run_id (str | None, default None):
+        extras:
+
+    Returns:
+        tuple[str, str]: A pair of parent running ID and running ID.
+    """
+    generated = gen_id(name, unique=True, extras=extras)
+    if run_id:
+        parent_run_id: str = run_id
+        run_id: str = generated
+    else:
+        run_id: str = generated
+        parent_run_id: str = run_id
+    return parent_run_id, run_id
+
+
 def default_gen_id() -> str:
     """Return running ID for making default ID for the Result model.
 
@@ -327,6 +334,8 @@ def cut_id(run_id: str, *, num: int = 8) -> str:
     Example:
         >>> cut_id(run_id='20240101081330000000T1354680202')
         '202401010813680202'
+        >>> cut_id(run_id='20240101081330000000T1354680202')
+        '54680202'
 
     Args:
         run_id: A running ID to cut.
@@ -394,3 +403,19 @@ def obj_name(obj: Optional[Union[str, object]] = None) -> Optional[str]:
     else:
         obj_type: str = obj.__class__.__name__
     return obj_type
+
+
+def pop_sys_extras(extras: DictData, scope: str = "exec") -> DictData:
+    """Remove key that starts with `__sys_` from the extra dict parameter.
+
+    Args:
+        extras:
+        scope (str):
+
+    Returns:
+        DictData:
+    """
+    keys: list[str] = [k for k in extras if not k.startswith(f"__sys_{scope}")]
+    for k in keys:
+        extras.pop(k)
+    return extras

ddeutil/workflow/workflow.py

@@ -37,9 +37,11 @@ from threading import Event as ThreadEvent
 from typing import Any, Literal, Optional, Union
 
 from pydantic import BaseModel, Field
+from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
+from . import DRYRUN
 from .__types import DictData
 from .audits import NORMAL, RERUN, Audit, ReleaseType, get_audit
 from .conf import YamlParser, dynamic
@@ -62,10 +64,10 @@ from .result import (
 from .reusables import has_template, param2template
 from .traces import Trace, get_trace
 from .utils import (
-    UTC,
+    extract_id,
     gen_id,
     get_dt_now,
-    replace_sec,
+    pop_sys_extras,
 )
 
 
@@ -245,6 +247,11 @@ class Workflow(BaseModel):
 
         return self
 
+    @field_serializer("extras")
+    def __serialize_extras(self, extras: DictData) -> DictData:
+        """Serialize extra parameter."""
+        return {k: extras[k] for k in extras if not k.startswith("__sys_")}
+
     def detail(self) -> DictData:  # pragma: no cov
         """Return the detail of this workflow for generate markdown."""
         return self.model_dump(by_alias=True)
@@ -256,8 +263,11 @@ class Workflow(BaseModel):
             author (str | None, default None): An author name.
         """
 
-        def align_newline(value: str) -> str:
-            return value.rstrip("\n").replace("\n", "\n                ")
+        def align_newline(value: Optional[str]) -> str:
+            space: str = " " * 16
+            if value is None:
+                return ""
+            return value.rstrip("\n").replace("\n", f"\n{space}")
 
         info: str = (
             f"| Author: {author or 'nobody'} "
@@ -284,8 +294,7 @@ class Workflow(BaseModel):
                 {align_newline(self.desc)}\n
                 ## Parameters\n
                 | name | type | default | description |
-                | --- | --- | --- | : --- : |
-
+                | --- | --- | --- | : --- : |\n\n
                 ## Jobs\n
                 {align_newline(jobs)}
                 """.lstrip(
@@ -314,8 +323,7 @@ class Workflow(BaseModel):
                 f"{self.name!r}"
             )
         job: Job = self.jobs[name]
-        if self.extras:
-            job.extras = self.extras
+        job.extras = self.extras
         return job
 
     def parameterize(self, params: DictData) -> DictData:
@@ -334,8 +342,8 @@ class Workflow(BaseModel):
                 execute method.
 
         Returns:
-            DictData: The parameter value that validate with its parameter fields and
-                adding jobs key to this parameter.
+            DictData: The parameter value that validate with its parameter fields
+                and adding jobs key to this parameter.
 
         Raises:
             WorkflowError: If parameter value that want to validate does
@@ -366,33 +374,6 @@ class Workflow(BaseModel):
             "jobs": {},
         }
 
-    def validate_release(self, dt: datetime) -> datetime:
-        """Validate the release datetime that should was replaced second and
-        millisecond to 0 and replaced timezone to None before checking it match
-        with the set `on` field.
-
-        Args:
-            dt (datetime): A datetime object that want to validate.
-
-        Returns:
-            datetime: The validated release datetime.
-        """
-        if dt.tzinfo is None:
-            dt = dt.replace(tzinfo=UTC)
-
-        release: datetime = replace_sec(dt.astimezone(UTC))
-
-        # NOTE: Return itself if schedule event does not set.
-        if not self.on.schedule:
-            return release
-
-        for on in self.on.schedule:
-            if release == on.cronjob.schedule(release, tz=UTC).next:
-                return release
-        raise WorkflowError(
-            "Release datetime does not support for this workflow"
-        )
-
     def release(
         self,
         release: datetime,
@@ -422,11 +403,12 @@ class Workflow(BaseModel):
 
         Args:
             release (datetime): A release datetime.
-            params: A workflow parameter that pass to execute method.
-            release_type:
+            params (DictData): A workflow parameter that pass to execute method.
+            release_type (ReleaseType): A release type that want to execute.
             run_id: (str) A workflow running ID.
             runs_metadata: (DictData)
-            audit: An audit class that want to save the execution result.
+            audit (Audit): An audit model that use to manage release log of this
+                execution.
             override_log_name: (str) An override logging name that use
                 instead the workflow name.
             timeout: (int) A workflow execution time out in second unit.
@@ -441,13 +423,9 @@ class Workflow(BaseModel):
         audit: Audit = audit or get_audit(extras=self.extras)
 
         # NOTE: Generate the parent running ID with not None value.
-        if run_id:
-            parent_run_id: str = run_id
-            run_id: str = gen_id(name, unique=True)
-        else:
-            run_id: str = gen_id(name, unique=True)
-            parent_run_id: str = run_id
-
+        parent_run_id, run_id = extract_id(
+            name, run_id=run_id, extras=self.extras
+        )
         context: DictData = {"status": WAIT}
         audit_data: DictData = {
             "name": name,
@@ -460,7 +438,7 @@ class Workflow(BaseModel):
         trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
-        release: datetime = self.validate_release(dt=release)
+        release: datetime = self.on.validate_dt(dt=release)
         trace.info(f"[RELEASE]: Start {name!r} : {release:%Y-%m-%d %H:%M:%S}")
         values: DictData = param2template(
             params,
@@ -475,7 +453,17 @@ class Workflow(BaseModel):
             extras=self.extras,
         )
 
-        if release_type == NORMAL and audit.is_pointed(data=audit_data):
+        if release_type == RERUN:
+            # TODO: It will load previous audit and use this data to run with
+            #   the `rerun` method.
+            raise NotImplementedError(
+                "Release does not support for rerun type yet. Please use the "
+                "`rerun` method instead."
+            )
+        elif release_type == DRYRUN:
+            self.extras.update({"__sys_release_dryrun_mode": True})
+            trace.debug("[RELEASE]: Mark dryrun mode to the extra params.")
+        elif release_type == NORMAL and audit.is_pointed(data=audit_data):
             trace.info("[RELEASE]: Skip this release because it already audit.")
             return Result(
                 run_id=run_id,
@@ -485,14 +473,6 @@ class Workflow(BaseModel):
                 extras=self.extras,
             )
 
-        if release_type == RERUN:
-            # TODO: It will load previous audit and use this data to run with
-            #   the `rerun` method.
-            raise NotImplementedError(
-                "Release does not support for rerun type yet. Please use the "
-                "`rerun` method instead."
-            )
-
         rs: Result = self.execute(
             params=values,
             run_id=parent_run_id,
@@ -501,27 +481,29 @@ class Workflow(BaseModel):
         catch(context, status=rs.status, updated=rs.context)
         trace.info(f"[RELEASE]: End {name!r} : {release:%Y-%m-%d %H:%M:%S}")
         trace.debug(f"[RELEASE]: Writing audit: {name!r}.")
-        (
-            audit.save(
-                data=audit_data
-                | {
-                    "context": context,
-                    "runs_metadata": (
-                        (runs_metadata or {})
-                        | rs.info
-                        | {
-                            "timeout": timeout,
-                            "original_name": self.name,
-                            "audit_excluded": audit_excluded,
-                        }
-                    ),
-                },
-                excluded=audit_excluded,
+        if release_type != DRYRUN:
+            (
+                audit.save(
+                    data=audit_data
+                    | {
+                        "context": context,
+                        "runs_metadata": (
+                            (runs_metadata or {})
+                            | rs.info
+                            | {
+                                "timeout": timeout,
+                                "original_name": self.name,
+                                "audit_excluded": audit_excluded,
+                            }
+                        ),
+                    },
+                    excluded=audit_excluded,
+                )
             )
-        )
-        return Result(
-            run_id=run_id,
-            parent_run_id=parent_run_id,
+
+        # NOTE: Pop system extra parameters.
+        pop_sys_extras(self.extras, scope="release")
+        return Result.from_trace(trace).catch(
             status=rs.status,
             context=catch(
                 context,
@@ -536,7 +518,6 @@ class Workflow(BaseModel):
                     **(context["errors"] if "errors" in context else {}),
                 },
             ),
-            extras=self.extras,
         )
 
     def execute_job(
@@ -684,8 +665,9 @@ class Workflow(BaseModel):
         :rtype: Result
         """
         ts: float = time.monotonic()
-        parent_run_id: Optional[str] = run_id
-        run_id: str = gen_id(self.name, unique=True, extras=self.extras)
+        parent_run_id, run_id = extract_id(
+            self.name, run_id=run_id, extras=self.extras
+        )
         trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
@@ -741,6 +723,8 @@ class Workflow(BaseModel):
                 extras=self.extras,
             )
 
+        # NOTE: Force update internal extras for handler circle execution.
+        self.extras.update({"__sys_exec_break_circle": self.name})
         with ThreadPoolExecutor(max_job_parallel, "wf") as executor:
             futures: list[Future] = []
 
@@ -769,6 +753,7 @@ class Workflow(BaseModel):
                 backoff_sleep = 0.01
 
                 if check == FAILED:  # pragma: no cov
+                    pop_sys_extras(self.extras)
                     return Result(
                         run_id=run_id,
                         parent_run_id=parent_run_id,
@@ -863,6 +848,7 @@ class Workflow(BaseModel):
                 for i, s in enumerate(sequence_statuses, start=0):
                     statuses[total + 1 + skip_count + i] = s
 
+                pop_sys_extras(self.extras)
                 st: Status = validate_statuses(statuses)
                 return Result(
                     run_id=run_id,
@@ -884,6 +870,7 @@ class Workflow(BaseModel):
 
             time.sleep(0.0025)
 
+        pop_sys_extras(self.extras)
         return Result(
             run_id=run_id,
             parent_run_id=parent_run_id,
@@ -913,6 +900,10 @@ class Workflow(BaseModel):
     ) -> Result:  # pragma: no cov
         """Re-Execute workflow with passing the error context data.
 
+        Warnings:
+            This rerun method allow to rerun job execution level only. That mean
+        it does not support rerun only stage.
+
         Args:
             context: A context result that get the failed status.
             run_id: (Optional[str]) A workflow running ID.

{ddeutil_workflow-0.0.82.dist-info → ddeutil_workflow-0.0.84.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.82
+Version: 0.0.84
 Summary: Lightweight workflow orchestration with YAML template
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.82.dist-info → ddeutil_workflow-0.0.84.dist-info}/RECORD

@@ -1,20 +1,20 @@
-ddeutil/workflow/__about__.py,sha256=GJmjvBMMhA0y8IwyPpPJLqLsuBWa6J90Pleir68QW5I,60
-ddeutil/workflow/__cron.py,sha256=avOagaHl9xXOmizeRWm13cOrty9Tw0vRjFq-xoEgpAY,29167
-ddeutil/workflow/__init__.py,sha256=elWSX2JPbjORH-CJIH_zR_nrdd8Xw95NjLK49R4Kqdg,3434
+ddeutil/workflow/__about__.py,sha256=zX5wSNry4MbxSK_Ujk9sD5ooaxsvpp2kdHcvlV5NcWY,60
+ddeutil/workflow/__cron.py,sha256=-1tqZG7GtUmusdl6NTy_Ck7nM_tGYTXYB7TB7tKeO60,29184
+ddeutil/workflow/__init__.py,sha256=Dvfjs7LpLerGCYGnbqKwznViTw7ire_6LR8obC1I4aM,3456
 ddeutil/workflow/__main__.py,sha256=Nqk5aO-HsZVKV2BmuJYeJEufJluipvCD9R1k2kMoJ3Y,8581
-ddeutil/workflow/__types.py,sha256=tA2vsr6mzTSzbWB1sb62c5GgxODlfVRz6FvgLNJtQao,4788
-ddeutil/workflow/audits.py,sha256=YPnWQYvhILPwPS6RhYKK0OG3lCeYFLuUNBxJvMlXf5w,26109
+ddeutil/workflow/__types.py,sha256=IOKuJCxTUPHh8Z2JoLu_K7a85oq0VOcKBhpabiJ6qEE,5001
+ddeutil/workflow/audits.py,sha256=H8yuMzXs_QAAKNox-HXdojk9CcilHYYQtklJYetoZv8,26955
 ddeutil/workflow/conf.py,sha256=VfPmwaBYEgOj8bu4eim13ayZwJ4Liy7I702aQf7vS8g,17644
-ddeutil/workflow/errors.py,sha256=Rqtuf1MGxA-hKGP5wMAkaeeayst-u4P2dX6Fp_pzbsA,5678
-ddeutil/workflow/event.py,sha256=qm7QHw-Pozm6oIUzAIxpDkPzzVZVtHgJIUlIle0vEfQ,13943
-ddeutil/workflow/job.py,sha256=WHWOVz0ErOUfbN_aqpDeNmvBvpbKhcFlzcvJmlCpJuI,48430
+ddeutil/workflow/errors.py,sha256=J4bEbtI7qtBX7eghod4djLf0y5i1r4mCz_uFU4roLhY,5713
+ddeutil/workflow/event.py,sha256=OumcZBlOZD0_J53GS4V2XJEqQ9HEcIl3UicQrCyL46M,14684
+ddeutil/workflow/job.py,sha256=VVTpxVR2iVEkjvP8r0O0LRtAPnrbsguYbKzHpe2TAVo,48146
 ddeutil/workflow/params.py,sha256=y9f6DEIyae1j4awbj3Kbeq75-U2UPFlKv9K57Hdo_Go,17188
-ddeutil/workflow/result.py,sha256=3Lpyv2Jn6T1Uc-lRbweDucSCoBr0ZByHjffKj14bj6s,9492
+ddeutil/workflow/result.py,sha256=0W3z5wAs3Dyr8r2vRMY5hl1MkvdsyXWJmQD4NmsDDOM,10194
 ddeutil/workflow/reusables.py,sha256=SBLJSxR8ELoWJErBfSMZS3Rr1O_93T-fFBpfn2AvxuA,25007
-ddeutil/workflow/stages.py,sha256=CCR_D6yqVo74PuMxfqhi8GeeAq8sRbyxReHob6yxrjI,123708
-ddeutil/workflow/traces.py,sha256=YN4XuRfQK523cNy8EVgz2iPh6s6WB865K9JezCNdM7E,74637
-ddeutil/workflow/utils.py,sha256=vQwFu-wPK-lDiX2L8AZIahCkKEF6I0MCrZ1LlP8xkoQ,12011
-ddeutil/workflow/workflow.py,sha256=VN8i0mVyuCUw1kk_CqLkN-8dWLON9i4vTLbHp6lyS-s,42961
+ddeutil/workflow/stages.py,sha256=zoTtD2w6ZqaAsi-ilH14Wvg_Wx_AJlZqKZCCOzSprj0,139303
+ddeutil/workflow/traces.py,sha256=pq1lOg2UMgDiSDmjHxXPoTaBHnfc7uzzlo1u2TCwN2Q,74733
+ddeutil/workflow/utils.py,sha256=Hqdh6I3OD4N0Kuehn9qWVOg54pO3BG3rgIacCrN7jqk,12298
+ddeutil/workflow/workflow.py,sha256=1db85LamgwFlYtOJv6ghd3WLREfsP4zQpNOlPXbeuy4,43124
 ddeutil/workflow/api/__init__.py,sha256=5DzYL3ngceoRshh5HYCSVWChqNJSiP01E1bEd8XxPi0,4799
 ddeutil/workflow/api/log_conf.py,sha256=WfS3udDLSyrP-C80lWOvxxmhd_XWKvQPkwDqKblcH3E,1834
 ddeutil/workflow/api/routes/__init__.py,sha256=JRaJZB0D6mgR17MbZo8yLtdYDtD62AA8MdKlFqhG84M,420
@@ -27,9 +27,9 @@ ddeutil/workflow/plugins/providers/aws.py,sha256=61uIFBEWt-_D5Sui24qUPier1Hiqlw_
 ddeutil/workflow/plugins/providers/az.py,sha256=o3dh011lEtmr7-d7FPZJPgXdT0ytFzKfc5xnVxSyXGU,34867
 ddeutil/workflow/plugins/providers/container.py,sha256=DSN0RWxMjTJN5ANheeMauDaPa3X6Z2E1eGUcctYkENw,22134
 ddeutil/workflow/plugins/providers/gcs.py,sha256=KgAOdMBvdbMLTH_z_FwVriBFtZfKEYx8_34jzUOVjTY,27460
-ddeutil_workflow-0.0.82.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.82.dist-info/METADATA,sha256=lB4PBeyneqHLwG0S4Lt_Xz8eCz4Lxn2bkDDHIFcBpxE,16087
-ddeutil_workflow-0.0.82.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ddeutil_workflow-0.0.82.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
-ddeutil_workflow-0.0.82.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.82.dist-info/RECORD,,
+ddeutil_workflow-0.0.84.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.84.dist-info/METADATA,sha256=3GFTGHqYEocyCDqwG7-RcaVfrJ61Es9iiC_tP3Aiy04,16087
+ddeutil_workflow-0.0.84.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ddeutil_workflow-0.0.84.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
+ddeutil_workflow-0.0.84.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.84.dist-info/RECORD,,