ddeutil-workflow 0.0.81__py3-none-any.whl → 0.0.83__py3-none-any.whl

ddeutil/workflow/workflow.py

@@ -30,19 +30,19 @@ from concurrent.futures import (
     as_completed,
 )
 from datetime import datetime
-from enum import Enum
 from pathlib import Path
 from queue import Queue
 from textwrap import dedent
 from threading import Event as ThreadEvent
-from typing import Any, Optional, Union
+from typing import Any, Literal, Optional, Union
 
 from pydantic import BaseModel, Field
+from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
 from .__types import DictData
-from .audits import Audit, get_audit
+from .audits import NORMAL, RERUN, Audit, ReleaseType, get_audit
 from .conf import YamlParser, dynamic
 from .errors import WorkflowCancelError, WorkflowError, WorkflowTimeoutError
 from .event import Event
@@ -61,40 +61,15 @@ from .result import (
     validate_statuses,
 )
 from .reusables import has_template, param2template
-from .traces import TraceManager, get_trace
+from .traces import Trace, get_trace
 from .utils import (
-    UTC,
+    extract_id,
     gen_id,
     get_dt_now,
-    replace_sec,
+    remove_sys_extras,
 )
 
 
-class ReleaseType(str, Enum):
-    """Release type enumeration for workflow execution modes.
-
-    This enum defines the different types of workflow releases that can be
-    triggered, each with specific behavior and use cases.
-
-    Attributes:
-        NORMAL: Standard workflow release execution
-        RERUN: Re-execution of previously failed workflow
-        EVENT: Event-triggered workflow execution
-        FORCE: Forced execution bypassing normal conditions
-    """
-
-    NORMAL = "normal"
-    RERUN = "rerun"
-    EVENT = "event"
-    FORCE = "force"
-
-
-NORMAL = ReleaseType.NORMAL
-RERUN = ReleaseType.RERUN
-EVENT = ReleaseType.EVENT
-FORCE = ReleaseType.FORCE
-
-
 class Workflow(BaseModel):
     """Main workflow orchestration model for job and schedule management.
 
@@ -113,17 +88,6 @@ class Workflow(BaseModel):
         on (list[Crontab]): Schedule definitions using cron expressions
         jobs (dict[str, Job]): Collection of jobs within this workflow
 
-    Example:
-        Create and execute a workflow:
-
-        ```python
-        workflow = Workflow.from_conf('my-workflow')
-        result = workflow.execute({
-            'param1': 'value1',
-            'param2': 'value2'
-        })
-        ```
-
     Note:
         Workflows can be executed immediately or scheduled for background
         execution using the cron-like scheduling system.
@@ -134,6 +98,7 @@ class Workflow(BaseModel):
         description="An extra parameters that want to override config values.",
     )
     name: str = Field(description="A workflow name.")
+    type: Literal["Workflow"] = Field(default="workflow")
     desc: Optional[str] = Field(
         default=None,
         description=(
@@ -198,10 +163,10 @@ class Workflow(BaseModel):
             FileNotFoundError: If workflow configuration file not found
 
         Example:
-            >>> # Load from default config path
+            >>> # NOTE: Load from default config path
             >>> workflow = Workflow.from_conf('data-pipeline')
 
-            >>> # Load with custom path and extras
+            >>> # NOTE: Load with custom path and extras
             >>> workflow = Workflow.from_conf(
             ...     'data-pipeline',
             ...     path=Path('./custom-configs'),
@@ -211,7 +176,6 @@ class Workflow(BaseModel):
         load: YamlParser = YamlParser(name, path=path, extras=extras, obj=cls)
         data: DictData = copy.deepcopy(load.data)
         data["name"] = name
-
         if extras:
             data["extras"] = extras
 
@@ -280,17 +244,29 @@ class Workflow(BaseModel):
                 f"{self.name!r}."
             )
 
+        # NOTE: Force update internal extras for handler circle execution.
+        self.extras.update({"__sys_break_circle_exec": self.name})
+
         return self
 
+    @field_serializer("extras")
+    def __serialize_extras(self, extras: DictData) -> DictData:
+        return remove_sys_extras(extras)
+
     def detail(self) -> DictData:  # pragma: no cov
         """Return the detail of this workflow for generate markdown."""
         return self.model_dump(by_alias=True)
 
     def md(self, author: Optional[str] = None) -> str:  # pragma: no cov
-        """Generate the markdown template."""
+        """Generate the markdown template from this Workflow model data.
+
+        Args:
+            author (str | None, default None): An author name.
+        """
 
         def align_newline(value: str) -> str:
-            return value.rstrip("\n").replace("\n", "\n                ")
+            space: str = " " * 16
+            return value.rstrip("\n").replace("\n", f"\n{space}")
 
         info: str = (
             f"| Author: {author or 'nobody'} "
@@ -317,8 +293,7 @@ class Workflow(BaseModel):
                 {align_newline(self.desc)}\n
                 ## Parameters\n
                 | name | type | default | description |
-                | --- | --- | --- | : --- : |
-
+                | --- | --- | --- | : --- : |\n\n
                 ## Jobs\n
                 {align_newline(jobs)}
                 """.lstrip(
@@ -347,8 +322,7 @@ class Workflow(BaseModel):
                 f"{self.name!r}"
             )
         job: Job = self.jobs[name]
-        if self.extras:
-            job.extras = self.extras
+        job.extras = self.extras
         return job
 
     def parameterize(self, params: DictData) -> DictData:
@@ -367,8 +341,8 @@ class Workflow(BaseModel):
                 execute method.
 
         Returns:
-            DictData: The parameter value that validate with its parameter fields and
-                adding jobs key to this parameter.
+            DictData: The parameter value that validate with its parameter fields
+                and adding jobs key to this parameter.
 
         Raises:
             WorkflowError: If parameter value that want to validate does
@@ -399,33 +373,6 @@ class Workflow(BaseModel):
             "jobs": {},
         }
 
-    def validate_release(self, dt: datetime) -> datetime:
-        """Validate the release datetime that should was replaced second and
-        millisecond to 0 and replaced timezone to None before checking it match
-        with the set `on` field.
-
-        Args:
-            dt (datetime): A datetime object that want to validate.
-
-        Returns:
-            datetime: The validated release datetime.
-        """
-        if dt.tzinfo is None:
-            dt = dt.replace(tzinfo=UTC)
-
-        release: datetime = replace_sec(dt.astimezone(UTC))
-
-        # NOTE: Return itself if schedule event does not set.
-        if not self.on.schedule:
-            return release
-
-        for on in self.on.schedule:
-            if release == on.cronjob.schedule(release, tz=UTC).next:
-                return release
-        raise WorkflowError(
-            "Release datetime does not support for this workflow"
-        )
-
     def release(
         self,
         release: datetime,
@@ -437,7 +384,7 @@ class Workflow(BaseModel):
         override_log_name: Optional[str] = None,
         timeout: int = 600,
         audit_excluded: Optional[list[str]] = None,
-        audit: type[Audit] = None,
+        audit: Audit = None,
     ) -> Result:
         """Release the workflow which is executes workflow with writing audit
         log tracking. The method is overriding parameter with the release
@@ -455,11 +402,12 @@ class Workflow(BaseModel):
 
         Args:
             release (datetime): A release datetime.
-            params: A workflow parameter that pass to execute method.
-            release_type:
+            params (DictData): A workflow parameter that pass to execute method.
+            release_type (ReleaseType): A release type that want to execute.
             run_id: (str) A workflow running ID.
             runs_metadata: (DictData)
-            audit: An audit class that want to save the execution result.
+            audit (Audit): An audit model that use to manage release log of this
+                execution.
             override_log_name: (str) An override logging name that use
                 instead the workflow name.
             timeout: (int) A workflow execution time out in second unit.
@@ -471,20 +419,25 @@ class Workflow(BaseModel):
                 method.
         """
         name: str = override_log_name or self.name
+        audit: Audit = audit or get_audit(extras=self.extras)
 
         # NOTE: Generate the parent running ID with not None value.
-        if run_id:
-            parent_run_id: str = run_id
-            run_id: str = gen_id(name, unique=True)
-        else:
-            run_id: str = gen_id(name, unique=True)
-            parent_run_id: str = run_id
-
+        parent_run_id, run_id = extract_id(
+            name, run_id=run_id, extras=self.extras
+        )
         context: DictData = {"status": WAIT}
-        trace: TraceManager = get_trace(
+        audit_data: DictData = {
+            "name": name,
+            "release": release,
+            "type": release_type,
+            "run_id": run_id,
+            "parent_run_id": parent_run_id,
+            "extras": self.extras,
+        }
+        trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
-        release: datetime = self.validate_release(dt=release)
+        release: datetime = self.on.validate_dt(dt=release)
         trace.info(f"[RELEASE]: Start {name!r} : {release:%Y-%m-%d %H:%M:%S}")
         values: DictData = param2template(
             params,
@@ -498,6 +451,25 @@ class Workflow(BaseModel):
             },
             extras=self.extras,
         )
+
+        if release_type == NORMAL and audit.is_pointed(data=audit_data):
+            trace.info("[RELEASE]: Skip this release because it already audit.")
+            return Result(
+                run_id=run_id,
+                parent_run_id=parent_run_id,
+                status=SKIP,
+                context=catch(context, status=SKIP),
+                extras=self.extras,
+            )
+
+        if release_type == RERUN:
+            # TODO: It will load previous audit and use this data to run with
+            #   the `rerun` method.
+            raise NotImplementedError(
+                "Release does not support for rerun type yet. Please use the "
+                "`rerun` method instead."
+            )
+
         rs: Result = self.execute(
             params=values,
             run_id=parent_run_id,
@@ -507,15 +479,10 @@ class Workflow(BaseModel):
         trace.info(f"[RELEASE]: End {name!r} : {release:%Y-%m-%d %H:%M:%S}")
         trace.debug(f"[RELEASE]: Writing audit: {name!r}.")
         (
-            (audit or get_audit(extras=self.extras)).save(
-                data={
-                    "name": name,
-                    "release": release,
-                    "type": release_type,
+            audit.save(
+                data=audit_data
+                | {
                     "context": context,
-                    "parent_run_id": parent_run_id,
-                    "run_id": run_id,
-                    "extras": self.extras,
                     "runs_metadata": (
                         (runs_metadata or {})
                         | rs.info
@@ -546,7 +513,7 @@ class Workflow(BaseModel):
                     **(context["errors"] if "errors" in context else {}),
                 },
             ),
-            extras=self.extras,
+            extras=remove_sys_extras(self.extras),
         )
 
     def execute_job(
@@ -579,7 +546,7 @@ class Workflow(BaseModel):
         Returns:
             tuple[Status, DictData]: The pair of status and result context data.
         """
-        trace: TraceManager = get_trace(
+        trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
         if event and event.is_set():
@@ -694,9 +661,10 @@ class Workflow(BaseModel):
         :rtype: Result
         """
         ts: float = time.monotonic()
-        parent_run_id: Optional[str] = run_id
-        run_id: str = gen_id(self.name, extras=self.extras)
-        trace: TraceManager = get_trace(
+        parent_run_id, run_id = extract_id(
+            self.name, run_id=run_id, extras=self.extras
+        )
+        trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
         context: DictData = self.parameterize(params)
@@ -733,6 +701,11 @@ class Workflow(BaseModel):
         )
         catch(context, status=WAIT)
         if event and event.is_set():
+            err_msg: str = (
+                "Execution was canceled from the event was set "
+                "before workflow execution."
+            )
+            trace.error(f"[WORKFLOW]: {err_msg}")
             return Result(
                 run_id=run_id,
                 parent_run_id=parent_run_id,
@@ -740,12 +713,7 @@ class Workflow(BaseModel):
                 context=catch(
                     context,
                     status=CANCEL,
-                    updated={
-                        "errors": WorkflowCancelError(
-                            "Execution was canceled from the event was set "
-                            "before workflow execution."
-                        ).to_dict(),
-                    },
+                    updated={"errors": WorkflowCancelError(err_msg).to_dict()},
                 ),
                 info={"execution_time": time.monotonic() - ts},
                 extras=self.extras,
@@ -799,7 +767,7 @@ class Workflow(BaseModel):
                     )
                 elif check == SKIP:  # pragma: no cov
                     trace.info(
-                        f"[JOB]: Skip job: {job_id!r} from trigger rule."
+                        f"[JOB]: ⏭️ Skip job: {job_id!r} from trigger rule."
                     )
                     job.set_outputs(output={"status": SKIP}, to=context)
                     job_queue.task_done()
@@ -923,16 +891,21 @@ class Workflow(BaseModel):
     ) -> Result:  # pragma: no cov
         """Re-Execute workflow with passing the error context data.
 
-        :param context: A context result that get the failed status.
-        :param run_id: (Optional[str]) A workflow running ID.
-        :param event: (Event) An Event manager instance that use to cancel this
-            execution if it forces stopped by parent execution.
-        :param timeout: (float) A workflow execution time out in second unit
-            that use for limit time of execution and waiting job dependency.
-            This value does not force stop the task that still running more than
-            this limit time. (Default: 60 * 60 seconds)
-        :param max_job_parallel: (int) The maximum workers that use for job
-            execution in `ThreadPoolExecutor` object. (Default: 2 workers)
+        Warnings:
+            This rerun method allow to rerun job execution level only. That mean
+        it does not support rerun only stage.
+
+        Args:
+            context: A context result that get the failed status.
+            run_id: (Optional[str]) A workflow running ID.
+            event: (Event) An Event manager instance that use to cancel this
+                execution if it forces stopped by parent execution.
+            timeout: (float) A workflow execution time out in second unit
+                that use for limit time of execution and waiting job dependency.
+                This value does not force stop the task that still running more
+                than this limit time. (Default: 60 * 60 seconds)
+            max_job_parallel: (int) The maximum workers that use for job
+                execution in `ThreadPoolExecutor` object. (Default: 2 workers)
 
         Returns
             Result: Return Result object that create from execution context with
@@ -941,7 +914,7 @@ class Workflow(BaseModel):
         ts: float = time.monotonic()
         parent_run_id: str = run_id
         run_id: str = gen_id(self.name, extras=self.extras)
-        trace: TraceManager = get_trace(
+        trace: Trace = get_trace(
             run_id, parent_run_id=parent_run_id, extras=self.extras
         )
         if context["status"] == SUCCESS:

{ddeutil_workflow-0.0.81.dist-info → ddeutil_workflow-0.0.83.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.81
+Version: 0.0.83
 Summary: Lightweight workflow orchestration with YAML template
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

ddeutil_workflow-0.0.83.dist-info/RECORD

@@ -0,0 +1,35 @@
+ddeutil/workflow/__about__.py,sha256=6n_dCde_BwTTJlP4Yu4HNgiRti7QydfShki703lt6Ro,60
+ddeutil/workflow/__cron.py,sha256=-1tqZG7GtUmusdl6NTy_Ck7nM_tGYTXYB7TB7tKeO60,29184
+ddeutil/workflow/__init__.py,sha256=Dvfjs7LpLerGCYGnbqKwznViTw7ire_6LR8obC1I4aM,3456
+ddeutil/workflow/__main__.py,sha256=Nqk5aO-HsZVKV2BmuJYeJEufJluipvCD9R1k2kMoJ3Y,8581
+ddeutil/workflow/__types.py,sha256=IOKuJCxTUPHh8Z2JoLu_K7a85oq0VOcKBhpabiJ6qEE,5001
+ddeutil/workflow/audits.py,sha256=H8yuMzXs_QAAKNox-HXdojk9CcilHYYQtklJYetoZv8,26955
+ddeutil/workflow/conf.py,sha256=VfPmwaBYEgOj8bu4eim13ayZwJ4Liy7I702aQf7vS8g,17644
+ddeutil/workflow/errors.py,sha256=J4bEbtI7qtBX7eghod4djLf0y5i1r4mCz_uFU4roLhY,5713
+ddeutil/workflow/event.py,sha256=OumcZBlOZD0_J53GS4V2XJEqQ9HEcIl3UicQrCyL46M,14684
+ddeutil/workflow/job.py,sha256=VVTpxVR2iVEkjvP8r0O0LRtAPnrbsguYbKzHpe2TAVo,48146
+ddeutil/workflow/params.py,sha256=y9f6DEIyae1j4awbj3Kbeq75-U2UPFlKv9K57Hdo_Go,17188
+ddeutil/workflow/result.py,sha256=0W3z5wAs3Dyr8r2vRMY5hl1MkvdsyXWJmQD4NmsDDOM,10194
+ddeutil/workflow/reusables.py,sha256=SBLJSxR8ELoWJErBfSMZS3Rr1O_93T-fFBpfn2AvxuA,25007
+ddeutil/workflow/stages.py,sha256=lWlzvpJ6YyhDf0ks5q_fzHjm4-o6UZfhiYp9CG-ffro,129661
+ddeutil/workflow/traces.py,sha256=pq1lOg2UMgDiSDmjHxXPoTaBHnfc7uzzlo1u2TCwN2Q,74733
+ddeutil/workflow/utils.py,sha256=XsH8DkcTiMmWt1e59b4bFQofsBdo7uW1-7gC2rghuW8,12128
+ddeutil/workflow/workflow.py,sha256=uc71PJh7e-Bjb3Xg7T83wlLrKTPGvmX7Qjsn6SJ1GDI,42544
+ddeutil/workflow/api/__init__.py,sha256=5DzYL3ngceoRshh5HYCSVWChqNJSiP01E1bEd8XxPi0,4799
+ddeutil/workflow/api/log_conf.py,sha256=WfS3udDLSyrP-C80lWOvxxmhd_XWKvQPkwDqKblcH3E,1834
+ddeutil/workflow/api/routes/__init__.py,sha256=JRaJZB0D6mgR17MbZo8yLtdYDtD62AA8MdKlFqhG84M,420
+ddeutil/workflow/api/routes/job.py,sha256=-lbZ_hS9pEdSy6zeke5qrXEgdNxtQ2w9in7cHuM2Jzs,2536
+ddeutil/workflow/api/routes/logs.py,sha256=9jiYsw8kepud4n3NyXB7SAr2OoQwRn5uNb9kIZ58XJM,3806
+ddeutil/workflow/api/routes/workflows.py,sha256=0pEZEsIrscRFBXG9gf6nttKw0aNbcdw7NsAZKLoKWtk,4392
+ddeutil/workflow/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ddeutil/workflow/plugins/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ddeutil/workflow/plugins/providers/aws.py,sha256=61uIFBEWt-_D5Sui24qUPier1Hiqlw_RP_eY-rXBCKc,31551
+ddeutil/workflow/plugins/providers/az.py,sha256=o3dh011lEtmr7-d7FPZJPgXdT0ytFzKfc5xnVxSyXGU,34867
+ddeutil/workflow/plugins/providers/container.py,sha256=DSN0RWxMjTJN5ANheeMauDaPa3X6Z2E1eGUcctYkENw,22134
+ddeutil/workflow/plugins/providers/gcs.py,sha256=KgAOdMBvdbMLTH_z_FwVriBFtZfKEYx8_34jzUOVjTY,27460
+ddeutil_workflow-0.0.83.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.83.dist-info/METADATA,sha256=pxD6FyTSV4ra5DujoPvWQnb2Z9WWHHqi97HBADqeeAo,16087
+ddeutil_workflow-0.0.83.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ddeutil_workflow-0.0.83.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
+ddeutil_workflow-0.0.83.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.83.dist-info/RECORD,,