highway-dsl 0.0.3__py3-none-any.whl → 1.0.3__py3-none-any.whl

highway_dsl/workflow_dsl.py

@@ -39,6 +39,9 @@ class BaseOperator(BaseModel, ABC):
     retry_policy: Optional[RetryPolicy] = None
     timeout_policy: Optional[TimeoutPolicy] = None
     metadata: Dict[str, Any] = Field(default_factory=dict)
+    is_internal_loop_task: bool = Field(
+        default=False, exclude=True
+    )  # Mark if task is internal to a loop
 
     model_config = ConfigDict(use_enum_values=True, arbitrary_types_allowed=True)
 
@@ -91,7 +94,16 @@ class ParallelOperator(BaseOperator):
 
 class ForEachOperator(BaseOperator):
     items: str
-    task_chain: List[str] = Field(default_factory=list)
+    loop_body: List[
+        Union[
+            TaskOperator,
+            ConditionOperator,
+            WaitOperator,
+            ParallelOperator,
+            "ForEachOperator",
+            "WhileOperator",
+        ]
+    ] = Field(default_factory=list)
     operator_type: OperatorType = Field(OperatorType.FOREACH, frozen=True)
 
 
@@ -204,12 +216,30 @@ class WorkflowBuilder:
         self._current_task: Optional[str] = None
         self.parent = parent
 
+    def _add_task(
+        self,
+        task: Union[
+            TaskOperator,
+            ConditionOperator,
+            WaitOperator,
+            ParallelOperator,
+            ForEachOperator,
+            WhileOperator,
+        ],
+        **kwargs,
+    ) -> None:
+        dependencies = kwargs.get("dependencies", [])
+        if self._current_task and not dependencies:
+            dependencies.append(self._current_task)
+
+        task.dependencies = sorted(list(set(dependencies)))
+
+        self.workflow.add_task(task)
+        self._current_task = task.task_id
+
     def task(self, task_id: str, function: str, **kwargs) -> "WorkflowBuilder":
         task = TaskOperator(task_id=task_id, function=function, **kwargs)
-        if self._current_task:
-            task.dependencies.append(self._current_task)
-        self.workflow.add_task(task)
-        self._current_task = task_id
+        self._add_task(task, **kwargs)
         return self
 
     def condition(
@@ -220,8 +250,8 @@ class WorkflowBuilder:
         if_false: Callable[["WorkflowBuilder"], "WorkflowBuilder"],
         **kwargs,
     ) -> "WorkflowBuilder":
-        true_builder = if_true(WorkflowBuilder(f"{{task_id}}_true", parent=self))
-        false_builder = if_false(WorkflowBuilder(f"{{task_id}}_false", parent=self))
+        true_builder = if_true(WorkflowBuilder(f"{task_id}_true", parent=self))
+        false_builder = if_false(WorkflowBuilder(f"{task_id}_false", parent=self))
 
         true_tasks = list(true_builder.workflow.tasks.keys())
         false_tasks = list(false_builder.workflow.tasks.keys())
@@ -234,14 +264,17 @@ class WorkflowBuilder:
             **kwargs,
         )
 
-        if self._current_task:
-            task.dependencies.append(self._current_task)
-
-        self.workflow.add_task(task)
+        self._add_task(task, **kwargs)
 
         for task_obj in true_builder.workflow.tasks.values():
+            # Only add the condition task as dependency, preserve original dependencies
+            if task_id not in task_obj.dependencies:
+                task_obj.dependencies.append(task_id)
             self.workflow.add_task(task_obj)
         for task_obj in false_builder.workflow.tasks.values():
+            # Only add the condition task as dependency, preserve original dependencies
+            if task_id not in task_obj.dependencies:
+                task_obj.dependencies.append(task_id)
             self.workflow.add_task(task_obj)
 
         self._current_task = task_id
@@ -251,10 +284,7 @@ class WorkflowBuilder:
         self, task_id: str, wait_for: Union[timedelta, datetime, str], **kwargs
     ) -> "WorkflowBuilder":
         task = WaitOperator(task_id=task_id, wait_for=wait_for, **kwargs)
-        if self._current_task:
-            task.dependencies.append(self._current_task)
-        self.workflow.add_task(task)
-        self._current_task = task_id
+        self._add_task(task, **kwargs)
         return self
 
     def parallel(
@@ -263,10 +293,12 @@ class WorkflowBuilder:
         branches: Dict[str, Callable[["WorkflowBuilder"], "WorkflowBuilder"]],
         **kwargs,
     ) -> "WorkflowBuilder":
-        branch_builders = {
-            name: branch_func(WorkflowBuilder(f"{{task_id}}_{{name}}", parent=self))
-            for name, branch_func in branches.items()
-        }
+        branch_builders = {}
+        for name, branch_func in branches.items():
+            branch_builder = branch_func(
+                WorkflowBuilder(f"{task_id}_{name}", parent=self)
+            )
+            branch_builders[name] = branch_builder
 
         branch_tasks = {
             name: list(builder.workflow.tasks.keys())
@@ -275,27 +307,60 @@ class WorkflowBuilder:
 
         task = ParallelOperator(task_id=task_id, branches=branch_tasks, **kwargs)
 
-        if self._current_task:
-            task.dependencies.append(self._current_task)
-
-        self.workflow.add_task(task)
+        self._add_task(task, **kwargs)
 
         for builder in branch_builders.values():
             for task_obj in builder.workflow.tasks.values():
+                # Only add the parallel task as dependency to non-internal tasks,
+                # preserve original dependencies
+                if (
+                    not getattr(task_obj, "is_internal_loop_task", False)
+                    and task_id not in task_obj.dependencies
+                ):
+                    task_obj.dependencies.append(task_id)
                 self.workflow.add_task(task_obj)
 
         self._current_task = task_id
         return self
 
     def foreach(
-        self, task_id: str, items: str, task_chain: List[str], **kwargs
+        self,
+        task_id: str,
+        items: str,
+        loop_body: Callable[["WorkflowBuilder"], "WorkflowBuilder"],
+        **kwargs,
     ) -> "WorkflowBuilder":
+        # Create a temporary builder for the loop body.
+        temp_builder = WorkflowBuilder(f"{task_id}_loop", parent=self)
+        loop_builder = loop_body(temp_builder)
+        loop_tasks = list(loop_builder.workflow.tasks.values())
+
+        # Mark all loop body tasks as internal to prevent parallel dependency injection
+        for task_obj in loop_tasks:
+            task_obj.is_internal_loop_task = True
+
+        # Create the foreach operator
         task = ForEachOperator(
-            task_id=task_id, items=items, task_chain=task_chain, **kwargs
+            task_id=task_id,
+            items=items,
+            loop_body=loop_tasks,
+            **kwargs,
         )
-        if self._current_task:
-            task.dependencies.append(self._current_task)
-        self.workflow.add_task(task)
+
+        # Add the foreach task to workflow to establish initial dependencies
+        self._add_task(task, **kwargs)
+
+        # Add the foreach task as dependency to the FIRST task in the loop body
+        # and preserve the original dependency chain within the loop
+        if loop_tasks:
+            first_task = loop_tasks[0]
+            if task_id not in first_task.dependencies:
+                first_task.dependencies.append(task_id)
+
+            # Add all loop tasks to workflow
+            for task_obj in loop_tasks:
+                self.workflow.add_task(task_obj)
+
         self._current_task = task_id
         return self
 
@@ -306,9 +371,13 @@ class WorkflowBuilder:
         loop_body: Callable[["WorkflowBuilder"], "WorkflowBuilder"],
         **kwargs,
     ) -> "WorkflowBuilder":
-        loop_builder = loop_body(WorkflowBuilder(f"{{task_id}}_loop", parent=self))
+        loop_builder = loop_body(WorkflowBuilder(f"{task_id}_loop", parent=self))
         loop_tasks = list(loop_builder.workflow.tasks.values())
 
+        # Mark all loop body tasks as internal to prevent parallel dependency injection
+        for task_obj in loop_tasks:
+            task_obj.is_internal_loop_task = True
+
         task = WhileOperator(
             task_id=task_id,
             condition=condition,
@@ -316,13 +385,18 @@ class WorkflowBuilder:
             **kwargs,
         )
 
-        if self._current_task:
-            task.dependencies.append(self._current_task)
+        self._add_task(task, **kwargs)
 
-        self.workflow.add_task(task)
+        # Fix: Only add the while task as dependency to the FIRST task in the loop body
+        # and preserve the original dependency chain within the loop
+        if loop_tasks:
+            first_task = loop_tasks[0]
+            if task_id not in first_task.dependencies:
+                first_task.dependencies.append(task_id)
 
-        for task_obj in loop_tasks:
-            self.workflow.add_task(task_obj)
+            # Add all loop tasks to workflow without modifying their dependencies further
+            for task_obj in loop_tasks:
+                self.workflow.add_task(task_obj)
 
         self._current_task = task_id
         return self

{highway_dsl-0.0.3.dist-info → highway_dsl-1.0.3.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: highway_dsl
-Version: 0.0.3
-Summary: A domain specific language (DSL) for defining and managing data processing pipelines.
+Version: 1.0.3
+Summary: A stable domain specific language (DSL) for defining and managing data processing pipelines and workflow engines.
 Author-email: Farseed Ashouri <farseed.ashouri@gmail.com>
 License: MIT
 Project-URL: Homepage, https://github.com/rodmena-limited/highway_dsl
@@ -25,21 +25,29 @@ Dynamic: license-file
 
 [![PyPI version](https://badge.fury.io/py/highway-dsl.svg)](https://badge.fury.io/py/highway-dsl)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Stable](https://img.shields.io/badge/Status-Stable-brightgreen)](https://pypi.org/project/highway-dsl/)
+[![Publish to PyPI](https://github.com/rodmena-limited/highway_dsl/actions/workflows/publish.yml/badge.svg)](https://github.com/rodmena-limited/highway_dsl/actions/workflows/publish.yml)
 
 **Highway DSL** is a Python-based domain-specific language for defining complex workflows in a clear, concise, and fluent manner. It is part of the larger **Highway** project, an advanced workflow engine capable of running complex DAG-based workflows.
 
+## Version 1.0.3 - Stable Release
+
+This is a stable release with important bug fixes and enhancements, including a critical fix for the ForEach operator dependency management issue.
+
 ## Features
 
 *   **Fluent API:** A powerful and intuitive `WorkflowBuilder` for defining workflows programmatically.
 *   **Pydantic-based:** All models are built on Pydantic, providing robust data validation, serialization, and documentation.
 *   **Rich Operators:** A comprehensive set of operators for handling various workflow scenarios:
-    *   `Task`
-    *   `Condition` (if/else)
-    *   `Parallel`
-    *   `ForEach`
-    *   `Wait`
-    *   `While`
+    *   `Task` - Basic workflow steps
+    *   `Condition` (if/else) - Conditional branching
+    *   `Parallel` - Execute multiple branches simultaneously 
+    *   `ForEach` - Iterate over collections with proper dependency management
+    *   `Wait` - Pause execution for scheduled tasks
+    *   `While` - Execute loops based on conditions
+*   **Fixed ForEach Bug:** Proper encapsulation of loop body tasks to prevent unwanted "grandparent" dependencies from containing parallel operators.
 *   **YAML/JSON Interoperability:** Workflows can be defined in Python and exported to YAML or JSON, and vice-versa.
+*   **Retry and Timeout Policies:** Built-in error handling and execution time management.
 *   **Extensible:** The DSL is designed to be extensible with custom operators and policies.
 
 ## Installation
@@ -135,6 +143,63 @@ builder.task("finalize_product", "workflows.tasks.finalize_product", dependencie
 workflow = builder.build()
 ```
 
+### For-Each Loops with Proper Dependency Management
+
+Fixed bug where foreach loops were incorrectly inheriting dependencies from containing parallel operators:
+
+```python
+# This loop now properly encapsulates its internal tasks
+builder.foreach(
+    "process_items",
+    items="{{data.items}}",
+    loop_body=lambda fb: fb.task("process_item", "processor.handle_item", args=["{{item.id}}"])
+    # Loop body tasks only have proper dependencies, not unwanted "grandparent" dependencies
+)
+```
+
+### Retry Policies
+
+```python
+from highway_dsl import RetryPolicy
+from datetime import timedelta
+
+builder.task(
+    "reliable_task",
+    "service.operation",
+    retry_policy=RetryPolicy(
+        max_retries=5,
+        delay=timedelta(seconds=10),
+        backoff_factor=2.0
+    )
+)
+```
+
+### Timeout Policies
+
+```python
+from highway_dsl import TimeoutPolicy
+from datetime import timedelta
+
+builder.task(
+    "timed_task",
+    "service.operation",
+    timeout_policy=TimeoutPolicy(
+        timeout=timedelta(hours=1),
+        kill_on_timeout=True
+    )
+)
+```
+
+## What's New in Version 1.0.2
+
+### Bug Fixes
+* **Fixed ForEach Operator Bug**: Resolved issue where foreach loops were incorrectly getting "grandparent" dependencies from containing parallel operators. Loop body tasks are now properly encapsulated and only depend on their parent loop operator and internal chain dependencies.
+
+### Enhancements
+* **Improved Loop Dependency Management**: While loops and ForEach loops now properly encapsulate their internal dependencies without being affected by containing parallel operators.
+* **Better Error Handling**: Enhanced error handling throughout the DSL.
+* **Comprehensive Test Suite**: Added functional tests for all example workflows to ensure consistency.
+
 ## Development
 
 To set up the development environment:
@@ -158,3 +223,7 @@ pytest
 ```bash
 mypy .
 ```
+
+## License
+
+MIT License

highway_dsl-1.0.3.dist-info/RECORD

@@ -0,0 +1,7 @@
+highway_dsl/__init__.py,sha256=mr1oMylxliFwu2VO2qpyM3sVQwYIoPL2P6JE-6ZuF7M,507
+highway_dsl/workflow_dsl.py,sha256=bhCKDPrMaIkEI4HduKoeqd2VlZsK8wjr8RURifPufGU,14700
+highway_dsl-1.0.3.dist-info/licenses/LICENSE,sha256=qdFq1H66BvKg67mf4-WGpFwtG2u_dNknxuJDQ1_ubaY,1072
+highway_dsl-1.0.3.dist-info/METADATA,sha256=8rf2NIvAJ5pgDXzs9r2VjgePNqXTOlJbOL0nHRTvsSg,7366
+highway_dsl-1.0.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+highway_dsl-1.0.3.dist-info/top_level.txt,sha256=_5uX-bbBsQ2rsi1XMr7WRyKbr6ack5GqVBcy-QjF1C8,12
+highway_dsl-1.0.3.dist-info/RECORD,,

highway_dsl-0.0.3.dist-info/RECORD

@@ -1,7 +0,0 @@
-highway_dsl/__init__.py,sha256=mr1oMylxliFwu2VO2qpyM3sVQwYIoPL2P6JE-6ZuF7M,507
-highway_dsl/workflow_dsl.py,sha256=yMTmFr5bbjxfVTleCvSsDZ__n9C7qH39RdzajkUEmiI,11882
-highway_dsl-0.0.3.dist-info/licenses/LICENSE,sha256=qdFq1H66BvKg67mf4-WGpFwtG2u_dNknxuJDQ1_ubaY,1072
-highway_dsl-0.0.3.dist-info/METADATA,sha256=--TFErjeBDZ1mAyNHk30CQavLuKWAaoxgHK7xFpT-Ok,4612
-highway_dsl-0.0.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-highway_dsl-0.0.3.dist-info/top_level.txt,sha256=_5uX-bbBsQ2rsi1XMr7WRyKbr6ack5GqVBcy-QjF1C8,12
-highway_dsl-0.0.3.dist-info/RECORD,,