highway-dsl 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

highway_dsl/__init__.py

@@ -0,0 +1,28 @@
+from .workflow_dsl import (
+    Workflow,
+    WorkflowBuilder,
+    TaskOperator,
+    ConditionOperator,
+    ParallelOperator,
+    WaitOperator,
+    ForEachOperator,
+    WhileOperator,
+    RetryPolicy,
+    TimeoutPolicy,
+    OperatorType,
+)
+
+__all__ = [
+    "Workflow",
+    "WorkflowBuilder",
+    "BaseOperator",
+    "TaskOperator",
+    "ConditionOperator",
+    "ParallelOperator",
+    "WaitOperator",
+    "ForEachOperator",
+    "WhileOperator",
+    "RetryPolicy",
+    "TimeoutPolicy",
+    "OperatorType",
+]

highway_dsl/workflow_dsl.py

@@ -16,6 +16,7 @@ class OperatorType(Enum):
     FOREACH = "foreach"
     SWITCH = "switch"
     TRY_CATCH = "try_catch"
+    WHILE = "while"
 
 
 class RetryPolicy(BaseModel):
@@ -26,7 +27,9 @@ class RetryPolicy(BaseModel):
 
 class TimeoutPolicy(BaseModel):
     timeout: timedelta = Field(..., description="Timeout duration")
-    kill_on_timeout: bool = Field(True, description="Whether to kill the task on timeout")
+    kill_on_timeout: bool = Field(
+        True, description="Whether to kill the task on timeout"
+    )
 
 
 class BaseOperator(BaseModel, ABC):
@@ -50,8 +53,8 @@ class TaskOperator(BaseOperator):
 
 class ConditionOperator(BaseOperator):
     condition: str
-    if_true: str
-    if_false: str
+    if_true: Optional[str]
+    if_false: Optional[str]
     operator_type: OperatorType = Field(OperatorType.CONDITION, frozen=True)
 
 
@@ -59,7 +62,7 @@ class WaitOperator(BaseOperator):
     wait_for: Union[timedelta, datetime, str]
     operator_type: OperatorType = Field(OperatorType.WAIT, frozen=True)
 
-    @model_validator(mode='before')
+    @model_validator(mode="before")
     @classmethod
     def parse_wait_for(cls, data: Any) -> Any:
         if isinstance(data, dict) and "wait_for" in data:
@@ -92,21 +95,40 @@ class ForEachOperator(BaseOperator):
     operator_type: OperatorType = Field(OperatorType.FOREACH, frozen=True)
 
 
+class WhileOperator(BaseOperator):
+    condition: str
+    loop_body: List[
+        Union[
+            TaskOperator,
+            ConditionOperator,
+            WaitOperator,
+            ParallelOperator,
+            ForEachOperator,
+            "WhileOperator",
+        ]
+    ] = Field(default_factory=list)
+    operator_type: OperatorType = Field(OperatorType.WHILE, frozen=True)
+
+
 class Workflow(BaseModel):
     name: str
     version: str = "1.0.0"
     description: str = ""
-    tasks: Dict[str, Union[
-        TaskOperator,
-        ConditionOperator,
-        WaitOperator,
-        ParallelOperator,
-        ForEachOperator,
-    ]] = Field(default_factory=dict)
+    tasks: Dict[
+        str,
+        Union[
+            TaskOperator,
+            ConditionOperator,
+            WaitOperator,
+            ParallelOperator,
+            ForEachOperator,
+            WhileOperator,
+        ],
+    ] = Field(default_factory=dict)
     variables: Dict[str, Any] = Field(default_factory=dict)
     start_task: Optional[str] = None
 
-    @model_validator(mode='before')
+    @model_validator(mode="before")
     @classmethod
     def validate_tasks(cls, data: Any) -> Any:
         if isinstance(data, dict) and "tasks" in data:
@@ -117,6 +139,7 @@ class Workflow(BaseModel):
                 OperatorType.WAIT.value: WaitOperator,
                 OperatorType.PARALLEL.value: ParallelOperator,
                 OperatorType.FOREACH.value: ForEachOperator,
+                OperatorType.WHILE.value: WhileOperator,
             }
             for task_id, task_data in data["tasks"].items():
                 operator_type = task_data.get("operator_type")
@@ -128,13 +151,17 @@ class Workflow(BaseModel):
             data["tasks"] = validated_tasks
         return data
 
-    def add_task(self, task: Union[
-        TaskOperator,
-        ConditionOperator,
-        WaitOperator,
-        ParallelOperator,
-        ForEachOperator,
-    ]) -> "Workflow":
+    def add_task(
+        self,
+        task: Union[
+            TaskOperator,
+            ConditionOperator,
+            WaitOperator,
+            ParallelOperator,
+            ForEachOperator,
+            WhileOperator,
+        ],
+    ) -> "Workflow":
         self.tasks[task.task_id] = task
         return self
 
@@ -147,7 +174,7 @@ class Workflow(BaseModel):
         return self
 
     def to_yaml(self) -> str:
-        data = self.model_dump(mode='json', by_alias=True, exclude_none=True)
+        data = self.model_dump(mode="json", by_alias=True, exclude_none=True)
         return yaml.dump(data, default_flow_style=False)
 
     def to_json(self) -> str:
@@ -164,12 +191,18 @@ class Workflow(BaseModel):
 
 
 class WorkflowBuilder:
-    def __init__(self, name: str, existing_workflow: Optional[Workflow] = None):
+    def __init__(
+        self,
+        name: str,
+        existing_workflow: Optional[Workflow] = None,
+        parent: Optional["WorkflowBuilder"] = None,
+    ):
         if existing_workflow:
             self.workflow = existing_workflow
         else:
             self.workflow = Workflow(name=name)
         self._current_task: Optional[str] = None
+        self.parent = parent
 
     def task(self, task_id: str, function: str, **kwargs) -> "WorkflowBuilder":
         task = TaskOperator(task_id=task_id, function=function, **kwargs)
@@ -180,14 +213,37 @@ class WorkflowBuilder:
         return self
 
     def condition(
-        self, task_id: str, condition: str, if_true: str, if_false: str, **kwargs
+        self,
+        task_id: str,
+        condition: str,
+        if_true: Callable[["WorkflowBuilder"], "WorkflowBuilder"],
+        if_false: Callable[["WorkflowBuilder"], "WorkflowBuilder"],
+        **kwargs,
     ) -> "WorkflowBuilder":
+        true_builder = if_true(WorkflowBuilder(f"{{task_id}}_true", parent=self))
+        false_builder = if_false(WorkflowBuilder(f"{{task_id}}_false", parent=self))
+
+        true_tasks = list(true_builder.workflow.tasks.keys())
+        false_tasks = list(false_builder.workflow.tasks.keys())
+
         task = ConditionOperator(
-            task_id=task_id, condition=condition, if_true=if_true, if_false=if_false, **kwargs
+            task_id=task_id,
+            condition=condition,
+            if_true=true_tasks[0] if true_tasks else None,
+            if_false=false_tasks[0] if false_tasks else None,
+            **kwargs,
         )
+
         if self._current_task:
             task.dependencies.append(self._current_task)
+
         self.workflow.add_task(task)
+
+        for task_obj in true_builder.workflow.tasks.values():
+            self.workflow.add_task(task_obj)
+        for task_obj in false_builder.workflow.tasks.values():
+            self.workflow.add_task(task_obj)
+
         self._current_task = task_id
         return self
 
@@ -202,22 +258,72 @@ class WorkflowBuilder:
         return self
 
     def parallel(
-        self, task_id: str, branches: Dict[str, List[str]], **kwargs
+        self,
+        task_id: str,
+        branches: Dict[str, Callable[["WorkflowBuilder"], "WorkflowBuilder"]],
+        **kwargs,
     ) -> "WorkflowBuilder":
-        task = ParallelOperator(task_id=task_id, branches=branches, **kwargs)
+        branch_builders = {
+            name: branch_func(WorkflowBuilder(f"{{task_id}}_{{name}}", parent=self))
+            for name, branch_func in branches.items()
+        }
+
+        branch_tasks = {
+            name: list(builder.workflow.tasks.keys())
+            for name, builder in branch_builders.items()
+        }
+
+        task = ParallelOperator(task_id=task_id, branches=branch_tasks, **kwargs)
+
         if self._current_task:
             task.dependencies.append(self._current_task)
+
         self.workflow.add_task(task)
+
+        for builder in branch_builders.values():
+            for task_obj in builder.workflow.tasks.values():
+                self.workflow.add_task(task_obj)
+
         self._current_task = task_id
         return self
 
     def foreach(
         self, task_id: str, items: str, task_chain: List[str], **kwargs
     ) -> "WorkflowBuilder":
-        task = ForEachOperator(task_id=task_id, items=items, task_chain=task_chain, **kwargs)
+        task = ForEachOperator(
+            task_id=task_id, items=items, task_chain=task_chain, **kwargs
+        )
+        if self._current_task:
+            task.dependencies.append(self._current_task)
+        self.workflow.add_task(task)
+        self._current_task = task_id
+        return self
+
+    def while_loop(
+        self,
+        task_id: str,
+        condition: str,
+        loop_body: Callable[["WorkflowBuilder"], "WorkflowBuilder"],
+        **kwargs,
+    ) -> "WorkflowBuilder":
+        loop_builder = loop_body(WorkflowBuilder(f"{{task_id}}_loop", parent=self))
+        loop_tasks = list(loop_builder.workflow.tasks.values())
+
+        task = WhileOperator(
+            task_id=task_id,
+            condition=condition,
+            loop_body=loop_tasks,
+            **kwargs,
+        )
+
         if self._current_task:
             task.dependencies.append(self._current_task)
+
         self.workflow.add_task(task)
+
+        for task_obj in loop_tasks:
+            self.workflow.add_task(task_obj)
+
         self._current_task = task_id
         return self
 

highway_dsl-0.0.3.dist-info/METADATA

@@ -0,0 +1,160 @@
+Metadata-Version: 2.4
+Name: highway_dsl
+Version: 0.0.3
+Summary: A domain specific language (DSL) for defining and managing data processing pipelines.
+Author-email: Farseed Ashouri <farseed.ashouri@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/rodmena-limited/highway_dsl
+Project-URL: Issues, https://github.com/rodmena-limited/highway_dsl/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pydantic>=2.12.3
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: types-PyYAML>=6.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=2.12.1; extra == "dev"
+Dynamic: license-file
+
+# Highway DSL
+
+[![PyPI version](https://badge.fury.io/py/highway-dsl.svg)](https://badge.fury.io/py/highway-dsl)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+**Highway DSL** is a Python-based domain-specific language for defining complex workflows in a clear, concise, and fluent manner. It is part of the larger **Highway** project, an advanced workflow engine capable of running complex DAG-based workflows.
+
+## Features
+
+*   **Fluent API:** A powerful and intuitive `WorkflowBuilder` for defining workflows programmatically.
+*   **Pydantic-based:** All models are built on Pydantic, providing robust data validation, serialization, and documentation.
+*   **Rich Operators:** A comprehensive set of operators for handling various workflow scenarios:
+    *   `Task`
+    *   `Condition` (if/else)
+    *   `Parallel`
+    *   `ForEach`
+    *   `Wait`
+    *   `While`
+*   **YAML/JSON Interoperability:** Workflows can be defined in Python and exported to YAML or JSON, and vice-versa.
+*   **Extensible:** The DSL is designed to be extensible with custom operators and policies.
+
+## Installation
+
+```bash
+pip install highway-dsl
+```
+
+## Quick Start
+
+Here's a simple example of how to define a workflow using the `WorkflowBuilder`:
+
+```python
+from datetime import timedelta
+from highway_dsl import WorkflowBuilder
+
+workflow = (
+    WorkflowBuilder("simple_etl")
+    .task("extract", "etl.extract_data", result_key="raw_data")
+    .task(
+        "transform",
+        "etl.transform_data",
+        args=["{{raw_data}}"],
+        result_key="transformed_data",
+    )
+    .retry(max_retries=3, delay=timedelta(seconds=10))
+    .task("load", "etl.load_data", args=["{{transformed_data}}"])
+    .timeout(timeout=timedelta(minutes=30))
+    .wait("wait_next", timedelta(hours=24))
+    .task("cleanup", "etl.cleanup")
+    .build()
+)
+
+print(workflow.to_yaml())
+```
+
+## Advanced Usage
+
+### Conditional Logic
+
+```python
+from highway_dsl import WorkflowBuilder, RetryPolicy
+from datetime import timedelta
+
+builder = WorkflowBuilder("data_processing_pipeline")
+
+builder.task("start", "workflows.tasks.initialize", result_key="init_data")
+builder.task(
+    "validate",
+    "workflows.tasks.validate_data",
+    args=["{{init_data}}"],
+    result_key="validated_data",
+)
+
+builder.condition(
+    "check_quality",
+    condition="{{validated_data.quality_score}} > 0.8",
+    if_true=lambda b: b.task(
+        "high_quality_processing",
+        "workflows.tasks.advanced_processing",
+        args=["{{validated_data}}"],
+        retry_policy=RetryPolicy(max_retries=5, delay=timedelta(seconds=10), backoff_factor=2.0),
+    ),
+    if_false=lambda b: b.task(
+        "standard_processing",
+        "workflows.tasks.basic_processing",
+        args=["{{validated_data}}"],
+    ),
+)
+
+workflow = builder.build()
+```
+
+### While Loops
+
+```python
+from highway_dsl import WorkflowBuilder
+
+builder = WorkflowBuilder("qa_rework_workflow")
+
+builder.task("start_qa", "workflows.tasks.start_qa", result_key="qa_results")
+
+builder.while_loop(
+    "qa_rework_loop",
+    condition="{{qa_results.status}} == 'failed'",
+    loop_body=lambda b: b.task("perform_rework", "workflows.tasks.perform_rework").task(
+        "re_run_qa", "workflows.tasks.run_qa", result_key="qa_results"
+    ),
+)
+
+builder.task("finalize_product", "workflows.tasks.finalize_product", dependencies=["qa_rework_loop"])
+
+workflow = builder.build()
+```
+
+## Development
+
+To set up the development environment:
+
+```bash
+git clone https://github.com/your-username/highway.git
+cd highway
+python -m venv .venv
+source .venv/bin/activate
+pip install -e .[dev]
+```
+
+### Running Tests
+
+```bash
+pytest
+```
+
+### Type Checking
+
+```bash
+mypy .
+```

highway_dsl-0.0.3.dist-info/RECORD

@@ -0,0 +1,7 @@
+highway_dsl/__init__.py,sha256=mr1oMylxliFwu2VO2qpyM3sVQwYIoPL2P6JE-6ZuF7M,507
+highway_dsl/workflow_dsl.py,sha256=yMTmFr5bbjxfVTleCvSsDZ__n9C7qH39RdzajkUEmiI,11882
+highway_dsl-0.0.3.dist-info/licenses/LICENSE,sha256=qdFq1H66BvKg67mf4-WGpFwtG2u_dNknxuJDQ1_ubaY,1072
+highway_dsl-0.0.3.dist-info/METADATA,sha256=--TFErjeBDZ1mAyNHk30CQavLuKWAaoxgHK7xFpT-Ok,4612
+highway_dsl-0.0.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+highway_dsl-0.0.3.dist-info/top_level.txt,sha256=_5uX-bbBsQ2rsi1XMr7WRyKbr6ack5GqVBcy-QjF1C8,12
+highway_dsl-0.0.3.dist-info/RECORD,,

highway_dsl-0.0.1.dist-info/METADATA

@@ -1,203 +0,0 @@
-Metadata-Version: 2.4
-Name: highway_dsl
-Version: 0.0.1
-Summary: A domain specific language (DSL) for defining and managing data processing pipelines.
-Author-email: Farseed Ashouri <farseed.ashouri@gmail.com>
-License: MIT
-Project-URL: Homepage, https://github.com/rodmena-limited/highway_dsl
-Project-URL: Issues, https://github.com/rodmena-limited/highway_dsl/issues
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: pydantic>=2.12.3
-Requires-Dist: pyyaml>=6.0
-Provides-Extra: dev
-Requires-Dist: pytest>=7.0.0; extra == "dev"
-Requires-Dist: mypy>=1.0.0; extra == "dev"
-Requires-Dist: types-PyYAML>=6.0.0; extra == "dev"
-Requires-Dist: pytest-cov>=2.12.1; extra == "dev"
-Dynamic: license-file
-
-# Highway DSL
-
-Highway DSL is a Python-based Domain Specific Language (DSL) for defining and managing complex workflows. It allows users to declaratively specify tasks, dependencies, and execution parameters, supporting various control flow mechanisms like conditions, parallel execution, and retries.
-
-## Features
-
-*   **Declarative Workflow Definition:** Define workflows using a clear and concise Python API or through YAML/JSON configurations.
-*   **Pydantic Models:** Leverages Pydantic for robust data validation and serialization/deserialization of workflow definitions.
-*   **Rich Task Types:** Supports various operators including:
-    *   `TaskOperator`: Executes a Python function.
-    *   `ConditionOperator`: Enables conditional branching based on expressions.
-    *   `WaitOperator`: Pauses workflow execution for a specified duration or until a specific datetime.
-    *   `ParallelOperator`: Executes multiple branches of tasks concurrently.
-    *   `ForEachOperator`: Iterates over a collection, executing a chain of tasks for each item.
-*   **Retry and Timeout Policies:** Define retry strategies and timeout limits for individual tasks.
-*   **Serialization/Deserialization:** Seamless conversion of workflow definitions between Python objects, YAML, and JSON formats.
-*   **Workflow Builder:** A fluent API for constructing workflows programmatically.
-
-## Installation
-
-To install Highway DSL, you can use pip:
-
-```bash
-pip install highway-dsl
-```
-
-If you want to install it for development, including testing dependencies:
-
-```bash
-pip install "highway-dsl[dev]"
-```
-
-## Usage
-
-### Defining a Simple Workflow
-
-```python
-from datetime import timedelta
-from workflow_dsl import WorkflowBuilder
-
-def demonstrate_basic_workflow():
-    """Show a simple complete workflow using just the builder"""
-
-    workflow = (
-        WorkflowBuilder("simple_etl")
-        .task("extract", "etl.extract_data", result_key="raw_data")
-        .task(
-            "transform",
-            "etl.transform_data",
-            args=["{{raw_data}}"],
-            result_key="transformed_data",
-        )
-        .retry(max_retries=3, delay=timedelta(seconds=10))
-        .task("load", "etl.load_data", args=["{{transformed_data}}"])
-        .timeout(timeout=timedelta(minutes=30))
-        .wait("wait_next", timedelta(hours=24))
-        .task("cleanup", "etl.cleanup")
-        .build()
-    )
-
-    workflow.set_variables(
-        {"database_url": "postgresql://localhost/mydb", "chunk_size": 1000}
-    )
-
-    return workflow
-
-if __name__ == "__main__":
-    basic_workflow = demonstrate_basic_workflow()
-    print(basic_workflow.to_yaml())
-```
-
-### Defining a Complex Workflow
-
-Refer to `example_usage.py` for a more complex example demonstrating conditional logic, parallel execution, and iteration.
-
-### YAML Configuration
-
-You can also define workflows directly in YAML:
-
-```yaml
-name: simple_etl
-version: 1.0.0
-description: Simple ETL workflow with retry and timeout
-variables:
-  database_url: postgresql://localhost/mydb
-  chunk_size: 1000
-start_task: extract
-tasks:
-  extract:
-    task_id: extract
-    operator_type: task
-    function: etl.extract_data
-    result_key: raw_data
-    dependencies: []
-    metadata: {}
-    
-  transform:
-    task_id: transform
-    operator_type: task
-    function: etl.transform_data
-    args: ["{{raw_data}}"]
-    result_key: transformed_data
-    dependencies: ["extract"]
-    retry_policy:
-      max_retries: 3
-      delay: PT10S
-      backoff_factor: 2.0
-    metadata: {}
-    
-  load:
-    task_id: load
-    operator_type: task
-    function: etl.load_data
-    args: ["{{transformed_data}}"]
-    dependencies: ["transform"]
-    timeout_policy:
-      timeout: PT30M
-      kill_on_timeout: true
-    metadata: {}
-    
-  wait_next:
-    task_id: wait_next
-    operator_type: wait
-    wait_for: "P1D"
-    dependencies: ["load"]
-    metadata: {}
-    
-  cleanup:
-    task_id: cleanup
-    operator_type: task
-    function: etl.cleanup
-    dependencies: ["wait_next"]
-    metadata: {}
-```
-
-To load this YAML:
-
-```python
-from workflow_dsl import Workflow
-
-yaml_content = """
-# ... (yaml content from above)
-"""
-
-workflow = Workflow.from_yaml(yaml_content)
-print(workflow.name)
-```
-
-## Development
-
-### Running Tests
-
-To run the unit tests, navigate to the project root and execute:
-
-```bash
-pytest
-```
-
-### Type Checking
-
-To perform static type checking with MyPy:
-
-```bash
-mypy .
-```
-
-## Project Structure
-
-```
-.highway/
-├── workflow_dsl.py         # Core DSL definitions (Pydantic models)
-├── example_usage.py        # Examples of how to use the DSL
-├── tests/
-│   ├── __init__.py
-│   ├── conftest.py         # Pytest configuration
-│   └── test_workflow_dsl.py # Unit and integration tests
-├── pyproject.toml          # Project metadata and dependencies
-├── README.md               # This file
-└── SUMMARY.md              # Summary of changes and future instructions
-```

highway_dsl-0.0.1.dist-info/RECORD

@@ -1,7 +0,0 @@
-highway_dsl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-highway_dsl/workflow_dsl.py,sha256=PDUCYFBt0SHSZxXv6HKZXlVOmjUcYWiX4i6Kwhsn4h8,9026
-highway_dsl-0.0.1.dist-info/licenses/LICENSE,sha256=qdFq1H66BvKg67mf4-WGpFwtG2u_dNknxuJDQ1_ubaY,1072
-highway_dsl-0.0.1.dist-info/METADATA,sha256=3DgCxoxYJki8WYd4gfDB9IWIq1iAK-liAmWQ7fWxFvU,5838
-highway_dsl-0.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-highway_dsl-0.0.1.dist-info/top_level.txt,sha256=_5uX-bbBsQ2rsi1XMr7WRyKbr6ack5GqVBcy-QjF1C8,12
-highway_dsl-0.0.1.dist-info/RECORD,,