highway-dsl 0.0.1__py3-none-any.whl

highway_dsl/workflow_dsl.py

@@ -0,0 +1,252 @@
+# workflow_dsl.py
+from typing import Any, Dict, List, Optional, Union, Callable, Type
+from enum import Enum
+from datetime import datetime, timedelta
+import yaml
+import json
+from abc import ABC, abstractmethod
+from pydantic import BaseModel, Field, model_validator, ConfigDict
+
+
+class OperatorType(Enum):
+    TASK = "task"
+    CONDITION = "condition"
+    WAIT = "wait"
+    PARALLEL = "parallel"
+    FOREACH = "foreach"
+    SWITCH = "switch"
+    TRY_CATCH = "try_catch"
+
+
+class RetryPolicy(BaseModel):
+    max_retries: int = Field(3, description="Maximum number of retries")
+    delay: timedelta = Field(timedelta(seconds=5), description="Delay between retries")
+    backoff_factor: float = Field(2.0, description="Factor by which to increase delay")
+
+
+class TimeoutPolicy(BaseModel):
+    timeout: timedelta = Field(..., description="Timeout duration")
+    kill_on_timeout: bool = Field(True, description="Whether to kill the task on timeout")
+
+
+class BaseOperator(BaseModel, ABC):
+    task_id: str
+    operator_type: OperatorType
+    dependencies: List[str] = Field(default_factory=list)
+    retry_policy: Optional[RetryPolicy] = None
+    timeout_policy: Optional[TimeoutPolicy] = None
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+    model_config = ConfigDict(use_enum_values=True, arbitrary_types_allowed=True)
+
+
+class TaskOperator(BaseOperator):
+    function: str
+    args: List[Any] = Field(default_factory=list)
+    kwargs: Dict[str, Any] = Field(default_factory=dict)
+    result_key: Optional[str] = None
+    operator_type: OperatorType = Field(OperatorType.TASK, frozen=True)
+
+
+class ConditionOperator(BaseOperator):
+    condition: str
+    if_true: str
+    if_false: str
+    operator_type: OperatorType = Field(OperatorType.CONDITION, frozen=True)
+
+
+class WaitOperator(BaseOperator):
+    wait_for: Union[timedelta, datetime, str]
+    operator_type: OperatorType = Field(OperatorType.WAIT, frozen=True)
+
+    @model_validator(mode='before')
+    @classmethod
+    def parse_wait_for(cls, data: Any) -> Any:
+        if isinstance(data, dict) and "wait_for" in data:
+            wait_for = data["wait_for"]
+            if isinstance(wait_for, str):
+                if wait_for.startswith("duration:"):
+                    data["wait_for"] = timedelta(seconds=float(wait_for.split(":")[1]))
+                elif wait_for.startswith("datetime:"):
+                    data["wait_for"] = datetime.fromisoformat(wait_for.split(":", 1)[1])
+        return data
+
+    def model_dump(self, **kwargs) -> Dict[str, Any]:
+        data = super().model_dump(**kwargs)
+        wait_for = data["wait_for"]
+        if isinstance(wait_for, timedelta):
+            data["wait_for"] = f"duration:{wait_for.total_seconds()}"
+        elif isinstance(wait_for, datetime):
+            data["wait_for"] = f"datetime:{wait_for.isoformat()}"
+        return data
+
+
+class ParallelOperator(BaseOperator):
+    branches: Dict[str, List[str]] = Field(default_factory=dict)
+    operator_type: OperatorType = Field(OperatorType.PARALLEL, frozen=True)
+
+
+class ForEachOperator(BaseOperator):
+    items: str
+    task_chain: List[str] = Field(default_factory=list)
+    operator_type: OperatorType = Field(OperatorType.FOREACH, frozen=True)
+
+
+class Workflow(BaseModel):
+    name: str
+    version: str = "1.0.0"
+    description: str = ""
+    tasks: Dict[str, Union[
+        TaskOperator,
+        ConditionOperator,
+        WaitOperator,
+        ParallelOperator,
+        ForEachOperator,
+    ]] = Field(default_factory=dict)
+    variables: Dict[str, Any] = Field(default_factory=dict)
+    start_task: Optional[str] = None
+
+    @model_validator(mode='before')
+    @classmethod
+    def validate_tasks(cls, data: Any) -> Any:
+        if isinstance(data, dict) and "tasks" in data:
+            validated_tasks = {}
+            operator_classes: Dict[str, Type[BaseOperator]] = {
+                OperatorType.TASK.value: TaskOperator,
+                OperatorType.CONDITION.value: ConditionOperator,
+                OperatorType.WAIT.value: WaitOperator,
+                OperatorType.PARALLEL.value: ParallelOperator,
+                OperatorType.FOREACH.value: ForEachOperator,
+            }
+            for task_id, task_data in data["tasks"].items():
+                operator_type = task_data.get("operator_type")
+                if operator_type and operator_type in operator_classes:
+                    operator_class = operator_classes[operator_type]
+                    validated_tasks[task_id] = operator_class.model_validate(task_data)
+                else:
+                    raise ValueError(f"Unknown operator type: {operator_type}")
+            data["tasks"] = validated_tasks
+        return data
+
+    def add_task(self, task: Union[
+        TaskOperator,
+        ConditionOperator,
+        WaitOperator,
+        ParallelOperator,
+        ForEachOperator,
+    ]) -> "Workflow":
+        self.tasks[task.task_id] = task
+        return self
+
+    def set_variables(self, variables: Dict[str, Any]) -> "Workflow":
+        self.variables.update(variables)
+        return self
+
+    def set_start_task(self, task_id: str) -> "Workflow":
+        self.start_task = task_id
+        return self
+
+    def to_yaml(self) -> str:
+        data = self.model_dump(mode='json', by_alias=True, exclude_none=True)
+        return yaml.dump(data, default_flow_style=False)
+
+    def to_json(self) -> str:
+        return self.model_dump_json(indent=2)
+
+    @classmethod
+    def from_yaml(cls, yaml_str: str) -> "Workflow":
+        data = yaml.safe_load(yaml_str)
+        return cls.model_validate(data)
+
+    @classmethod
+    def from_json(cls, json_str: str) -> "Workflow":
+        return cls.model_validate_json(json_str)
+
+
+class WorkflowBuilder:
+    def __init__(self, name: str, existing_workflow: Optional[Workflow] = None):
+        if existing_workflow:
+            self.workflow = existing_workflow
+        else:
+            self.workflow = Workflow(name=name)
+        self._current_task: Optional[str] = None
+
+    def task(self, task_id: str, function: str, **kwargs) -> "WorkflowBuilder":
+        task = TaskOperator(task_id=task_id, function=function, **kwargs)
+        if self._current_task:
+            task.dependencies.append(self._current_task)
+        self.workflow.add_task(task)
+        self._current_task = task_id
+        return self
+
+    def condition(
+        self, task_id: str, condition: str, if_true: str, if_false: str, **kwargs
+    ) -> "WorkflowBuilder":
+        task = ConditionOperator(
+            task_id=task_id, condition=condition, if_true=if_true, if_false=if_false, **kwargs
+        )
+        if self._current_task:
+            task.dependencies.append(self._current_task)
+        self.workflow.add_task(task)
+        self._current_task = task_id
+        return self
+
+    def wait(
+        self, task_id: str, wait_for: Union[timedelta, datetime, str], **kwargs
+    ) -> "WorkflowBuilder":
+        task = WaitOperator(task_id=task_id, wait_for=wait_for, **kwargs)
+        if self._current_task:
+            task.dependencies.append(self._current_task)
+        self.workflow.add_task(task)
+        self._current_task = task_id
+        return self
+
+    def parallel(
+        self, task_id: str, branches: Dict[str, List[str]], **kwargs
+    ) -> "WorkflowBuilder":
+        task = ParallelOperator(task_id=task_id, branches=branches, **kwargs)
+        if self._current_task:
+            task.dependencies.append(self._current_task)
+        self.workflow.add_task(task)
+        self._current_task = task_id
+        return self
+
+    def foreach(
+        self, task_id: str, items: str, task_chain: List[str], **kwargs
+    ) -> "WorkflowBuilder":
+        task = ForEachOperator(task_id=task_id, items=items, task_chain=task_chain, **kwargs)
+        if self._current_task:
+            task.dependencies.append(self._current_task)
+        self.workflow.add_task(task)
+        self._current_task = task_id
+        return self
+
+    def retry(
+        self,
+        max_retries: int = 3,
+        delay: timedelta = timedelta(seconds=5),
+        backoff_factor: float = 2.0,
+    ) -> "WorkflowBuilder":
+        if self._current_task and isinstance(
+            self.workflow.tasks[self._current_task], TaskOperator
+        ):
+            self.workflow.tasks[self._current_task].retry_policy = RetryPolicy(
+                max_retries=max_retries, delay=delay, backoff_factor=backoff_factor
+            )
+        return self
+
+    def timeout(
+        self, timeout: timedelta, kill_on_timeout: bool = True
+    ) -> "WorkflowBuilder":
+        if self._current_task and isinstance(
+            self.workflow.tasks[self._current_task], TaskOperator
+        ):
+            self.workflow.tasks[self._current_task].timeout_policy = TimeoutPolicy(
+                timeout=timeout, kill_on_timeout=kill_on_timeout
+            )
+        return self
+
+    def build(self) -> Workflow:
+        if not self.workflow.start_task and self.workflow.tasks:
+            self.workflow.start_task = next(iter(self.workflow.tasks.keys()))
+        return self.workflow

highway_dsl-0.0.1.dist-info/METADATA

@@ -0,0 +1,203 @@
+Metadata-Version: 2.4
+Name: highway_dsl
+Version: 0.0.1
+Summary: A domain specific language (DSL) for defining and managing data processing pipelines.
+Author-email: Farseed Ashouri <farseed.ashouri@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/rodmena-limited/highway_dsl
+Project-URL: Issues, https://github.com/rodmena-limited/highway_dsl/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pydantic>=2.12.3
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: types-PyYAML>=6.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=2.12.1; extra == "dev"
+Dynamic: license-file
+
+# Highway DSL
+
+Highway DSL is a Python-based Domain Specific Language (DSL) for defining and managing complex workflows. It allows users to declaratively specify tasks, dependencies, and execution parameters, supporting various control flow mechanisms like conditions, parallel execution, and retries.
+
+## Features
+
+*   **Declarative Workflow Definition:** Define workflows using a clear and concise Python API or through YAML/JSON configurations.
+*   **Pydantic Models:** Leverages Pydantic for robust data validation and serialization/deserialization of workflow definitions.
+*   **Rich Task Types:** Supports various operators including:
+    *   `TaskOperator`: Executes a Python function.
+    *   `ConditionOperator`: Enables conditional branching based on expressions.
+    *   `WaitOperator`: Pauses workflow execution for a specified duration or until a specific datetime.
+    *   `ParallelOperator`: Executes multiple branches of tasks concurrently.
+    *   `ForEachOperator`: Iterates over a collection, executing a chain of tasks for each item.
+*   **Retry and Timeout Policies:** Define retry strategies and timeout limits for individual tasks.
+*   **Serialization/Deserialization:** Seamless conversion of workflow definitions between Python objects, YAML, and JSON formats.
+*   **Workflow Builder:** A fluent API for constructing workflows programmatically.
+
+## Installation
+
+To install Highway DSL, you can use pip:
+
+```bash
+pip install highway-dsl
+```
+
+If you want to install it for development, including testing dependencies:
+
+```bash
+pip install "highway-dsl[dev]"
+```
+
+## Usage
+
+### Defining a Simple Workflow
+
+```python
+from datetime import timedelta
+from workflow_dsl import WorkflowBuilder
+
+def demonstrate_basic_workflow():
+    """Show a simple complete workflow using just the builder"""
+
+    workflow = (
+        WorkflowBuilder("simple_etl")
+        .task("extract", "etl.extract_data", result_key="raw_data")
+        .task(
+            "transform",
+            "etl.transform_data",
+            args=["{{raw_data}}"],
+            result_key="transformed_data",
+        )
+        .retry(max_retries=3, delay=timedelta(seconds=10))
+        .task("load", "etl.load_data", args=["{{transformed_data}}"])
+        .timeout(timeout=timedelta(minutes=30))
+        .wait("wait_next", timedelta(hours=24))
+        .task("cleanup", "etl.cleanup")
+        .build()
+    )
+
+    workflow.set_variables(
+        {"database_url": "postgresql://localhost/mydb", "chunk_size": 1000}
+    )
+
+    return workflow
+
+if __name__ == "__main__":
+    basic_workflow = demonstrate_basic_workflow()
+    print(basic_workflow.to_yaml())
+```
+
+### Defining a Complex Workflow
+
+Refer to `example_usage.py` for a more complex example demonstrating conditional logic, parallel execution, and iteration.
+
+### YAML Configuration
+
+You can also define workflows directly in YAML:
+
+```yaml
+name: simple_etl
+version: 1.0.0
+description: Simple ETL workflow with retry and timeout
+variables:
+  database_url: postgresql://localhost/mydb
+  chunk_size: 1000
+start_task: extract
+tasks:
+  extract:
+    task_id: extract
+    operator_type: task
+    function: etl.extract_data
+    result_key: raw_data
+    dependencies: []
+    metadata: {}
+    
+  transform:
+    task_id: transform
+    operator_type: task
+    function: etl.transform_data
+    args: ["{{raw_data}}"]
+    result_key: transformed_data
+    dependencies: ["extract"]
+    retry_policy:
+      max_retries: 3
+      delay: PT10S
+      backoff_factor: 2.0
+    metadata: {}
+    
+  load:
+    task_id: load
+    operator_type: task
+    function: etl.load_data
+    args: ["{{transformed_data}}"]
+    dependencies: ["transform"]
+    timeout_policy:
+      timeout: PT30M
+      kill_on_timeout: true
+    metadata: {}
+    
+  wait_next:
+    task_id: wait_next
+    operator_type: wait
+    wait_for: "P1D"
+    dependencies: ["load"]
+    metadata: {}
+    
+  cleanup:
+    task_id: cleanup
+    operator_type: task
+    function: etl.cleanup
+    dependencies: ["wait_next"]
+    metadata: {}
+```
+
+To load this YAML:
+
+```python
+from workflow_dsl import Workflow
+
+yaml_content = """
+# ... (yaml content from above)
+"""
+
+workflow = Workflow.from_yaml(yaml_content)
+print(workflow.name)
+```
+
+## Development
+
+### Running Tests
+
+To run the unit tests, navigate to the project root and execute:
+
+```bash
+pytest
+```
+
+### Type Checking
+
+To perform static type checking with MyPy:
+
+```bash
+mypy .
+```
+
+## Project Structure
+
+```
+.highway/
+├── workflow_dsl.py         # Core DSL definitions (Pydantic models)
+├── example_usage.py        # Examples of how to use the DSL
+├── tests/
+│   ├── __init__.py
+│   ├── conftest.py         # Pytest configuration
+│   └── test_workflow_dsl.py # Unit and integration tests
+├── pyproject.toml          # Project metadata and dependencies
+├── README.md               # This file
+└── SUMMARY.md              # Summary of changes and future instructions
+```

highway_dsl-0.0.1.dist-info/RECORD

@@ -0,0 +1,7 @@
+highway_dsl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+highway_dsl/workflow_dsl.py,sha256=PDUCYFBt0SHSZxXv6HKZXlVOmjUcYWiX4i6Kwhsn4h8,9026
+highway_dsl-0.0.1.dist-info/licenses/LICENSE,sha256=qdFq1H66BvKg67mf4-WGpFwtG2u_dNknxuJDQ1_ubaY,1072
+highway_dsl-0.0.1.dist-info/METADATA,sha256=3DgCxoxYJki8WYd4gfDB9IWIq1iAK-liAmWQ7fWxFvU,5838
+highway_dsl-0.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+highway_dsl-0.0.1.dist-info/top_level.txt,sha256=_5uX-bbBsQ2rsi1XMr7WRyKbr6ack5GqVBcy-QjF1C8,12
+highway_dsl-0.0.1.dist-info/RECORD,,

highway_dsl-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

highway_dsl-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Farseed Ashouri
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

highway_dsl-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+highway_dsl