highway-dsl 0.0.1__py3-none-any.whl → 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of highway-dsl might be problematic. Click here for more details.
- highway_dsl/__init__.py +25 -0
- highway_dsl/workflow_dsl.py +34 -20
- {highway_dsl-0.0.1.dist-info → highway_dsl-0.0.2.dist-info}/METADATA +28 -4
- highway_dsl-0.0.2.dist-info/RECORD +7 -0
- highway_dsl-0.0.1.dist-info/RECORD +0 -7
- {highway_dsl-0.0.1.dist-info → highway_dsl-0.0.2.dist-info}/WHEEL +0 -0
- {highway_dsl-0.0.1.dist-info → highway_dsl-0.0.2.dist-info}/licenses/LICENSE +0 -0
- {highway_dsl-0.0.1.dist-info → highway_dsl-0.0.2.dist-info}/top_level.txt +0 -0
highway_dsl/__init__.py
CHANGED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from .workflow_dsl import (
|
|
2
|
+
Workflow,
|
|
3
|
+
WorkflowBuilder,
|
|
4
|
+
TaskOperator,
|
|
5
|
+
ConditionOperator,
|
|
6
|
+
ParallelOperator,
|
|
7
|
+
WaitOperator,
|
|
8
|
+
ForEachOperator,
|
|
9
|
+
RetryPolicy,
|
|
10
|
+
TimeoutPolicy,
|
|
11
|
+
OperatorType,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"Workflow",
|
|
16
|
+
"WorkflowBuilder",
|
|
17
|
+
"TaskOperator",
|
|
18
|
+
"ConditionOperator",
|
|
19
|
+
"ParallelOperator",
|
|
20
|
+
"WaitOperator",
|
|
21
|
+
"ForEachOperator",
|
|
22
|
+
"RetryPolicy",
|
|
23
|
+
"TimeoutPolicy",
|
|
24
|
+
"OperatorType",
|
|
25
|
+
]
|
highway_dsl/workflow_dsl.py
CHANGED
|
@@ -26,7 +26,9 @@ class RetryPolicy(BaseModel):
|
|
|
26
26
|
|
|
27
27
|
class TimeoutPolicy(BaseModel):
|
|
28
28
|
timeout: timedelta = Field(..., description="Timeout duration")
|
|
29
|
-
kill_on_timeout: bool = Field(
|
|
29
|
+
kill_on_timeout: bool = Field(
|
|
30
|
+
True, description="Whether to kill the task on timeout"
|
|
31
|
+
)
|
|
30
32
|
|
|
31
33
|
|
|
32
34
|
class BaseOperator(BaseModel, ABC):
|
|
@@ -59,7 +61,7 @@ class WaitOperator(BaseOperator):
|
|
|
59
61
|
wait_for: Union[timedelta, datetime, str]
|
|
60
62
|
operator_type: OperatorType = Field(OperatorType.WAIT, frozen=True)
|
|
61
63
|
|
|
62
|
-
@model_validator(mode=
|
|
64
|
+
@model_validator(mode="before")
|
|
63
65
|
@classmethod
|
|
64
66
|
def parse_wait_for(cls, data: Any) -> Any:
|
|
65
67
|
if isinstance(data, dict) and "wait_for" in data:
|
|
@@ -96,17 +98,20 @@ class Workflow(BaseModel):
|
|
|
96
98
|
name: str
|
|
97
99
|
version: str = "1.0.0"
|
|
98
100
|
description: str = ""
|
|
99
|
-
tasks: Dict[
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
101
|
+
tasks: Dict[
|
|
102
|
+
str,
|
|
103
|
+
Union[
|
|
104
|
+
TaskOperator,
|
|
105
|
+
ConditionOperator,
|
|
106
|
+
WaitOperator,
|
|
107
|
+
ParallelOperator,
|
|
108
|
+
ForEachOperator,
|
|
109
|
+
],
|
|
110
|
+
] = Field(default_factory=dict)
|
|
106
111
|
variables: Dict[str, Any] = Field(default_factory=dict)
|
|
107
112
|
start_task: Optional[str] = None
|
|
108
113
|
|
|
109
|
-
@model_validator(mode=
|
|
114
|
+
@model_validator(mode="before")
|
|
110
115
|
@classmethod
|
|
111
116
|
def validate_tasks(cls, data: Any) -> Any:
|
|
112
117
|
if isinstance(data, dict) and "tasks" in data:
|
|
@@ -128,13 +133,16 @@ class Workflow(BaseModel):
|
|
|
128
133
|
data["tasks"] = validated_tasks
|
|
129
134
|
return data
|
|
130
135
|
|
|
131
|
-
def add_task(
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
136
|
+
def add_task(
|
|
137
|
+
self,
|
|
138
|
+
task: Union[
|
|
139
|
+
TaskOperator,
|
|
140
|
+
ConditionOperator,
|
|
141
|
+
WaitOperator,
|
|
142
|
+
ParallelOperator,
|
|
143
|
+
ForEachOperator,
|
|
144
|
+
],
|
|
145
|
+
) -> "Workflow":
|
|
138
146
|
self.tasks[task.task_id] = task
|
|
139
147
|
return self
|
|
140
148
|
|
|
@@ -147,7 +155,7 @@ class Workflow(BaseModel):
|
|
|
147
155
|
return self
|
|
148
156
|
|
|
149
157
|
def to_yaml(self) -> str:
|
|
150
|
-
data = self.model_dump(mode=
|
|
158
|
+
data = self.model_dump(mode="json", by_alias=True, exclude_none=True)
|
|
151
159
|
return yaml.dump(data, default_flow_style=False)
|
|
152
160
|
|
|
153
161
|
def to_json(self) -> str:
|
|
@@ -183,7 +191,11 @@ class WorkflowBuilder:
|
|
|
183
191
|
self, task_id: str, condition: str, if_true: str, if_false: str, **kwargs
|
|
184
192
|
) -> "WorkflowBuilder":
|
|
185
193
|
task = ConditionOperator(
|
|
186
|
-
task_id=task_id,
|
|
194
|
+
task_id=task_id,
|
|
195
|
+
condition=condition,
|
|
196
|
+
if_true=if_true,
|
|
197
|
+
if_false=if_false,
|
|
198
|
+
**kwargs,
|
|
187
199
|
)
|
|
188
200
|
if self._current_task:
|
|
189
201
|
task.dependencies.append(self._current_task)
|
|
@@ -214,7 +226,9 @@ class WorkflowBuilder:
|
|
|
214
226
|
def foreach(
|
|
215
227
|
self, task_id: str, items: str, task_chain: List[str], **kwargs
|
|
216
228
|
) -> "WorkflowBuilder":
|
|
217
|
-
task = ForEachOperator(
|
|
229
|
+
task = ForEachOperator(
|
|
230
|
+
task_id=task_id, items=items, task_chain=task_chain, **kwargs
|
|
231
|
+
)
|
|
218
232
|
if self._current_task:
|
|
219
233
|
task.dependencies.append(self._current_task)
|
|
220
234
|
self.workflow.add_task(task)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: highway_dsl
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.2
|
|
4
4
|
Summary: A domain specific language (DSL) for defining and managing data processing pipelines.
|
|
5
5
|
Author-email: Farseed Ashouri <farseed.ashouri@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -39,6 +39,28 @@ Highway DSL is a Python-based Domain Specific Language (DSL) for defining and ma
|
|
|
39
39
|
* **Serialization/Deserialization:** Seamless conversion of workflow definitions between Python objects, YAML, and JSON formats.
|
|
40
40
|
* **Workflow Builder:** A fluent API for constructing workflows programmatically.
|
|
41
41
|
|
|
42
|
+
### Feature Overview
|
|
43
|
+
|
|
44
|
+
```mermaid
|
|
45
|
+
graph TD
|
|
46
|
+
A[Workflow] --> B{TaskOperator};
|
|
47
|
+
A --> C{ConditionOperator};
|
|
48
|
+
A --> D{WaitOperator};
|
|
49
|
+
A --> E{ParallelOperator};
|
|
50
|
+
A --> F{ForEachOperator};
|
|
51
|
+
|
|
52
|
+
B --> G[Executes Python Function];
|
|
53
|
+
C --> H{If/Else Branching};
|
|
54
|
+
D --> I[Pauses Execution];
|
|
55
|
+
E --> J[Concurrent Branches];
|
|
56
|
+
F --> K[Iterates Over Items];
|
|
57
|
+
|
|
58
|
+
subgraph Policies
|
|
59
|
+
B --> L[RetryPolicy];
|
|
60
|
+
B --> M[TimeoutPolicy];
|
|
61
|
+
end
|
|
62
|
+
```
|
|
63
|
+
|
|
42
64
|
## Installation
|
|
43
65
|
|
|
44
66
|
To install Highway DSL, you can use pip:
|
|
@@ -59,7 +81,7 @@ pip install "highway-dsl[dev]"
|
|
|
59
81
|
|
|
60
82
|
```python
|
|
61
83
|
from datetime import timedelta
|
|
62
|
-
from
|
|
84
|
+
from highway_dsl import WorkflowBuilder
|
|
63
85
|
|
|
64
86
|
def demonstrate_basic_workflow():
|
|
65
87
|
"""Show a simple complete workflow using just the builder"""
|
|
@@ -159,7 +181,7 @@ tasks:
|
|
|
159
181
|
To load this YAML:
|
|
160
182
|
|
|
161
183
|
```python
|
|
162
|
-
from
|
|
184
|
+
from highway_dsl import Workflow
|
|
163
185
|
|
|
164
186
|
yaml_content = """
|
|
165
187
|
# ... (yaml content from above)
|
|
@@ -191,7 +213,9 @@ mypy .
|
|
|
191
213
|
|
|
192
214
|
```
|
|
193
215
|
.highway/
|
|
194
|
-
├──
|
|
216
|
+
├── highway_dsl/
|
|
217
|
+
│ ├── __init__.py # Exposes the public API
|
|
218
|
+
│ └── workflow_dsl.py # Core DSL definitions (Pydantic models)
|
|
195
219
|
├── example_usage.py # Examples of how to use the DSL
|
|
196
220
|
├── tests/
|
|
197
221
|
│ ├── __init__.py
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
highway_dsl/__init__.py,sha256=8qmPd9ZZNgwPGZuWwPYvMOljg73BJIT2SSM7iIRycmw,447
|
|
2
|
+
highway_dsl/workflow_dsl.py,sha256=2QWDhbXLPulq_kTZk_Yjs6L3BNwws_H6EDV0S1CjOXs,9205
|
|
3
|
+
highway_dsl-0.0.2.dist-info/licenses/LICENSE,sha256=qdFq1H66BvKg67mf4-WGpFwtG2u_dNknxuJDQ1_ubaY,1072
|
|
4
|
+
highway_dsl-0.0.2.dist-info/METADATA,sha256=uLLXSVlLWM8H6F5wR1huiAtgXfkIVdmLV-XsYwZkW6s,6390
|
|
5
|
+
highway_dsl-0.0.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
6
|
+
highway_dsl-0.0.2.dist-info/top_level.txt,sha256=_5uX-bbBsQ2rsi1XMr7WRyKbr6ack5GqVBcy-QjF1C8,12
|
|
7
|
+
highway_dsl-0.0.2.dist-info/RECORD,,
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
highway_dsl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
highway_dsl/workflow_dsl.py,sha256=PDUCYFBt0SHSZxXv6HKZXlVOmjUcYWiX4i6Kwhsn4h8,9026
|
|
3
|
-
highway_dsl-0.0.1.dist-info/licenses/LICENSE,sha256=qdFq1H66BvKg67mf4-WGpFwtG2u_dNknxuJDQ1_ubaY,1072
|
|
4
|
-
highway_dsl-0.0.1.dist-info/METADATA,sha256=3DgCxoxYJki8WYd4gfDB9IWIq1iAK-liAmWQ7fWxFvU,5838
|
|
5
|
-
highway_dsl-0.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
6
|
-
highway_dsl-0.0.1.dist-info/top_level.txt,sha256=_5uX-bbBsQ2rsi1XMr7WRyKbr6ack5GqVBcy-QjF1C8,12
|
|
7
|
-
highway_dsl-0.0.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|