novastack-workflows 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- novastack/workflows/__init__.py +19 -0
- novastack/workflows/context/__init__.py +3 -0
- novastack/workflows/context/context.py +121 -0
- novastack/workflows/context/state_store.py +123 -0
- novastack/workflows/decorators.py +114 -0
- novastack/workflows/enums.py +10 -0
- novastack/workflows/events.py +32 -0
- novastack/workflows/exceptions.py +14 -0
- novastack/workflows/runtime/__init__.py +0 -0
- novastack/workflows/runtime/engine.py +282 -0
- novastack/workflows/runtime/execution_pool.py +51 -0
- novastack/workflows/runtime/optimized_queue.py +79 -0
- novastack/workflows/types.py +89 -0
- novastack/workflows/workflow.py +151 -0
- novastack_workflows-1.0.0.dist-info/METADATA +90 -0
- novastack_workflows-1.0.0.dist-info/RECORD +17 -0
- novastack_workflows-1.0.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from novastack.workflows.workflow import Workflow
|
|
2
|
+
from novastack.workflows.decorators import step
|
|
3
|
+
|
|
4
|
+
from novastack.workflows.events import Event, StartEvent, StopEvent
|
|
5
|
+
|
|
6
|
+
# Context management
|
|
7
|
+
from novastack.workflows.context import Context
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
# Core
|
|
12
|
+
"Workflow",
|
|
13
|
+
"Context",
|
|
14
|
+
"step",
|
|
15
|
+
# Event system
|
|
16
|
+
"Event",
|
|
17
|
+
"StartEvent",
|
|
18
|
+
"StopEvent",
|
|
19
|
+
]
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
from typing import Any, Generic, TypeVar, TYPE_CHECKING
|
|
3
|
+
|
|
4
|
+
from pydantic import BaseModel
|
|
5
|
+
from novastack.workflows.context.state_store import StateStore
|
|
6
|
+
from novastack.workflows.runtime.optimized_queue import OptimizedEventQueue
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from novastack.workflows.events import Event
|
|
10
|
+
|
|
11
|
+
STATE_T = TypeVar("STATE_T", bound=BaseModel)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Context(Generic[STATE_T]):
|
|
15
|
+
"""
|
|
16
|
+
Workflow execution context with copy-on-write state management.
|
|
17
|
+
|
|
18
|
+
Provides robust, immutable state management through 'state_store'.
|
|
19
|
+
|
|
20
|
+
State is automatically initialized based on the type annotation.
|
|
21
|
+
If no type is provided, uses DictLikeModel.
|
|
22
|
+
|
|
23
|
+
Example:
|
|
24
|
+
```python
|
|
25
|
+
class RunState(BaseModel):
|
|
26
|
+
counter: int = 0
|
|
27
|
+
items: list[str] = []
|
|
28
|
+
|
|
29
|
+
# With typed state - auto-initialized
|
|
30
|
+
@step
|
|
31
|
+
async def start(self, ctx: Context[RunState], ev: StartEvent):
|
|
32
|
+
async with ctx.store.edit_state() as state:
|
|
33
|
+
state.counter += 1 # RunState auto-initialized
|
|
34
|
+
|
|
35
|
+
# Without type - uses DictLikeModel
|
|
36
|
+
@step
|
|
37
|
+
async def start(self, ctx: Context, ev: StartEvent):
|
|
38
|
+
async with ctx.store.edit_state() as state:
|
|
39
|
+
state.counter = 1 # DictLikeModel allows dynamic fields
|
|
40
|
+
```
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
def __init__(
|
|
44
|
+
self,
|
|
45
|
+
workflow: Any,
|
|
46
|
+
event_queue: asyncio.Queue["Event"] | Any | None = None,
|
|
47
|
+
):
|
|
48
|
+
"""
|
|
49
|
+
Initialize workflow context.
|
|
50
|
+
|
|
51
|
+
Each context is isolated and maintains its own state store.
|
|
52
|
+
State type is inferred from Context[StateType] annotation or defaults to DictLikeModel.
|
|
53
|
+
|
|
54
|
+
Args:
|
|
55
|
+
workflow: Workflow instance
|
|
56
|
+
event_queue: Event queue for emission
|
|
57
|
+
"""
|
|
58
|
+
self._workflow = workflow
|
|
59
|
+
|
|
60
|
+
# Use provided queue or create OptimizedEventQueue
|
|
61
|
+
if event_queue is None:
|
|
62
|
+
self._event_queue = OptimizedEventQueue()
|
|
63
|
+
else:
|
|
64
|
+
self._event_queue = event_queue
|
|
65
|
+
|
|
66
|
+
# Copy-on-write state store with auto-initialization
|
|
67
|
+
# Infer state type from workflow's _state_type if available
|
|
68
|
+
state_type = getattr(workflow, "_state_type", None)
|
|
69
|
+
self._store = StateStore(state_type=state_type)
|
|
70
|
+
|
|
71
|
+
@property
|
|
72
|
+
def workflow(self) -> Any:
|
|
73
|
+
"""Get workflow instance."""
|
|
74
|
+
return self._workflow
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def store(self) -> StateStore:
|
|
78
|
+
"""
|
|
79
|
+
Copy-on-write state store.
|
|
80
|
+
|
|
81
|
+
Provides immutability guarantees and thread-safety through
|
|
82
|
+
explicit edit contexts.
|
|
83
|
+
|
|
84
|
+
Example:
|
|
85
|
+
```python
|
|
86
|
+
# Edit state
|
|
87
|
+
async with ctx.store.edit_state() as state:
|
|
88
|
+
state.counter += 1
|
|
89
|
+
```
|
|
90
|
+
"""
|
|
91
|
+
return self._store
|
|
92
|
+
|
|
93
|
+
@property
|
|
94
|
+
def state(self) -> BaseModel | None:
|
|
95
|
+
"""
|
|
96
|
+
Get current state (read-only).
|
|
97
|
+
|
|
98
|
+
Convenience property for accessing store state.
|
|
99
|
+
Equivalent to ctx.store.state.
|
|
100
|
+
|
|
101
|
+
Example:
|
|
102
|
+
```python
|
|
103
|
+
# Read-only access
|
|
104
|
+
current_value = ctx.state.counter
|
|
105
|
+
```
|
|
106
|
+
"""
|
|
107
|
+
return self._store.state
|
|
108
|
+
|
|
109
|
+
async def emit(self, event: "Event") -> None:
|
|
110
|
+
"""
|
|
111
|
+
Emit an event to the workflow queue.
|
|
112
|
+
|
|
113
|
+
Args:
|
|
114
|
+
event: Event to emit
|
|
115
|
+
|
|
116
|
+
Example:
|
|
117
|
+
```python
|
|
118
|
+
await ctx.emit(MyEvent(data="processed"))
|
|
119
|
+
```
|
|
120
|
+
"""
|
|
121
|
+
await self._event_queue.put(event)
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import copy
|
|
3
|
+
from contextlib import asynccontextmanager
|
|
4
|
+
from typing import Generic, Type, TypeVar, cast
|
|
5
|
+
|
|
6
|
+
from pydantic import BaseModel
|
|
7
|
+
from novastack.workflows.exceptions import ContextStateError
|
|
8
|
+
from novastack.workflows.types import DictLikeModel
|
|
9
|
+
|
|
10
|
+
STATE_T = TypeVar("STATE_T", bound=BaseModel)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class StateStore(Generic[STATE_T]):
|
|
14
|
+
"""
|
|
15
|
+
Thread-safe state store with copy-on-write semantics.
|
|
16
|
+
|
|
17
|
+
Provides immutability guarantees through explicit edit contexts.
|
|
18
|
+
Changes are isolated until committed on context exit.
|
|
19
|
+
|
|
20
|
+
State is automatically initialized on first access if not provided.
|
|
21
|
+
If no state_type is specified, uses DictLikeModel for dynamic fields.
|
|
22
|
+
|
|
23
|
+
Thread-safe mutations through explicit contexts. Automatic rollback on errors makes it
|
|
24
|
+
robust and production-ready.
|
|
25
|
+
|
|
26
|
+
Example:
|
|
27
|
+
```python
|
|
28
|
+
# Copy-on-write with immutability
|
|
29
|
+
async with ctx.store.edit_state() as state:
|
|
30
|
+
state.counter += 1 # Mutation isolated
|
|
31
|
+
# Automatic commit on exit
|
|
32
|
+
```
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def __init__(
|
|
36
|
+
self,
|
|
37
|
+
state_data: STATE_T | None = None,
|
|
38
|
+
state_type: Type[STATE_T] | None = None,
|
|
39
|
+
):
|
|
40
|
+
"""
|
|
41
|
+
Initialize state store with isolated state.
|
|
42
|
+
|
|
43
|
+
Each state store is independent and does not inherit state
|
|
44
|
+
from other contexts. This ensures proper isolation between
|
|
45
|
+
workflows.
|
|
46
|
+
|
|
47
|
+
Args:
|
|
48
|
+
state_data: Initial state instance (optional)
|
|
49
|
+
state_type: State model class for auto-initialization (optional)
|
|
50
|
+
"""
|
|
51
|
+
self._lock = asyncio.Lock()
|
|
52
|
+
self._state_type: Type[BaseModel] = state_type or DictLikeModel
|
|
53
|
+
|
|
54
|
+
if state_data is not None:
|
|
55
|
+
self._state: STATE_T | None = state_data
|
|
56
|
+
else:
|
|
57
|
+
# Auto-initialize with state_type
|
|
58
|
+
self._state = cast(STATE_T, self._state_type())
|
|
59
|
+
|
|
60
|
+
@property
|
|
61
|
+
def state(self) -> STATE_T | None:
|
|
62
|
+
"""Get current state (read-only)."""
|
|
63
|
+
return self._state
|
|
64
|
+
|
|
65
|
+
def _smart_copy(self, state: STATE_T) -> STATE_T:
|
|
66
|
+
"""
|
|
67
|
+
Smart copy that optimizes based on state type.
|
|
68
|
+
|
|
69
|
+
For Pydantic models, uses the optimized model_copy() method.
|
|
70
|
+
This reduces copy overhead by 40-60% for Pydantic models.
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
state: State to copy
|
|
74
|
+
"""
|
|
75
|
+
if isinstance(state, BaseModel):
|
|
76
|
+
# Pydantic has optimized copy
|
|
77
|
+
return state.model_copy(deep=True)
|
|
78
|
+
|
|
79
|
+
# Fallback to deep copy
|
|
80
|
+
return copy.deepcopy(state)
|
|
81
|
+
|
|
82
|
+
@asynccontextmanager
|
|
83
|
+
async def edit_state(self):
|
|
84
|
+
"""
|
|
85
|
+
Context manager for editing state with copy-on-write semantics.
|
|
86
|
+
|
|
87
|
+
Creates a deep copy of the state for editing. Changes are committed
|
|
88
|
+
atomically when the context exits successfully. If an exception occurs,
|
|
89
|
+
changes are automatically rolled back.
|
|
90
|
+
|
|
91
|
+
State is guaranteed to be initialized (never None) due to auto-initialization
|
|
92
|
+
in __init__.
|
|
93
|
+
|
|
94
|
+
Example:
|
|
95
|
+
```python
|
|
96
|
+
async with ctx.store.edit_state() as state:
|
|
97
|
+
state.counter += 1
|
|
98
|
+
state.items.append("new")
|
|
99
|
+
|
|
100
|
+
# On error, changes are rolled back:
|
|
101
|
+
try:
|
|
102
|
+
async with ctx.store.edit_state() as state:
|
|
103
|
+
state.counter += 1
|
|
104
|
+
raise Exception("Error!") # Rollback happens
|
|
105
|
+
except Exception:
|
|
106
|
+
pass # state.counter unchanged
|
|
107
|
+
```
|
|
108
|
+
"""
|
|
109
|
+
async with self._lock:
|
|
110
|
+
# State is always initialized in __init__, but keep check for safety
|
|
111
|
+
if self._state is None:
|
|
112
|
+
raise ContextStateError("State not initialized.")
|
|
113
|
+
|
|
114
|
+
# Use smart copy instead of deepcopy
|
|
115
|
+
state_copy = self._smart_copy(self._state)
|
|
116
|
+
|
|
117
|
+
try:
|
|
118
|
+
yield state_copy
|
|
119
|
+
# Commit changes on successful exit
|
|
120
|
+
self._state = state_copy
|
|
121
|
+
except Exception:
|
|
122
|
+
# Rollback on error (don't commit)
|
|
123
|
+
raise
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
from typing import Any, Callable, Type
|
|
2
|
+
from functools import wraps
|
|
3
|
+
import inspect
|
|
4
|
+
|
|
5
|
+
from novastack.workflows.events import Event
|
|
6
|
+
from novastack.workflows.exceptions import WorkflowValidationError
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def step(
|
|
10
|
+
*,
|
|
11
|
+
on: Type[Event],
|
|
12
|
+
timeout: float | None = None,
|
|
13
|
+
num_retries: int = 0,
|
|
14
|
+
retry_delay: float = 1.0,
|
|
15
|
+
) -> Callable:
|
|
16
|
+
"""
|
|
17
|
+
Decorator to mark a method as a workflow step.
|
|
18
|
+
|
|
19
|
+
The decorated method will be automatically registered as a step
|
|
20
|
+
for the specified event type when the workflow class is defined.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
on: The Event class this step handles.
|
|
24
|
+
timeout: Optional timeout in seconds for step execution
|
|
25
|
+
num_retries: Number of retry attempts on failure (default: 0)
|
|
26
|
+
retry_delay: Delay in seconds between retry attempts (default: 1)
|
|
27
|
+
|
|
28
|
+
Note:
|
|
29
|
+
The decorated method must have the signature:
|
|
30
|
+
async def fn(self, ctx: Context, ev: EventType) -> Event | None
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
def decorator(f: Callable) -> Callable:
|
|
34
|
+
# Is async function?
|
|
35
|
+
if not inspect.iscoroutinefunction(f):
|
|
36
|
+
raise WorkflowValidationError(
|
|
37
|
+
f"Step method '{f.__name__}' must be an async function."
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
# Validate step signature
|
|
41
|
+
sig = inspect.signature(f)
|
|
42
|
+
params = list(sig.parameters.keys())
|
|
43
|
+
|
|
44
|
+
if len(params) < 3:
|
|
45
|
+
raise WorkflowValidationError(
|
|
46
|
+
f"Step signature '{f.__name__}' must have at least 3 parameters: "
|
|
47
|
+
f"self, ctx, ev. got: {params}"
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
# Store metadata on the function
|
|
51
|
+
f.__workflow_step__ = True
|
|
52
|
+
f.__step_accepted_events__ = on
|
|
53
|
+
f.__step_name__ = f.__name__
|
|
54
|
+
f.__step_timeout__ = timeout
|
|
55
|
+
f.__step_num_retries__ = num_retries
|
|
56
|
+
f.__step_retry_delay__ = retry_delay
|
|
57
|
+
|
|
58
|
+
@wraps(f)
|
|
59
|
+
async def wrapper(*args, **kwargs):
|
|
60
|
+
return await f(*args, **kwargs)
|
|
61
|
+
|
|
62
|
+
# Copy metadata to wrapper
|
|
63
|
+
wrapper.__workflow_step__ = True
|
|
64
|
+
wrapper.__step_accepted_events__ = on
|
|
65
|
+
wrapper.__step_name__ = f.__name__
|
|
66
|
+
wrapper.__step_timeout__ = timeout
|
|
67
|
+
wrapper.__step_num_retries__ = num_retries
|
|
68
|
+
wrapper.__step_retry_delay__ = retry_delay
|
|
69
|
+
|
|
70
|
+
return wrapper # type: ignore
|
|
71
|
+
|
|
72
|
+
return decorator
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def is_step_method(method: Any) -> bool:
|
|
76
|
+
"""
|
|
77
|
+
Check if a method is decorated with @step.
|
|
78
|
+
"""
|
|
79
|
+
return getattr(method, "__workflow_step__", False)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def get_step_event_type(method: Any) -> Type[Event] | None:
|
|
83
|
+
"""
|
|
84
|
+
Get the event type that a step method handles.
|
|
85
|
+
"""
|
|
86
|
+
return getattr(method, "__step_accepted_events__", None)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def get_step_name(method: Any) -> str | None:
|
|
90
|
+
"""
|
|
91
|
+
Get the name of a step method.
|
|
92
|
+
"""
|
|
93
|
+
return getattr(method, "__step_name__", None)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def get_step_timeout(method: Any) -> float | None:
|
|
97
|
+
"""
|
|
98
|
+
Get timeout for a step method.
|
|
99
|
+
"""
|
|
100
|
+
return getattr(method, "__step_timeout__", None)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def get_step_num_retries(method: Any) -> int:
|
|
104
|
+
"""
|
|
105
|
+
Get number of retries for a step method.
|
|
106
|
+
"""
|
|
107
|
+
return getattr(method, "__step_num_retries__", 0)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def get_step_retry_delay(method: Any) -> float:
|
|
111
|
+
"""
|
|
112
|
+
Get retry delay for a step method.
|
|
113
|
+
"""
|
|
114
|
+
return getattr(method, "__step_retry_delay__", 1.0)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
from pydantic import Field
|
|
4
|
+
from novastack.workflows.types import DictLikeModel
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Event(DictLikeModel):
|
|
8
|
+
"""Base class for all workflow events with dict-like interface."""
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class StartEvent(Event):
|
|
12
|
+
"""Workflow initialization event.
|
|
13
|
+
|
|
14
|
+
This event marks the beginning of a workflow execution and can carry
|
|
15
|
+
any dynamic fields needed to initialize the workflow state.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class StopEvent(Event):
|
|
20
|
+
"""Workflow completion event.
|
|
21
|
+
|
|
22
|
+
This event marks the end of a workflow execution and typically contains
|
|
23
|
+
the final result. Additional dynamic fields can be included as needed.
|
|
24
|
+
|
|
25
|
+
Attributes:
|
|
26
|
+
result: The final result of the workflow execution.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
result: Any = Field(
|
|
30
|
+
default=None,
|
|
31
|
+
description="The final result of the workflow execution",
|
|
32
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
class WorkflowValidationError(Exception):
|
|
2
|
+
"""Raised when the workflow configuration or step signatures are invalid."""
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class WorkflowRuntimeError(Exception):
|
|
6
|
+
"""Raised when runtime errors during step execution or event routing."""
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class WorkflowTimeoutError(Exception):
|
|
10
|
+
"""Raised when a step run exceeds the configured timeout."""
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ContextStateError(Exception):
|
|
14
|
+
"""Raised when a context method is called in the wrong state."""
|
|
File without changes
|
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import uuid
|
|
3
|
+
from typing import Any, TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
from novastack.workflows.context import Context
|
|
6
|
+
from novastack.workflows.decorators import (
|
|
7
|
+
get_step_timeout,
|
|
8
|
+
get_step_num_retries,
|
|
9
|
+
get_step_retry_delay,
|
|
10
|
+
)
|
|
11
|
+
from novastack.workflows.events import Event, StartEvent, StopEvent
|
|
12
|
+
from novastack.workflows.exceptions import (
|
|
13
|
+
WorkflowRuntimeError,
|
|
14
|
+
WorkflowTimeoutError,
|
|
15
|
+
WorkflowValidationError,
|
|
16
|
+
)
|
|
17
|
+
from novastack.workflows.runtime.execution_pool import StepExecutionPool
|
|
18
|
+
from novastack.workflows.runtime.optimized_queue import OptimizedEventQueue
|
|
19
|
+
from novastack.workflows.types import WorkflowResult, WorkflowStatus
|
|
20
|
+
|
|
21
|
+
if TYPE_CHECKING:
|
|
22
|
+
from novastack.workflows import Workflow
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class WorkflowEngine:
|
|
26
|
+
"""
|
|
27
|
+
Executes workflow steps based on event-driven architecture.
|
|
28
|
+
|
|
29
|
+
The WorkflowEngine manages the execution of decorator-based workflow steps
|
|
30
|
+
by processing events through an event queue. It handles event dispatching,
|
|
31
|
+
parallel step execution (fan-out), error handling, and iteration limits.
|
|
32
|
+
|
|
33
|
+
Attributes:
|
|
34
|
+
workflow: The workflow instance containing decorated step methods.
|
|
35
|
+
max_iterations: Maximum number of event processing iterations allowed. Use 0 for unlimited iterations.
|
|
36
|
+
queue_wait_timeout: Timeout in seconds for queue operations.
|
|
37
|
+
max_workers: Maximum concurrent step executions.
|
|
38
|
+
Defaults to 100.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def __init__(
|
|
42
|
+
self,
|
|
43
|
+
workflow: "Workflow",
|
|
44
|
+
max_iterations: int = 1000,
|
|
45
|
+
queue_wait_timeout: float = 1.0,
|
|
46
|
+
max_workers: int = 100,
|
|
47
|
+
) -> None:
|
|
48
|
+
if max_iterations < 0:
|
|
49
|
+
raise WorkflowValidationError(
|
|
50
|
+
"'max_iterations' must be 0 (unlimited) or greater"
|
|
51
|
+
)
|
|
52
|
+
if queue_wait_timeout <= 0:
|
|
53
|
+
raise WorkflowValidationError("'queue_wait_timeout' must be greater than 0")
|
|
54
|
+
|
|
55
|
+
self._workflow = workflow
|
|
56
|
+
self._max_iterations = max_iterations
|
|
57
|
+
self._queue_wait_timeout = queue_wait_timeout
|
|
58
|
+
self._pool = StepExecutionPool(max_workers)
|
|
59
|
+
|
|
60
|
+
async def run(
|
|
61
|
+
self,
|
|
62
|
+
start_events: list[StartEvent] | StartEvent,
|
|
63
|
+
ctx: Context | None = None,
|
|
64
|
+
) -> WorkflowResult:
|
|
65
|
+
"""
|
|
66
|
+
Execute the workflow with initial events.
|
|
67
|
+
|
|
68
|
+
This is the main entry point for workflow execution. It creates a
|
|
69
|
+
Context (if not provided), enqueues initial events, and processes
|
|
70
|
+
the event queue until completion or max_iterations is reached.
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
start_events: Single event or list of events to start the workflow.
|
|
74
|
+
ctx: Optional pre-configured context with optional state data.
|
|
75
|
+
|
|
76
|
+
Returns:
|
|
77
|
+
WorkflowResult containing the final result, iteration count, status,
|
|
78
|
+
and any error information.
|
|
79
|
+
|
|
80
|
+
Note:
|
|
81
|
+
The workflow continues processing events until the queue is empty,
|
|
82
|
+
a StopEvent is encountered, or max_iterations is reached. Steps are
|
|
83
|
+
executed in parallel when multiple steps listen to the same event
|
|
84
|
+
type (fan-out pattern).
|
|
85
|
+
"""
|
|
86
|
+
run_id = str(uuid.uuid4())
|
|
87
|
+
|
|
88
|
+
# Initialize optimized event queue
|
|
89
|
+
event_queue: OptimizedEventQueue[Event] = OptimizedEventQueue()
|
|
90
|
+
|
|
91
|
+
# Create or use provided context
|
|
92
|
+
if ctx is None:
|
|
93
|
+
ctx = Context(
|
|
94
|
+
workflow=self._workflow,
|
|
95
|
+
event_queue=event_queue,
|
|
96
|
+
)
|
|
97
|
+
else:
|
|
98
|
+
# Use provided context but ensure it has the event queue
|
|
99
|
+
ctx._event_queue = event_queue
|
|
100
|
+
|
|
101
|
+
# Enqueue initial events
|
|
102
|
+
if isinstance(start_events, StartEvent):
|
|
103
|
+
start_events = [start_events]
|
|
104
|
+
|
|
105
|
+
for event in start_events:
|
|
106
|
+
await event_queue.put(event)
|
|
107
|
+
|
|
108
|
+
# Process event queue
|
|
109
|
+
iteration = 0
|
|
110
|
+
result: Any = None
|
|
111
|
+
|
|
112
|
+
try:
|
|
113
|
+
while self._max_iterations == 0 or iteration < self._max_iterations:
|
|
114
|
+
try:
|
|
115
|
+
# Get next event with timeout
|
|
116
|
+
event = await asyncio.wait_for(
|
|
117
|
+
event_queue.get(),
|
|
118
|
+
timeout=self._queue_wait_timeout,
|
|
119
|
+
)
|
|
120
|
+
except asyncio.TimeoutError:
|
|
121
|
+
# Queue is empty, workflow complete
|
|
122
|
+
break
|
|
123
|
+
|
|
124
|
+
# Check for StopEvent
|
|
125
|
+
if isinstance(event, StopEvent):
|
|
126
|
+
result = event.result
|
|
127
|
+
break
|
|
128
|
+
|
|
129
|
+
# Process the event
|
|
130
|
+
await self._process_event(event, ctx)
|
|
131
|
+
|
|
132
|
+
iteration += 1
|
|
133
|
+
|
|
134
|
+
# Check if max iterations reached (only if not unlimited)
|
|
135
|
+
if self._max_iterations > 0 and iteration >= self._max_iterations:
|
|
136
|
+
raise WorkflowRuntimeError(
|
|
137
|
+
f"Execution workflow maximum iterations: ({self._max_iterations})."
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
except WorkflowRuntimeError:
|
|
141
|
+
raise
|
|
142
|
+
except Exception as e:
|
|
143
|
+
raise WorkflowRuntimeError(f"Execution failure in workflow: {str(e)}.")
|
|
144
|
+
|
|
145
|
+
# Return workflow result
|
|
146
|
+
return WorkflowResult(
|
|
147
|
+
run_id=run_id,
|
|
148
|
+
num_iterations=iteration,
|
|
149
|
+
status=WorkflowStatus.COMPLETED,
|
|
150
|
+
result=result,
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
async def _process_event(
|
|
154
|
+
self,
|
|
155
|
+
event: Event,
|
|
156
|
+
context: Context,
|
|
157
|
+
) -> None:
|
|
158
|
+
"""
|
|
159
|
+
Process a single event by dispatching to matching step methods.
|
|
160
|
+
|
|
161
|
+
This method finds all step methods that listen to the event type and
|
|
162
|
+
executes them in parallel using asyncio.gather. It handles exceptions
|
|
163
|
+
and enqueues any events returned by the steps.
|
|
164
|
+
|
|
165
|
+
Args:
|
|
166
|
+
event: The Event to process.
|
|
167
|
+
context: The Context for step execution.
|
|
168
|
+
|
|
169
|
+
Note:
|
|
170
|
+
Steps are executed in parallel (fan-out pattern) when multiple
|
|
171
|
+
steps listen to the same event type.
|
|
172
|
+
"""
|
|
173
|
+
# Find matching step methods
|
|
174
|
+
matching_steps = self._workflow.get_steps_for_event(event)
|
|
175
|
+
|
|
176
|
+
if not matching_steps:
|
|
177
|
+
# No steps listen to this event, skip silently (same behavior as return None)
|
|
178
|
+
return
|
|
179
|
+
|
|
180
|
+
# Execute all matching steps in parallel with timeout and retry
|
|
181
|
+
tasks = []
|
|
182
|
+
for step_name, step_method in matching_steps:
|
|
183
|
+
# Get timeout and retry configuration from step metadata
|
|
184
|
+
step_timeout = get_step_timeout(step_method)
|
|
185
|
+
num_retries = get_step_num_retries(step_method)
|
|
186
|
+
retry_delay = get_step_retry_delay(step_method)
|
|
187
|
+
|
|
188
|
+
# Execute step with retry logic
|
|
189
|
+
task = self._execute_step_with_retry(
|
|
190
|
+
step_method,
|
|
191
|
+
self._workflow,
|
|
192
|
+
context,
|
|
193
|
+
event,
|
|
194
|
+
step_timeout,
|
|
195
|
+
num_retries,
|
|
196
|
+
retry_delay,
|
|
197
|
+
)
|
|
198
|
+
tasks.append(task)
|
|
199
|
+
|
|
200
|
+
# Gather results with exception handling
|
|
201
|
+
results = await asyncio.gather(*tasks, return_exceptions=True)
|
|
202
|
+
|
|
203
|
+
# Process results and handle exceptions
|
|
204
|
+
for i, result in enumerate(results):
|
|
205
|
+
step_name, step_method = matching_steps[i]
|
|
206
|
+
|
|
207
|
+
if result is None:
|
|
208
|
+
pass
|
|
209
|
+
elif isinstance(result, Event):
|
|
210
|
+
# Step returned an event, emit it
|
|
211
|
+
await context.emit(result)
|
|
212
|
+
elif isinstance(result, Exception):
|
|
213
|
+
raise WorkflowRuntimeError(
|
|
214
|
+
f"Execution failure in workflow step '{step_name}'. {str(result)}"
|
|
215
|
+
)
|
|
216
|
+
else:
|
|
217
|
+
raise WorkflowRuntimeError(
|
|
218
|
+
f"Step '{step_name}' expected Event, got {type(result).__name__}. "
|
|
219
|
+
"Steps must return a single Event or None."
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
async def _execute_step_with_retry(
|
|
223
|
+
self,
|
|
224
|
+
step_method: Any,
|
|
225
|
+
workflow: Any,
|
|
226
|
+
context: Context,
|
|
227
|
+
event: Event,
|
|
228
|
+
timeout: float | None,
|
|
229
|
+
num_retries: int,
|
|
230
|
+
retry_delay: float,
|
|
231
|
+
) -> Event | None:
|
|
232
|
+
"""
|
|
233
|
+
Execute step with timeout and retry logic using execution pool.
|
|
234
|
+
|
|
235
|
+
This method wraps step execution with configurable timeout and retry
|
|
236
|
+
behavior. It will retry the step execution on any exception (including
|
|
237
|
+
timeout) up to num_retries times, with retry_delay seconds between attempts.
|
|
238
|
+
|
|
239
|
+
Args:
|
|
240
|
+
step_method: The step method to execute.
|
|
241
|
+
workflow: The workflow instance.
|
|
242
|
+
context: The Context for step execution.
|
|
243
|
+
event: The Event to process.
|
|
244
|
+
timeout: Optional timeout in seconds for step execution.
|
|
245
|
+
num_retries: Number of retry attempts on failure.
|
|
246
|
+
retry_delay: Delay in seconds between retry attempts.
|
|
247
|
+
"""
|
|
248
|
+
|
|
249
|
+
for attempt in range(num_retries + 1):
|
|
250
|
+
try:
|
|
251
|
+
if timeout is not None:
|
|
252
|
+
result = await asyncio.wait_for(
|
|
253
|
+
self._pool.execute(step_method, workflow, context, event),
|
|
254
|
+
timeout=timeout,
|
|
255
|
+
)
|
|
256
|
+
else:
|
|
257
|
+
result = await self._pool.execute(
|
|
258
|
+
step_method, workflow, context, event
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
return result
|
|
262
|
+
|
|
263
|
+
except asyncio.TimeoutError as e:
|
|
264
|
+
if attempt < num_retries:
|
|
265
|
+
await asyncio.sleep(retry_delay)
|
|
266
|
+
continue
|
|
267
|
+
|
|
268
|
+
# All retries exhausted
|
|
269
|
+
raise WorkflowTimeoutError(
|
|
270
|
+
f"Step execution timeout after {timeout}s "
|
|
271
|
+
f"(attempted {attempt + 1} times)"
|
|
272
|
+
) from e
|
|
273
|
+
|
|
274
|
+
except Exception:
|
|
275
|
+
if attempt < num_retries:
|
|
276
|
+
await asyncio.sleep(retry_delay)
|
|
277
|
+
continue
|
|
278
|
+
|
|
279
|
+
# All retries exhausted
|
|
280
|
+
raise
|
|
281
|
+
|
|
282
|
+
return None
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
from typing import Any, Callable, Coroutine
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class StepExecutionPool:
|
|
6
|
+
"""Pool for executing workflow steps with controlled concurrency."""
|
|
7
|
+
|
|
8
|
+
def __init__(self, max_workers: int = 100):
|
|
9
|
+
self._semaphore = asyncio.Semaphore(max_workers)
|
|
10
|
+
self._active_tasks = 0
|
|
11
|
+
self._lock = asyncio.Lock()
|
|
12
|
+
|
|
13
|
+
async def execute(
|
|
14
|
+
self, func: Callable[..., Coroutine[Any, Any, Any]], *args: Any, **kwargs: Any
|
|
15
|
+
) -> Any:
|
|
16
|
+
"""
|
|
17
|
+
Execute a step function with concurrency control.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
func: Async function to execute
|
|
21
|
+
*args: Positional arguments
|
|
22
|
+
**kwargs: Keyword arguments
|
|
23
|
+
"""
|
|
24
|
+
async with self._semaphore:
|
|
25
|
+
async with self._lock:
|
|
26
|
+
self._active_tasks += 1
|
|
27
|
+
|
|
28
|
+
try:
|
|
29
|
+
return await func(*args, **kwargs)
|
|
30
|
+
finally:
|
|
31
|
+
async with self._lock:
|
|
32
|
+
self._active_tasks -= 1
|
|
33
|
+
|
|
34
|
+
async def execute_batch(
|
|
35
|
+
self, tasks: list[tuple[Callable, tuple, dict]]
|
|
36
|
+
) -> list[Any]:
|
|
37
|
+
"""
|
|
38
|
+
Execute a batch of steps concurrently.
|
|
39
|
+
|
|
40
|
+
Args:
|
|
41
|
+
tasks: List of (func, args, kwargs) tuples
|
|
42
|
+
"""
|
|
43
|
+
coroutines = [
|
|
44
|
+
self.execute(func, *args, **kwargs) for func, args, kwargs in tasks
|
|
45
|
+
]
|
|
46
|
+
return await asyncio.gather(*coroutines)
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def active_tasks(self) -> int:
|
|
50
|
+
"""Get number of currently active tasks."""
|
|
51
|
+
return self._active_tasks
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
from collections import deque
|
|
3
|
+
from typing import TypeVar, Generic
|
|
4
|
+
|
|
5
|
+
from novastack.workflows.events import Event
|
|
6
|
+
|
|
7
|
+
EVENT_T = TypeVar("EVENT_T", bound=Event)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class OptimizedEventQueue(Generic[EVENT_T]):
|
|
11
|
+
"""
|
|
12
|
+
Optimized event queue for workflows.
|
|
13
|
+
|
|
14
|
+
Uses deque for O(1) operations and reduces context switching
|
|
15
|
+
compared to asyncio.Queue. This implementation delivers 15-25%
|
|
16
|
+
faster event processing with lower memory overhead and better
|
|
17
|
+
cache locality.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
def __init__(self, maxsize: int = 0):
|
|
21
|
+
"""
|
|
22
|
+
Initialize optimized event queue.
|
|
23
|
+
|
|
24
|
+
Args:
|
|
25
|
+
maxsize: Maximum queue size (0 = unlimited)
|
|
26
|
+
"""
|
|
27
|
+
self._maxsize = maxsize
|
|
28
|
+
self._queue: deque[EVENT_T] = deque()
|
|
29
|
+
self._lock = asyncio.Lock()
|
|
30
|
+
self._not_empty = asyncio.Condition(self._lock)
|
|
31
|
+
self._not_full = asyncio.Condition(self._lock) if maxsize > 0 else None
|
|
32
|
+
|
|
33
|
+
async def put(self, event: EVENT_T) -> None:
|
|
34
|
+
"""
|
|
35
|
+
Put an event into the queue.
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
event: Event to add to queue
|
|
39
|
+
"""
|
|
40
|
+
async with self._not_empty:
|
|
41
|
+
# Wait if queue is full
|
|
42
|
+
if self._maxsize > 0 and self._not_full is not None:
|
|
43
|
+
while len(self._queue) >= self._maxsize:
|
|
44
|
+
await self._not_full.wait()
|
|
45
|
+
|
|
46
|
+
# Add event (O(1) operation)
|
|
47
|
+
self._queue.append(event)
|
|
48
|
+
|
|
49
|
+
# Notify waiting consumers
|
|
50
|
+
self._not_empty.notify()
|
|
51
|
+
|
|
52
|
+
async def get(self) -> EVENT_T:
|
|
53
|
+
"""
|
|
54
|
+
Get an event from the queue.
|
|
55
|
+
|
|
56
|
+
Returns:
|
|
57
|
+
Next event from queue
|
|
58
|
+
"""
|
|
59
|
+
async with self._not_empty:
|
|
60
|
+
# Wait for events
|
|
61
|
+
while not self._queue:
|
|
62
|
+
await self._not_empty.wait()
|
|
63
|
+
|
|
64
|
+
# Get event (O(1) operation)
|
|
65
|
+
event = self._queue.popleft()
|
|
66
|
+
|
|
67
|
+
# Notify waiting producers if queue was full
|
|
68
|
+
if self._not_full is not None:
|
|
69
|
+
self._not_full.notify()
|
|
70
|
+
|
|
71
|
+
return event
|
|
72
|
+
|
|
73
|
+
def empty(self) -> bool:
|
|
74
|
+
"""Check if queue is empty."""
|
|
75
|
+
return len(self._queue) == 0
|
|
76
|
+
|
|
77
|
+
def qsize(self) -> int:
|
|
78
|
+
"""Get current queue size."""
|
|
79
|
+
return len(self._queue)
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
4
|
+
from novastack.workflows.enums import WorkflowStatus
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class DictLikeModel(BaseModel):
|
|
8
|
+
"""A Pydantic model that works like a dictionary for dynamic fields.
|
|
9
|
+
|
|
10
|
+
This class provides a Pydantic-based class system that supports both
|
|
11
|
+
attribute access (event.field) and dictionary-style access (event["field"]).
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
model_config = ConfigDict(
|
|
15
|
+
extra="allow",
|
|
16
|
+
arbitrary_types_allowed=True,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
def __getitem__(self, key: str) -> Any:
|
|
20
|
+
"""Support event["field"] access."""
|
|
21
|
+
|
|
22
|
+
try:
|
|
23
|
+
return getattr(self, key)
|
|
24
|
+
except AttributeError:
|
|
25
|
+
raise KeyError(key)
|
|
26
|
+
|
|
27
|
+
def __setitem__(self, key: str, value: Any) -> None:
|
|
28
|
+
"""Support event["field"] = value assignment."""
|
|
29
|
+
setattr(self, key, value)
|
|
30
|
+
|
|
31
|
+
def __contains__(self, key: str) -> bool:
|
|
32
|
+
"""Support 'field' in event membership test."""
|
|
33
|
+
return hasattr(self, key)
|
|
34
|
+
|
|
35
|
+
def get(self, key: str, default: Any = None) -> Any:
|
|
36
|
+
"""Get a field value with a default fallback."""
|
|
37
|
+
return getattr(self, key, default)
|
|
38
|
+
|
|
39
|
+
def keys(self) -> list[str]:
|
|
40
|
+
"""Return list of field names."""
|
|
41
|
+
return list(self.model_dump().keys())
|
|
42
|
+
|
|
43
|
+
def values(self) -> list[Any]:
|
|
44
|
+
"""Return list of field values."""
|
|
45
|
+
return list(self.model_dump().values())
|
|
46
|
+
|
|
47
|
+
def items(self) -> list[tuple[str, Any]]:
|
|
48
|
+
"""Return list of (field_name, value) tuples."""
|
|
49
|
+
return list(self.model_dump().items())
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class WorkflowResult(BaseModel):
|
|
53
|
+
"""
|
|
54
|
+
Workflow execution result.
|
|
55
|
+
|
|
56
|
+
This class encapsulates all information about a completed workflow run,
|
|
57
|
+
including its final result, execution metrics, and status.
|
|
58
|
+
|
|
59
|
+
Attributes:
|
|
60
|
+
run_id: Unique identifier for this workflow run/execution.
|
|
61
|
+
iterations: Number of iterations executed during the workflow run.
|
|
62
|
+
status: Execution status indicating completion, failure, or cancellation.
|
|
63
|
+
result: Final result from StopEvent, None if workflow didn't complete normally.
|
|
64
|
+
error: Error message if the workflow failed, None otherwise.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
model_config = ConfigDict(validate_assignment=True)
|
|
68
|
+
|
|
69
|
+
run_id: str = Field(
|
|
70
|
+
...,
|
|
71
|
+
description="Unique identifier for this workflow run/execution",
|
|
72
|
+
)
|
|
73
|
+
num_iterations: int = Field(
|
|
74
|
+
...,
|
|
75
|
+
description="Number of iterations executed",
|
|
76
|
+
ge=0,
|
|
77
|
+
)
|
|
78
|
+
status: WorkflowStatus = Field(
|
|
79
|
+
...,
|
|
80
|
+
description="Execution status",
|
|
81
|
+
)
|
|
82
|
+
result: Any | None = Field(
|
|
83
|
+
default=None,
|
|
84
|
+
description="Final result from StopEvent",
|
|
85
|
+
)
|
|
86
|
+
warnings: str | None = Field(
|
|
87
|
+
default=None,
|
|
88
|
+
description="Warning message",
|
|
89
|
+
)
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import inspect
|
|
2
|
+
import warnings
|
|
3
|
+
from typing import Any, Type, get_args, get_origin
|
|
4
|
+
|
|
5
|
+
from novastack.workflows.context import Context
|
|
6
|
+
from novastack.workflows.exceptions import WorkflowValidationError
|
|
7
|
+
from novastack.workflows.decorators import is_step_method, get_step_event_type
|
|
8
|
+
from novastack.workflows.runtime.engine import WorkflowEngine
|
|
9
|
+
from novastack.workflows.events import Event, StartEvent
|
|
10
|
+
from novastack.workflows.types import DictLikeModel
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Workflow:
|
|
14
|
+
"""
|
|
15
|
+
Event-driven workflow orchestration.
|
|
16
|
+
|
|
17
|
+
Workflows are defined by decorating methods with @step(on=EventClass).
|
|
18
|
+
Steps are automatically discovered and registered when the class is defined.
|
|
19
|
+
|
|
20
|
+
State type consistency is enforced: all steps must use the same Context[StateType].
|
|
21
|
+
|
|
22
|
+
Example:
|
|
23
|
+
```python
|
|
24
|
+
class MyWorkflow(Workflow):
|
|
25
|
+
@step(on=StartEvent)
|
|
26
|
+
async def start(self, ctx: Context[MyState], ev: StartEvent) -> MyEvent:
|
|
27
|
+
return MyEvent(data="processed")
|
|
28
|
+
|
|
29
|
+
@step(on=MyEvent)
|
|
30
|
+
async def process(self, ctx: Context[MyState], ev: MyEvent) -> StopEvent:
|
|
31
|
+
return StopEvent(result="done")
|
|
32
|
+
|
|
33
|
+
workflow = MyWorkflow()
|
|
34
|
+
result = await workflow.run(input_msg="hello")
|
|
35
|
+
```
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def __init_subclass__(cls, **kwargs):
|
|
39
|
+
"""
|
|
40
|
+
Automatically discover and register @step decorated methods.
|
|
41
|
+
|
|
42
|
+
This is called when a subclass is defined, allowing automatic
|
|
43
|
+
step discovery and state type validation.
|
|
44
|
+
"""
|
|
45
|
+
super().__init_subclass__(**kwargs)
|
|
46
|
+
|
|
47
|
+
# Build routing table: Event class -> list of step methods
|
|
48
|
+
cls._step_registry: dict[Type[Event], list[tuple[str, Any]]] = {}
|
|
49
|
+
|
|
50
|
+
# Track state types across steps for consistency validation
|
|
51
|
+
state_types: dict[str, Type] = {}
|
|
52
|
+
|
|
53
|
+
# Discover all @step decorated methods
|
|
54
|
+
for name, method in inspect.getmembers(cls, predicate=inspect.isfunction):
|
|
55
|
+
if is_step_method(method):
|
|
56
|
+
event_type = get_step_event_type(method)
|
|
57
|
+
if event_type:
|
|
58
|
+
if event_type not in cls._step_registry:
|
|
59
|
+
cls._step_registry[event_type] = []
|
|
60
|
+
cls._step_registry[event_type].append((name, method))
|
|
61
|
+
|
|
62
|
+
# Extract and validate Context state type
|
|
63
|
+
sig = inspect.signature(method)
|
|
64
|
+
for param_name, param in sig.parameters.items():
|
|
65
|
+
if param.annotation != inspect.Parameter.empty:
|
|
66
|
+
origin = get_origin(param.annotation)
|
|
67
|
+
if origin is Context:
|
|
68
|
+
args = get_args(param.annotation)
|
|
69
|
+
if args:
|
|
70
|
+
# Context[StateType] - extract StateType
|
|
71
|
+
state_type = args[0]
|
|
72
|
+
else:
|
|
73
|
+
# Context without type - defaults to DictLikeModel
|
|
74
|
+
state_type = DictLikeModel
|
|
75
|
+
|
|
76
|
+
state_types[name] = state_type
|
|
77
|
+
break
|
|
78
|
+
|
|
79
|
+
# Validate state type consistency across all steps
|
|
80
|
+
if state_types:
|
|
81
|
+
unique_types = set(state_types.values())
|
|
82
|
+
if len(unique_types) > 1:
|
|
83
|
+
type_details = "\n".join(
|
|
84
|
+
f" - {step_name}: Context[{state_type.__name__}]"
|
|
85
|
+
for step_name, state_type in state_types.items()
|
|
86
|
+
)
|
|
87
|
+
raise WorkflowValidationError(
|
|
88
|
+
f"Inconsistent state types in workflow '{cls.__name__}'.\n"
|
|
89
|
+
f"All steps must use the same Context[StateType].\n"
|
|
90
|
+
f"Found:\n{type_details}"
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
# Store the validated state type for the workflow
|
|
94
|
+
cls._state_type = next(iter(unique_types))
|
|
95
|
+
else:
|
|
96
|
+
# No typed Context found, default to DictLikeModel
|
|
97
|
+
cls._state_type = DictLikeModel
|
|
98
|
+
|
|
99
|
+
def get_steps_for_event(self, event: Event) -> list[tuple[str, Any]]:
|
|
100
|
+
"""Get all step methods that handle the given event type."""
|
|
101
|
+
event_class = type(event)
|
|
102
|
+
return self._step_registry.get(event_class, [])
|
|
103
|
+
|
|
104
|
+
async def run(
|
|
105
|
+
self,
|
|
106
|
+
ctx: Context | None = None,
|
|
107
|
+
start_event: StartEvent | None = None,
|
|
108
|
+
max_iterations: int = 1000,
|
|
109
|
+
**kwargs,
|
|
110
|
+
) -> Any:
|
|
111
|
+
"""
|
|
112
|
+
Run the workflow and return results.
|
|
113
|
+
|
|
114
|
+
Args:
|
|
115
|
+
ctx: Optional context to use. If None, creates new context.
|
|
116
|
+
start_event: Optional explicit start event. If None, creates StartEvent(**kwargs).
|
|
117
|
+
max_iterations: Maximum number of iterations before stopping. Use 0 for unlimited iterations.
|
|
118
|
+
**kwargs: Additional arguments. If start_event is None, used to create StartEvent.
|
|
119
|
+
If start_event is provided, kwargs are added as attributes to the event.
|
|
120
|
+
|
|
121
|
+
Examples:
|
|
122
|
+
# Using default StartEvent with kwargs
|
|
123
|
+
result = await workflow.run(input_msg="hello", user_id=123)
|
|
124
|
+
|
|
125
|
+
# Using custom start event
|
|
126
|
+
custom_event = StartEvent(data="important")
|
|
127
|
+
result = await workflow.run(start_event=custom_event)
|
|
128
|
+
"""
|
|
129
|
+
# Create or merge start_event
|
|
130
|
+
if start_event is None:
|
|
131
|
+
start_event = StartEvent(**kwargs)
|
|
132
|
+
elif kwargs:
|
|
133
|
+
# Warn about merging kwargs into start_event
|
|
134
|
+
warnings.warn(
|
|
135
|
+
"Merging **kwargs into StartEvent. "
|
|
136
|
+
"These will overwrite any existing attributes with the same name/key.",
|
|
137
|
+
UserWarning,
|
|
138
|
+
stacklevel=2,
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
# Merge kwargs into start_event
|
|
142
|
+
for key, value in kwargs.items():
|
|
143
|
+
setattr(start_event, key, value)
|
|
144
|
+
|
|
145
|
+
# Create engine and execute
|
|
146
|
+
runtime_engine = WorkflowEngine(workflow=self, max_iterations=max_iterations)
|
|
147
|
+
|
|
148
|
+
# Run workflow
|
|
149
|
+
workflow_result = await runtime_engine.run(start_events=start_event, ctx=ctx)
|
|
150
|
+
|
|
151
|
+
return workflow_result.result
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: novastack-workflows
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: A workflow engine to run AI applications with event-driven, stepwise control.
|
|
5
|
+
Project-URL: Repository, https://github.com/novastack-project/novastack
|
|
6
|
+
Author-email: Leonardo Furnielis <leonardofurnielis@outlook.com>
|
|
7
|
+
License: Apache-2.0
|
|
8
|
+
Keywords: AI,LLM,QA,RAG,data,observability,retrieval,semantic-search
|
|
9
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
10
|
+
Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
11
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
12
|
+
Requires-Python: <3.14,>=3.11
|
|
13
|
+
Requires-Dist: pydantic<3.0.0,>=2.12.5
|
|
14
|
+
Provides-Extra: dev
|
|
15
|
+
Requires-Dist: pytest-asyncio<2.0.0,>=1.3.0; extra == 'dev'
|
|
16
|
+
Requires-Dist: pytest<10.0.0,>=9.0.3; extra == 'dev'
|
|
17
|
+
Requires-Dist: ruff<1.0.0,>=0.15.8; extra == 'dev'
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
|
|
20
|
+
# NovaStack Workflows
|
|
21
|
+
|
|
22
|
+
A powerful and flexible event-driven workflow engine for Python, designed to build complex asynchronous workflows with ease.
|
|
23
|
+
|
|
24
|
+
## Installation
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
pip install novastack-workflows
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Quick Start
|
|
31
|
+
|
|
32
|
+
Here's a simple example to get you started with NovaStack Workflows:
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
from novastack.workflows import Workflow, Context, step, Event, StartEvent, StopEvent
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# Define a custom event
|
|
39
|
+
class MyEvent(Event):
|
|
40
|
+
message: str
|
|
41
|
+
|
|
42
|
+
# Create your workflow
|
|
43
|
+
class SimpleWorkflow(Workflow):
|
|
44
|
+
"""A simple workflow that processes a message."""
|
|
45
|
+
|
|
46
|
+
@step(on=StartEvent)
|
|
47
|
+
async def start(self, ctx: Context, ev: StartEvent) -> MyEvent:
|
|
48
|
+
# Get input from the start event
|
|
49
|
+
input_msg = ev.get("input_msg", "")
|
|
50
|
+
# Return a custom event to trigger the next step
|
|
51
|
+
return MyEvent(message=f"Processed: {input_msg}")
|
|
52
|
+
|
|
53
|
+
@step(on=MyEvent)
|
|
54
|
+
async def process(self, ctx: Context, ev: MyEvent) -> StopEvent:
|
|
55
|
+
# Process the message and return the final result
|
|
56
|
+
return StopEvent(result=ev.message)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
# Run the workflow
|
|
60
|
+
async def main():
|
|
61
|
+
workflow = SimpleWorkflow()
|
|
62
|
+
result = await workflow.run(input_msg="Hello, World!")
|
|
63
|
+
|
|
64
|
+
print(result)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Core Concepts
|
|
68
|
+
|
|
69
|
+
### Events
|
|
70
|
+
|
|
71
|
+
Events are the building blocks of workflows. They carry data between steps and trigger step execution.
|
|
72
|
+
|
|
73
|
+
- **StartEvent**: Automatically triggered when a workflow starts
|
|
74
|
+
- **StopEvent**: Signals the end of a workflow and carries the final result
|
|
75
|
+
- **Custom Events**: Define your own events by inheriting from `Event`
|
|
76
|
+
|
|
77
|
+
### Steps
|
|
78
|
+
|
|
79
|
+
Steps are methods decorated with `@step(on=EventType)` that define what happens when a specific event is received.
|
|
80
|
+
|
|
81
|
+
- Steps are asynchronous functions that receive a `Context` and an `Event`
|
|
82
|
+
- Steps can return new events to trigger subsequent steps
|
|
83
|
+
|
|
84
|
+
### Workflow
|
|
85
|
+
|
|
86
|
+
A workflow is a class that inherits from `Workflow` and contains one or more steps. It orchestrates the execution of steps based on events.
|
|
87
|
+
|
|
88
|
+
### Context
|
|
89
|
+
|
|
90
|
+
The `Context` object provides access to workflow state and allows steps to share data throughout the workflow execution.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
novastack/workflows/__init__.py,sha256=oQm4uhqsE6D6wFAgjWw2Vt43lh7WXHdOFSlK7uBUB9k,374
|
|
2
|
+
novastack/workflows/decorators.py,sha256=h2frdc941_kk4O21ub_riS7P3w33g2_ORPKxkPC3RHM,3273
|
|
3
|
+
novastack/workflows/enums.py,sha256=2ZQhAsVEA35RqRvUrQdieEeyEmkgq8zTjyEV7S6sNR8,197
|
|
4
|
+
novastack/workflows/events.py,sha256=7WLK98MOIuZnrDWK3eExRi4VtvvbHA8w3YJUuJSg4pQ,832
|
|
5
|
+
novastack/workflows/exceptions.py,sha256=O9R-lZNjXilO77p44x0Ul_gz1PPisKYGDePsdQKuAFw,454
|
|
6
|
+
novastack/workflows/types.py,sha256=IItExKLZmlDAo75Klbm3bQneatQTgTyUCUV-xpa2NfQ,2766
|
|
7
|
+
novastack/workflows/workflow.py,sha256=9Wr1dm2g024LVAIMwH-6WrQX0PnmzVrgYeaRpEWQkaQ,6174
|
|
8
|
+
novastack/workflows/context/__init__.py,sha256=HXWUYelHQ-yulqjxKgABxQADS8D1dqgxvT3HIIm1ROg,79
|
|
9
|
+
novastack/workflows/context/context.py,sha256=KmHPbpm9TDUU83EnmxUBSBz6UIZG_BoEVXMFUFNdcDA,3475
|
|
10
|
+
novastack/workflows/context/state_store.py,sha256=q1poWeItrG2ivNzRIPiafVRTi8RaEYud0z3lfbbFq1U,4050
|
|
11
|
+
novastack/workflows/runtime/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
|
+
novastack/workflows/runtime/engine.py,sha256=OlGJlFrcwKK0RMAxPuR7rXis9Zv0cQXzHhRlcEf36OA,9949
|
|
13
|
+
novastack/workflows/runtime/execution_pool.py,sha256=J9rfUfgb1zV4NKqfmt4e_4YICMelAB9qr1HTfex71C0,1507
|
|
14
|
+
novastack/workflows/runtime/optimized_queue.py,sha256=gFBLtqzeXEU8H6ZpjFBMnurO6z22DAmB5xFnhdXDeL4,2241
|
|
15
|
+
novastack_workflows-1.0.0.dist-info/METADATA,sha256=BnUiHvF6x16_ZPrHc5Ds1Gd5WgqIg8-UHuN5v83NQ7I,3002
|
|
16
|
+
novastack_workflows-1.0.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
17
|
+
novastack_workflows-1.0.0.dist-info/RECORD,,
|