puffinflow 2.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- puffinflow/__init__.py +132 -0
- puffinflow/core/__init__.py +110 -0
- puffinflow/core/agent/__init__.py +320 -0
- puffinflow/core/agent/base.py +1635 -0
- puffinflow/core/agent/checkpoint.py +50 -0
- puffinflow/core/agent/context.py +521 -0
- puffinflow/core/agent/decorators/__init__.py +90 -0
- puffinflow/core/agent/decorators/builder.py +454 -0
- puffinflow/core/agent/decorators/flexible.py +714 -0
- puffinflow/core/agent/decorators/inspection.py +144 -0
- puffinflow/core/agent/dependencies.py +57 -0
- puffinflow/core/agent/scheduling/__init__.py +21 -0
- puffinflow/core/agent/scheduling/builder.py +160 -0
- puffinflow/core/agent/scheduling/exceptions.py +35 -0
- puffinflow/core/agent/scheduling/inputs.py +137 -0
- puffinflow/core/agent/scheduling/parser.py +209 -0
- puffinflow/core/agent/scheduling/scheduler.py +413 -0
- puffinflow/core/agent/state.py +141 -0
- puffinflow/core/config.py +62 -0
- puffinflow/core/coordination/__init__.py +137 -0
- puffinflow/core/coordination/agent_group.py +359 -0
- puffinflow/core/coordination/agent_pool.py +629 -0
- puffinflow/core/coordination/agent_team.py +577 -0
- puffinflow/core/coordination/coordinator.py +720 -0
- puffinflow/core/coordination/deadlock.py +1759 -0
- puffinflow/core/coordination/fluent_api.py +421 -0
- puffinflow/core/coordination/primitives.py +478 -0
- puffinflow/core/coordination/rate_limiter.py +520 -0
- puffinflow/core/observability/__init__.py +47 -0
- puffinflow/core/observability/agent.py +139 -0
- puffinflow/core/observability/alerting.py +73 -0
- puffinflow/core/observability/config.py +127 -0
- puffinflow/core/observability/context.py +88 -0
- puffinflow/core/observability/core.py +147 -0
- puffinflow/core/observability/decorators.py +105 -0
- puffinflow/core/observability/events.py +71 -0
- puffinflow/core/observability/interfaces.py +196 -0
- puffinflow/core/observability/metrics.py +137 -0
- puffinflow/core/observability/tracing.py +209 -0
- puffinflow/core/reliability/__init__.py +27 -0
- puffinflow/core/reliability/bulkhead.py +96 -0
- puffinflow/core/reliability/circuit_breaker.py +149 -0
- puffinflow/core/reliability/leak_detector.py +122 -0
- puffinflow/core/resources/__init__.py +77 -0
- puffinflow/core/resources/allocation.py +790 -0
- puffinflow/core/resources/pool.py +645 -0
- puffinflow/core/resources/quotas.py +567 -0
- puffinflow/core/resources/requirements.py +217 -0
- puffinflow/version.py +21 -0
- puffinflow-2.dev0.dist-info/METADATA +334 -0
- puffinflow-2.dev0.dist-info/RECORD +55 -0
- puffinflow-2.dev0.dist-info/WHEEL +5 -0
- puffinflow-2.dev0.dist-info/entry_points.txt +3 -0
- puffinflow-2.dev0.dist-info/licenses/LICENSE +21 -0
- puffinflow-2.dev0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,714 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Flexible state decorator with optional parameters and multiple configuration methods.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from functools import wraps
|
|
7
|
+
from typing import Any, Callable, Optional, Union
|
|
8
|
+
|
|
9
|
+
from ...coordination.primitives import PrimitiveType
|
|
10
|
+
from ...resources.requirements import ResourceRequirements, ResourceType
|
|
11
|
+
from ..state import Priority
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class StateProfile:
|
|
16
|
+
"""Predefined state configuration profiles."""
|
|
17
|
+
|
|
18
|
+
name: str
|
|
19
|
+
cpu: float = 1.0
|
|
20
|
+
memory: float = 100.0
|
|
21
|
+
io: float = 1.0
|
|
22
|
+
network: float = 1.0
|
|
23
|
+
gpu: float = 0.0
|
|
24
|
+
priority: Priority = Priority.NORMAL
|
|
25
|
+
timeout: Optional[float] = None
|
|
26
|
+
rate_limit: Optional[float] = None
|
|
27
|
+
burst_limit: Optional[int] = None
|
|
28
|
+
coordination: Optional[str] = None # 'mutex', 'semaphore:5', 'barrier:3', etc.
|
|
29
|
+
max_retries: int = 3
|
|
30
|
+
tags: dict[str, Any] = field(default_factory=dict)
|
|
31
|
+
description: Optional[str] = None
|
|
32
|
+
|
|
33
|
+
# NEW: Reliability patterns
|
|
34
|
+
circuit_breaker: bool = False
|
|
35
|
+
circuit_breaker_config: Optional[dict[str, Any]] = None
|
|
36
|
+
bulkhead: bool = False
|
|
37
|
+
bulkhead_config: Optional[dict[str, Any]] = None
|
|
38
|
+
leak_detection: bool = True # Default enabled
|
|
39
|
+
|
|
40
|
+
def to_dict(self) -> dict[str, Any]:
|
|
41
|
+
"""Convert to dictionary, excluding None values."""
|
|
42
|
+
import copy
|
|
43
|
+
|
|
44
|
+
result = {}
|
|
45
|
+
for key, value in self.__dict__.items():
|
|
46
|
+
if value is not None and key != "name":
|
|
47
|
+
# Deep copy to avoid modifying original profile configurations
|
|
48
|
+
if isinstance(value, dict):
|
|
49
|
+
result[key] = copy.deepcopy(value)
|
|
50
|
+
else:
|
|
51
|
+
result[key] = value
|
|
52
|
+
return result
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# Predefined profiles
|
|
56
|
+
PROFILES = {
|
|
57
|
+
"minimal": StateProfile(
|
|
58
|
+
name="minimal",
|
|
59
|
+
cpu=0.1,
|
|
60
|
+
memory=50.0,
|
|
61
|
+
priority=Priority.NORMAL,
|
|
62
|
+
max_retries=1,
|
|
63
|
+
circuit_breaker=False, # Keep minimal lightweight
|
|
64
|
+
bulkhead=False,
|
|
65
|
+
leak_detection=False,
|
|
66
|
+
tags={"profile": "minimal"},
|
|
67
|
+
),
|
|
68
|
+
"standard": StateProfile(
|
|
69
|
+
name="standard",
|
|
70
|
+
cpu=1.0,
|
|
71
|
+
memory=100.0,
|
|
72
|
+
priority=Priority.NORMAL,
|
|
73
|
+
max_retries=3,
|
|
74
|
+
circuit_breaker=False, # Standard doesn't need extra protection
|
|
75
|
+
bulkhead=False,
|
|
76
|
+
leak_detection=True, # But enable leak detection
|
|
77
|
+
tags={"profile": "standard"},
|
|
78
|
+
),
|
|
79
|
+
"cpu_intensive": StateProfile(
|
|
80
|
+
name="cpu_intensive",
|
|
81
|
+
cpu=4.0,
|
|
82
|
+
memory=1024.0,
|
|
83
|
+
priority=Priority.HIGH,
|
|
84
|
+
timeout=300.0,
|
|
85
|
+
max_retries=3,
|
|
86
|
+
circuit_breaker=True, # CPU intensive operations can fail
|
|
87
|
+
bulkhead=True, # Isolate CPU intensive work
|
|
88
|
+
bulkhead_config={"max_concurrent": 2}, # Limit concurrent CPU work
|
|
89
|
+
leak_detection=True,
|
|
90
|
+
tags={"profile": "cpu_intensive", "workload": "compute"},
|
|
91
|
+
),
|
|
92
|
+
"memory_intensive": StateProfile(
|
|
93
|
+
name="memory_intensive",
|
|
94
|
+
cpu=2.0,
|
|
95
|
+
memory=4096.0,
|
|
96
|
+
priority=Priority.HIGH,
|
|
97
|
+
timeout=600.0,
|
|
98
|
+
max_retries=3,
|
|
99
|
+
circuit_breaker=True,
|
|
100
|
+
bulkhead=True,
|
|
101
|
+
bulkhead_config={"max_concurrent": 3},
|
|
102
|
+
leak_detection=True,
|
|
103
|
+
tags={"profile": "memory_intensive", "workload": "memory"},
|
|
104
|
+
),
|
|
105
|
+
"io_intensive": StateProfile(
|
|
106
|
+
name="io_intensive",
|
|
107
|
+
cpu=1.0,
|
|
108
|
+
memory=256.0,
|
|
109
|
+
io=10.0,
|
|
110
|
+
priority=Priority.NORMAL,
|
|
111
|
+
timeout=120.0,
|
|
112
|
+
max_retries=5,
|
|
113
|
+
circuit_breaker=True, # IO operations can fail often
|
|
114
|
+
circuit_breaker_config={"failure_threshold": 3, "recovery_timeout": 30.0},
|
|
115
|
+
bulkhead=True,
|
|
116
|
+
bulkhead_config={"max_concurrent": 5},
|
|
117
|
+
leak_detection=True,
|
|
118
|
+
tags={"profile": "io_intensive", "workload": "io"},
|
|
119
|
+
),
|
|
120
|
+
"gpu_accelerated": StateProfile(
|
|
121
|
+
name="gpu_accelerated",
|
|
122
|
+
cpu=2.0,
|
|
123
|
+
memory=2048.0,
|
|
124
|
+
gpu=1.0,
|
|
125
|
+
priority=Priority.HIGH,
|
|
126
|
+
timeout=900.0,
|
|
127
|
+
max_retries=2,
|
|
128
|
+
circuit_breaker=True,
|
|
129
|
+
bulkhead=True,
|
|
130
|
+
bulkhead_config={"max_concurrent": 1}, # Only one GPU operation at a time
|
|
131
|
+
leak_detection=True,
|
|
132
|
+
tags={"profile": "gpu_accelerated", "workload": "gpu"},
|
|
133
|
+
),
|
|
134
|
+
"network_intensive": StateProfile(
|
|
135
|
+
name="network_intensive",
|
|
136
|
+
cpu=1.0,
|
|
137
|
+
memory=512.0,
|
|
138
|
+
network=10.0,
|
|
139
|
+
priority=Priority.NORMAL,
|
|
140
|
+
timeout=60.0,
|
|
141
|
+
max_retries=5,
|
|
142
|
+
circuit_breaker=True, # Network operations are unreliable
|
|
143
|
+
circuit_breaker_config={"failure_threshold": 2, "recovery_timeout": 20.0},
|
|
144
|
+
bulkhead=True,
|
|
145
|
+
bulkhead_config={"max_concurrent": 10},
|
|
146
|
+
leak_detection=True,
|
|
147
|
+
tags={"profile": "network_intensive", "workload": "network"},
|
|
148
|
+
),
|
|
149
|
+
"quick": StateProfile(
|
|
150
|
+
name="quick",
|
|
151
|
+
cpu=0.5,
|
|
152
|
+
memory=50.0,
|
|
153
|
+
priority=Priority.NORMAL,
|
|
154
|
+
timeout=30.0,
|
|
155
|
+
rate_limit=100.0,
|
|
156
|
+
max_retries=2,
|
|
157
|
+
circuit_breaker=False, # Quick operations shouldn't need circuit breaker
|
|
158
|
+
bulkhead=True,
|
|
159
|
+
bulkhead_config={"max_concurrent": 20}, # Allow many quick operations
|
|
160
|
+
leak_detection=False, # Quick operations shouldn't leak
|
|
161
|
+
tags={"profile": "quick", "speed": "fast"},
|
|
162
|
+
),
|
|
163
|
+
"batch": StateProfile(
|
|
164
|
+
name="batch",
|
|
165
|
+
cpu=2.0,
|
|
166
|
+
memory=1024.0,
|
|
167
|
+
priority=Priority.LOW,
|
|
168
|
+
timeout=1800.0,
|
|
169
|
+
max_retries=3,
|
|
170
|
+
circuit_breaker=True,
|
|
171
|
+
bulkhead=True,
|
|
172
|
+
bulkhead_config={"max_concurrent": 3},
|
|
173
|
+
leak_detection=True,
|
|
174
|
+
tags={"profile": "batch", "workload": "batch"},
|
|
175
|
+
),
|
|
176
|
+
"critical": StateProfile(
|
|
177
|
+
name="critical",
|
|
178
|
+
cpu=2.0,
|
|
179
|
+
memory=512.0,
|
|
180
|
+
priority=Priority.CRITICAL,
|
|
181
|
+
coordination="mutex",
|
|
182
|
+
max_retries=3,
|
|
183
|
+
circuit_breaker=False, # Critical operations should not be circuit broken
|
|
184
|
+
bulkhead=True,
|
|
185
|
+
bulkhead_config={"max_concurrent": 1}, # Exclusive execution
|
|
186
|
+
leak_detection=True,
|
|
187
|
+
tags={"profile": "critical", "importance": "high"},
|
|
188
|
+
),
|
|
189
|
+
"concurrent": StateProfile(
|
|
190
|
+
name="concurrent",
|
|
191
|
+
cpu=1.0,
|
|
192
|
+
memory=256.0,
|
|
193
|
+
priority=Priority.NORMAL,
|
|
194
|
+
coordination="semaphore:5",
|
|
195
|
+
max_retries=3,
|
|
196
|
+
circuit_breaker=True,
|
|
197
|
+
bulkhead=True,
|
|
198
|
+
bulkhead_config={"max_concurrent": 5}, # Match semaphore limit
|
|
199
|
+
leak_detection=True,
|
|
200
|
+
tags={"profile": "concurrent", "concurrency": "limited"},
|
|
201
|
+
),
|
|
202
|
+
"synchronized": StateProfile(
|
|
203
|
+
name="synchronized",
|
|
204
|
+
cpu=1.0,
|
|
205
|
+
memory=200.0,
|
|
206
|
+
priority=Priority.NORMAL,
|
|
207
|
+
coordination="barrier:3",
|
|
208
|
+
max_retries=3,
|
|
209
|
+
circuit_breaker=False, # Barrier synchronization shouldn't be circuit broken
|
|
210
|
+
bulkhead=False, # Barriers need to coordinate
|
|
211
|
+
leak_detection=True,
|
|
212
|
+
tags={"profile": "synchronized", "sync": "barrier"},
|
|
213
|
+
),
|
|
214
|
+
# Dead letter specific profiles
|
|
215
|
+
"resilient": StateProfile(
|
|
216
|
+
name="resilient",
|
|
217
|
+
cpu=1.0,
|
|
218
|
+
memory=200.0,
|
|
219
|
+
priority=Priority.NORMAL,
|
|
220
|
+
max_retries=5,
|
|
221
|
+
circuit_breaker=True, # Resilient means protected
|
|
222
|
+
circuit_breaker_config={"failure_threshold": 5, "recovery_timeout": 60.0},
|
|
223
|
+
bulkhead=True,
|
|
224
|
+
bulkhead_config={"max_concurrent": 3},
|
|
225
|
+
leak_detection=True,
|
|
226
|
+
tags={"profile": "resilient", "dead_letter": "enabled"},
|
|
227
|
+
),
|
|
228
|
+
"critical_no_dlq": StateProfile(
|
|
229
|
+
name="critical_no_dlq",
|
|
230
|
+
cpu=2.0,
|
|
231
|
+
memory=512.0,
|
|
232
|
+
priority=Priority.CRITICAL,
|
|
233
|
+
max_retries=3,
|
|
234
|
+
circuit_breaker=False, # Critical should fail fast, not be circuit broken
|
|
235
|
+
bulkhead=True,
|
|
236
|
+
bulkhead_config={"max_concurrent": 1},
|
|
237
|
+
leak_detection=True,
|
|
238
|
+
tags={"profile": "critical", "dead_letter": "disabled"},
|
|
239
|
+
),
|
|
240
|
+
# NEW: Reliability-focused profiles
|
|
241
|
+
"fault_tolerant": StateProfile(
|
|
242
|
+
name="fault_tolerant",
|
|
243
|
+
cpu=1.0,
|
|
244
|
+
memory=256.0,
|
|
245
|
+
priority=Priority.NORMAL,
|
|
246
|
+
max_retries=5,
|
|
247
|
+
circuit_breaker=True,
|
|
248
|
+
circuit_breaker_config={"failure_threshold": 3, "recovery_timeout": 45.0},
|
|
249
|
+
bulkhead=True,
|
|
250
|
+
bulkhead_config={"max_concurrent": 4, "max_queue_size": 20},
|
|
251
|
+
leak_detection=True,
|
|
252
|
+
tags={"profile": "fault_tolerant", "reliability": "high"},
|
|
253
|
+
),
|
|
254
|
+
"external_service": StateProfile(
|
|
255
|
+
name="external_service",
|
|
256
|
+
cpu=0.5,
|
|
257
|
+
memory=128.0,
|
|
258
|
+
priority=Priority.NORMAL,
|
|
259
|
+
timeout=30.0,
|
|
260
|
+
max_retries=3,
|
|
261
|
+
circuit_breaker=True,
|
|
262
|
+
circuit_breaker_config={"failure_threshold": 2, "recovery_timeout": 30.0},
|
|
263
|
+
bulkhead=True,
|
|
264
|
+
bulkhead_config={"max_concurrent": 8, "timeout": 10.0},
|
|
265
|
+
leak_detection=True,
|
|
266
|
+
tags={"profile": "external_service", "type": "integration"},
|
|
267
|
+
),
|
|
268
|
+
"high_availability": StateProfile(
|
|
269
|
+
name="high_availability",
|
|
270
|
+
cpu=1.5,
|
|
271
|
+
memory=512.0,
|
|
272
|
+
priority=Priority.HIGH,
|
|
273
|
+
max_retries=10,
|
|
274
|
+
circuit_breaker=True,
|
|
275
|
+
circuit_breaker_config={"failure_threshold": 5, "recovery_timeout": 120.0},
|
|
276
|
+
bulkhead=True,
|
|
277
|
+
bulkhead_config={"max_concurrent": 2, "max_queue_size": 50},
|
|
278
|
+
leak_detection=True,
|
|
279
|
+
tags={"profile": "high_availability", "sla": "99.9%"},
|
|
280
|
+
),
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
class FlexibleStateDecorator:
|
|
285
|
+
"""
|
|
286
|
+
Flexible state decorator that supports multiple configuration methods.
|
|
287
|
+
"""
|
|
288
|
+
|
|
289
|
+
def __init__(self) -> None:
|
|
290
|
+
self.default_config: dict[str, Any] = {}
|
|
291
|
+
|
|
292
|
+
def __call__(self, *args: Any, **kwargs: Any) -> Callable[..., Any]:
|
|
293
|
+
"""
|
|
294
|
+
Handle multiple call patterns:
|
|
295
|
+
- @state
|
|
296
|
+
- @state()
|
|
297
|
+
- @state(profile='cpu_intensive')
|
|
298
|
+
- @state(cpu=2.0, memory=512.0)
|
|
299
|
+
- @state(config={'cpu': 2.0})
|
|
300
|
+
"""
|
|
301
|
+
# Case 1: @state (direct decoration without parentheses)
|
|
302
|
+
if len(args) == 1 and callable(args[0]) and not kwargs:
|
|
303
|
+
func = args[0]
|
|
304
|
+
# Still need to merge configurations to resolve profiles
|
|
305
|
+
final_config = self._merge_configurations()
|
|
306
|
+
return self._decorate_function(func, final_config)
|
|
307
|
+
|
|
308
|
+
# Case 2: @state() or @state(params...)
|
|
309
|
+
def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
|
|
310
|
+
# Merge all configuration sources
|
|
311
|
+
final_config = self._merge_configurations(*args, **kwargs)
|
|
312
|
+
return self._decorate_function(func, final_config)
|
|
313
|
+
|
|
314
|
+
return decorator
|
|
315
|
+
|
|
316
|
+
def _merge_configurations(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
|
|
317
|
+
"""Merge configuration from multiple sources in priority order."""
|
|
318
|
+
final_config = {}
|
|
319
|
+
|
|
320
|
+
# Apply default profile first if present in default_config
|
|
321
|
+
default_profile = self.default_config.get("profile")
|
|
322
|
+
if default_profile and default_profile in PROFILES:
|
|
323
|
+
profile_config = PROFILES[default_profile].to_dict()
|
|
324
|
+
final_config.update(profile_config)
|
|
325
|
+
|
|
326
|
+
# Then apply other default config (excluding profile key)
|
|
327
|
+
for key, value in self.default_config.items():
|
|
328
|
+
if key != "profile":
|
|
329
|
+
final_config[key] = value
|
|
330
|
+
|
|
331
|
+
# Process positional arguments
|
|
332
|
+
for arg in args:
|
|
333
|
+
if isinstance(arg, dict):
|
|
334
|
+
# Direct config dictionary
|
|
335
|
+
final_config.update(arg)
|
|
336
|
+
elif isinstance(arg, str):
|
|
337
|
+
# Profile name - validate it exists
|
|
338
|
+
if arg not in PROFILES:
|
|
339
|
+
raise ValueError(f"Unknown profile: {arg}")
|
|
340
|
+
profile_config = PROFILES[arg].to_dict()
|
|
341
|
+
final_config.update(profile_config)
|
|
342
|
+
elif isinstance(arg, StateProfile):
|
|
343
|
+
# Profile object
|
|
344
|
+
profile_config = arg.to_dict()
|
|
345
|
+
final_config.update(profile_config)
|
|
346
|
+
|
|
347
|
+
# Process keyword arguments (highest priority)
|
|
348
|
+
# Handle special cases
|
|
349
|
+
config_dict = kwargs.pop("config", {})
|
|
350
|
+
profile_name = kwargs.pop("profile", None)
|
|
351
|
+
|
|
352
|
+
# Apply profile first
|
|
353
|
+
if profile_name:
|
|
354
|
+
if profile_name not in PROFILES:
|
|
355
|
+
raise ValueError(f"Unknown profile: {profile_name}")
|
|
356
|
+
profile_config = PROFILES[profile_name].to_dict()
|
|
357
|
+
final_config.update(profile_config)
|
|
358
|
+
|
|
359
|
+
# Apply config dict
|
|
360
|
+
if config_dict:
|
|
361
|
+
final_config.update(config_dict)
|
|
362
|
+
|
|
363
|
+
# Apply direct keyword arguments (highest priority)
|
|
364
|
+
final_config.update(kwargs)
|
|
365
|
+
|
|
366
|
+
return final_config
|
|
367
|
+
|
|
368
|
+
def _decorate_function(self, func: Callable, config: dict[str, Any]) -> Callable:
|
|
369
|
+
"""Apply decoration with merged configuration."""
|
|
370
|
+
# Set default values for any missing configuration
|
|
371
|
+
defaults: dict[str, Any] = {
|
|
372
|
+
"cpu": 1.0,
|
|
373
|
+
"memory": 100.0,
|
|
374
|
+
"io": 1.0,
|
|
375
|
+
"network": 1.0,
|
|
376
|
+
"gpu": 0.0,
|
|
377
|
+
"priority": Priority.NORMAL,
|
|
378
|
+
"timeout": None,
|
|
379
|
+
"rate_limit": None,
|
|
380
|
+
"burst_limit": None,
|
|
381
|
+
"coordination": None,
|
|
382
|
+
"depends_on": [],
|
|
383
|
+
"max_retries": 3,
|
|
384
|
+
"tags": {},
|
|
385
|
+
"description": None,
|
|
386
|
+
# NEW: Reliability defaults
|
|
387
|
+
"circuit_breaker": False,
|
|
388
|
+
"circuit_breaker_config": None,
|
|
389
|
+
"bulkhead": False,
|
|
390
|
+
"bulkhead_config": None,
|
|
391
|
+
"leak_detection": True,
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
# Merge defaults with provided config (only for missing keys)
|
|
395
|
+
for key, default_value in defaults.items():
|
|
396
|
+
if key not in config:
|
|
397
|
+
config[key] = default_value
|
|
398
|
+
|
|
399
|
+
# Process and validate configuration
|
|
400
|
+
config = self._process_configuration(config, func)
|
|
401
|
+
|
|
402
|
+
# Apply configuration to function
|
|
403
|
+
return self._apply_configuration(func, config)
|
|
404
|
+
|
|
405
|
+
def _process_configuration(
|
|
406
|
+
self, config: dict[str, Any], func: Callable
|
|
407
|
+
) -> dict[str, Any]:
|
|
408
|
+
"""Process and validate configuration."""
|
|
409
|
+
# Normalize priority
|
|
410
|
+
priority = config["priority"]
|
|
411
|
+
if isinstance(priority, str):
|
|
412
|
+
try:
|
|
413
|
+
priority = getattr(Priority, priority.upper())
|
|
414
|
+
except AttributeError as e:
|
|
415
|
+
raise KeyError(f"Invalid priority: {priority}") from e
|
|
416
|
+
elif isinstance(priority, int):
|
|
417
|
+
priority = Priority(priority)
|
|
418
|
+
config["priority"] = priority
|
|
419
|
+
|
|
420
|
+
# Process coordination string
|
|
421
|
+
coordination = config.get("coordination")
|
|
422
|
+
if coordination and isinstance(coordination, str):
|
|
423
|
+
coord_config = self._parse_coordination_string(coordination)
|
|
424
|
+
config.update(coord_config)
|
|
425
|
+
|
|
426
|
+
# Normalize dependencies
|
|
427
|
+
depends_on = config.get("depends_on")
|
|
428
|
+
if isinstance(depends_on, str):
|
|
429
|
+
config["depends_on"] = [depends_on]
|
|
430
|
+
elif depends_on is None:
|
|
431
|
+
config["depends_on"] = []
|
|
432
|
+
|
|
433
|
+
# Auto-generate description if not provided
|
|
434
|
+
if not config.get("description"):
|
|
435
|
+
# Use function docstring if available, otherwise generate from name
|
|
436
|
+
if func.__doc__ and func.__doc__.strip():
|
|
437
|
+
config["description"] = func.__doc__.strip()
|
|
438
|
+
else:
|
|
439
|
+
config["description"] = f"State: {func.__name__}"
|
|
440
|
+
|
|
441
|
+
# Process tags
|
|
442
|
+
tags = config.get("tags", {})
|
|
443
|
+
if not isinstance(tags, dict):
|
|
444
|
+
tags = {}
|
|
445
|
+
|
|
446
|
+
# Add automatic tags
|
|
447
|
+
auto_tags = {"function_name": func.__name__, "decorated_at": "runtime"}
|
|
448
|
+
auto_tags.update(tags)
|
|
449
|
+
config["tags"] = auto_tags
|
|
450
|
+
|
|
451
|
+
# Process dead letter configuration
|
|
452
|
+
dead_letter_enabled = config.get("dead_letter", True)
|
|
453
|
+
no_dead_letter = config.get("no_dead_letter", False)
|
|
454
|
+
|
|
455
|
+
if no_dead_letter:
|
|
456
|
+
dead_letter_enabled = False
|
|
457
|
+
|
|
458
|
+
# Process retry configuration with dead letter
|
|
459
|
+
retry_config = config.get("retry_config")
|
|
460
|
+
max_retries = config.get("max_retries", config.get("retries", 3))
|
|
461
|
+
|
|
462
|
+
if retry_config or max_retries != 3 or not dead_letter_enabled:
|
|
463
|
+
if isinstance(retry_config, dict):
|
|
464
|
+
retry_config["dead_letter_on_max_retries"] = dead_letter_enabled
|
|
465
|
+
retry_config["dead_letter_on_timeout"] = dead_letter_enabled
|
|
466
|
+
else:
|
|
467
|
+
retry_config = {
|
|
468
|
+
"max_retries": max_retries,
|
|
469
|
+
"dead_letter_on_max_retries": dead_letter_enabled,
|
|
470
|
+
"dead_letter_on_timeout": dead_letter_enabled,
|
|
471
|
+
}
|
|
472
|
+
config["retry_config"] = retry_config
|
|
473
|
+
|
|
474
|
+
# NEW: Process reliability patterns
|
|
475
|
+
self._process_reliability_config(config, func)
|
|
476
|
+
|
|
477
|
+
return config
|
|
478
|
+
|
|
479
|
+
def _process_reliability_config(
|
|
480
|
+
self, config: dict[str, Any], func: Callable
|
|
481
|
+
) -> None:
|
|
482
|
+
"""Process reliability pattern configuration."""
|
|
483
|
+
# Circuit breaker configuration
|
|
484
|
+
if config.get("circuit_breaker"):
|
|
485
|
+
cb_config = config.get("circuit_breaker_config", {})
|
|
486
|
+
if not isinstance(cb_config, dict):
|
|
487
|
+
cb_config = {}
|
|
488
|
+
|
|
489
|
+
# Set defaults
|
|
490
|
+
cb_defaults = {
|
|
491
|
+
"name": f"{func.__name__}_circuit_breaker",
|
|
492
|
+
"failure_threshold": 5,
|
|
493
|
+
"recovery_timeout": 60.0,
|
|
494
|
+
"success_threshold": 3,
|
|
495
|
+
"timeout": 30.0,
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
for key, default in cb_defaults.items():
|
|
499
|
+
if key not in cb_config:
|
|
500
|
+
cb_config[key] = default
|
|
501
|
+
|
|
502
|
+
config["circuit_breaker_config"] = cb_config
|
|
503
|
+
|
|
504
|
+
# Bulkhead configuration
|
|
505
|
+
if config.get("bulkhead"):
|
|
506
|
+
bh_config = config.get("bulkhead_config", {})
|
|
507
|
+
if not isinstance(bh_config, dict):
|
|
508
|
+
bh_config = {}
|
|
509
|
+
|
|
510
|
+
# Set defaults
|
|
511
|
+
bh_defaults = {
|
|
512
|
+
"name": f"{func.__name__}_bulkhead",
|
|
513
|
+
"max_concurrent": 5,
|
|
514
|
+
"max_queue_size": 100,
|
|
515
|
+
"timeout": 30.0,
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
for key, default in bh_defaults.items():
|
|
519
|
+
if key not in bh_config:
|
|
520
|
+
bh_config[key] = default
|
|
521
|
+
|
|
522
|
+
config["bulkhead_config"] = bh_config
|
|
523
|
+
|
|
524
|
+
def _parse_coordination_string(self, coordination: str) -> dict[str, Any]:
|
|
525
|
+
"""Parse coordination string like 'mutex', 'semaphore:5', 'barrier:3'."""
|
|
526
|
+
if ":" in coordination:
|
|
527
|
+
coord_type, param_str = coordination.split(":", 1)
|
|
528
|
+
try:
|
|
529
|
+
param: Optional[int] = int(param_str)
|
|
530
|
+
except ValueError:
|
|
531
|
+
param = None
|
|
532
|
+
else:
|
|
533
|
+
coord_type = coordination
|
|
534
|
+
param = None
|
|
535
|
+
|
|
536
|
+
coord_type = coord_type.lower()
|
|
537
|
+
|
|
538
|
+
if coord_type == "mutex":
|
|
539
|
+
return {"mutex": True}
|
|
540
|
+
elif coord_type == "semaphore":
|
|
541
|
+
if param is None:
|
|
542
|
+
raise ValueError(f"Unknown coordination type: {coord_type}")
|
|
543
|
+
return {"semaphore": param}
|
|
544
|
+
elif coord_type == "barrier":
|
|
545
|
+
if param is None:
|
|
546
|
+
raise ValueError(f"Unknown coordination type: {coord_type}")
|
|
547
|
+
return {"barrier": param}
|
|
548
|
+
elif coord_type == "lease":
|
|
549
|
+
if param is None:
|
|
550
|
+
raise ValueError(f"Unknown coordination type: {coord_type}")
|
|
551
|
+
return {"lease": param}
|
|
552
|
+
elif coord_type == "quota":
|
|
553
|
+
if param is None:
|
|
554
|
+
raise ValueError(f"Unknown coordination type: {coord_type}")
|
|
555
|
+
return {"quota": param}
|
|
556
|
+
else:
|
|
557
|
+
raise ValueError(f"Unknown coordination type: {coord_type}")
|
|
558
|
+
|
|
559
|
+
def _apply_configuration(self, func: Callable, config: dict[str, Any]) -> Callable:
|
|
560
|
+
"""Apply the final configuration to the function."""
|
|
561
|
+
# Create resource requirements
|
|
562
|
+
resource_types = ResourceType.NONE
|
|
563
|
+
|
|
564
|
+
if config["cpu"] > 0:
|
|
565
|
+
resource_types |= ResourceType.CPU
|
|
566
|
+
if config["memory"] > 0:
|
|
567
|
+
resource_types |= ResourceType.MEMORY
|
|
568
|
+
if config["io"] > 0:
|
|
569
|
+
resource_types |= ResourceType.IO
|
|
570
|
+
if config["network"] > 0:
|
|
571
|
+
resource_types |= ResourceType.NETWORK
|
|
572
|
+
if config["gpu"] > 0:
|
|
573
|
+
resource_types |= ResourceType.GPU
|
|
574
|
+
|
|
575
|
+
requirements = ResourceRequirements(
|
|
576
|
+
cpu_units=config["cpu"],
|
|
577
|
+
memory_mb=config["memory"],
|
|
578
|
+
io_weight=config["io"],
|
|
579
|
+
network_weight=config["network"],
|
|
580
|
+
gpu_units=config["gpu"],
|
|
581
|
+
priority_boost=config["priority"].value,
|
|
582
|
+
timeout=config["timeout"],
|
|
583
|
+
resource_types=resource_types,
|
|
584
|
+
)
|
|
585
|
+
|
|
586
|
+
# Create dependency configurations
|
|
587
|
+
from ..dependencies import DependencyType
|
|
588
|
+
|
|
589
|
+
class DependencyConfig:
|
|
590
|
+
def __init__(self, dep_type: Any) -> None:
|
|
591
|
+
self.type = dep_type
|
|
592
|
+
|
|
593
|
+
dependency_configs = {}
|
|
594
|
+
deps = config.get("depends_on", [])
|
|
595
|
+
for dep in deps:
|
|
596
|
+
dependency_configs[dep] = DependencyConfig(DependencyType.REQUIRED)
|
|
597
|
+
|
|
598
|
+
# Determine coordination primitive
|
|
599
|
+
coordination_primitive = None
|
|
600
|
+
coordination_config = {}
|
|
601
|
+
|
|
602
|
+
if config.get("mutex"):
|
|
603
|
+
coordination_primitive = PrimitiveType.MUTEX
|
|
604
|
+
coordination_config = {"ttl": 30.0}
|
|
605
|
+
elif config.get("semaphore"):
|
|
606
|
+
coordination_primitive = PrimitiveType.SEMAPHORE
|
|
607
|
+
coordination_config = {"max_count": config["semaphore"], "ttl": 30.0}
|
|
608
|
+
elif config.get("barrier"):
|
|
609
|
+
coordination_primitive = PrimitiveType.BARRIER
|
|
610
|
+
coordination_config = {"parties": config["barrier"]}
|
|
611
|
+
elif config.get("lease"):
|
|
612
|
+
coordination_primitive = PrimitiveType.LEASE
|
|
613
|
+
coordination_config = {"ttl": config["lease"], "auto_renew": True}
|
|
614
|
+
elif config.get("quota"):
|
|
615
|
+
coordination_primitive = PrimitiveType.QUOTA
|
|
616
|
+
coordination_config = {"limit": config["quota"]}
|
|
617
|
+
|
|
618
|
+
# CRITICAL: Mark as PuffinFlow state
|
|
619
|
+
func._puffinflow_state = True # type: ignore
|
|
620
|
+
func._state_name = func.__name__ # type: ignore
|
|
621
|
+
func._state_config = config # type: ignore
|
|
622
|
+
|
|
623
|
+
# Store all configuration as function attributes
|
|
624
|
+
func._resource_requirements = requirements # type: ignore
|
|
625
|
+
func._priority = config["priority"] # type: ignore
|
|
626
|
+
func._dependency_configs = dependency_configs # type: ignore
|
|
627
|
+
func._coordination_primitive = coordination_primitive # type: ignore
|
|
628
|
+
func._coordination_config = coordination_config # type: ignore
|
|
629
|
+
|
|
630
|
+
# Store rate limiting
|
|
631
|
+
if config["rate_limit"]:
|
|
632
|
+
func._rate_limit = config["rate_limit"] # type: ignore
|
|
633
|
+
func._burst_limit = config["burst_limit"] or int(config["rate_limit"] * 2) # type: ignore
|
|
634
|
+
|
|
635
|
+
# NEW: Store reliability configurations
|
|
636
|
+
if config.get("circuit_breaker"):
|
|
637
|
+
func._circuit_breaker_enabled = True # type: ignore
|
|
638
|
+
func._circuit_breaker_config = config.get("circuit_breaker_config") # type: ignore
|
|
639
|
+
else:
|
|
640
|
+
func._circuit_breaker_enabled = False # type: ignore
|
|
641
|
+
|
|
642
|
+
if config.get("bulkhead"):
|
|
643
|
+
func._bulkhead_enabled = True # type: ignore
|
|
644
|
+
func._bulkhead_config = config.get("bulkhead_config") # type: ignore
|
|
645
|
+
else:
|
|
646
|
+
func._bulkhead_enabled = False # type: ignore
|
|
647
|
+
|
|
648
|
+
func._leak_detection_enabled = config.get("leak_detection", True) # type: ignore
|
|
649
|
+
|
|
650
|
+
# Store metadata
|
|
651
|
+
func._state_config = config # type: ignore
|
|
652
|
+
func._state_tags = config["tags"] # type: ignore
|
|
653
|
+
func._state_description = config["description"] # type: ignore
|
|
654
|
+
|
|
655
|
+
# Preserve function metadata
|
|
656
|
+
func = wraps(func)(func)
|
|
657
|
+
|
|
658
|
+
return func
|
|
659
|
+
|
|
660
|
+
def with_defaults(self, **defaults: Any) -> "FlexibleStateDecorator":
|
|
661
|
+
"""Create a new decorator with different default values."""
|
|
662
|
+
new_decorator = FlexibleStateDecorator()
|
|
663
|
+
new_decorator.default_config = {**self.default_config, **defaults}
|
|
664
|
+
return new_decorator
|
|
665
|
+
|
|
666
|
+
def create_profile(self, name: str, **config: Any) -> StateProfile:
|
|
667
|
+
"""Create a new profile."""
|
|
668
|
+
return StateProfile(name=name, **config)
|
|
669
|
+
|
|
670
|
+
def register_profile(
|
|
671
|
+
self, profile: Union[StateProfile, str], **config: Any
|
|
672
|
+
) -> None:
|
|
673
|
+
"""Register a new profile globally."""
|
|
674
|
+
if isinstance(profile, str):
|
|
675
|
+
profile = StateProfile(name=profile, **config)
|
|
676
|
+
|
|
677
|
+
PROFILES[profile.name] = profile
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
# Create the main decorator instance
|
|
681
|
+
state = FlexibleStateDecorator()
|
|
682
|
+
|
|
683
|
+
# Create specialized decorators with defaults
|
|
684
|
+
minimal_state = state.with_defaults(profile="minimal")
|
|
685
|
+
cpu_intensive = state.with_defaults(profile="cpu_intensive")
|
|
686
|
+
memory_intensive = state.with_defaults(profile="memory_intensive")
|
|
687
|
+
io_intensive = state.with_defaults(profile="io_intensive")
|
|
688
|
+
gpu_accelerated = state.with_defaults(profile="gpu_accelerated")
|
|
689
|
+
network_intensive = state.with_defaults(profile="network_intensive")
|
|
690
|
+
quick_state = state.with_defaults(profile="quick")
|
|
691
|
+
batch_state = state.with_defaults(profile="batch")
|
|
692
|
+
critical_state = state.with_defaults(profile="critical")
|
|
693
|
+
concurrent_state = state.with_defaults(profile="concurrent")
|
|
694
|
+
synchronized_state = state.with_defaults(profile="synchronized")
|
|
695
|
+
|
|
696
|
+
# NEW: Reliability-focused decorators
|
|
697
|
+
fault_tolerant = state.with_defaults(profile="fault_tolerant")
|
|
698
|
+
external_service = state.with_defaults(profile="external_service")
|
|
699
|
+
high_availability = state.with_defaults(profile="high_availability")
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
def get_profile(name: str) -> Optional[StateProfile]:
|
|
703
|
+
"""Get a profile by name."""
|
|
704
|
+
return PROFILES.get(name)
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
def list_profiles() -> list[str]:
|
|
708
|
+
"""List all available profile names."""
|
|
709
|
+
return list(PROFILES.keys())
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
def create_custom_decorator(**defaults: Any) -> FlexibleStateDecorator:
|
|
713
|
+
"""Create a custom decorator with specific defaults."""
|
|
714
|
+
return state.with_defaults(**defaults)
|