pyworkflow-engine 0.1.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. dashboard/backend/app/__init__.py +1 -0
  2. dashboard/backend/app/config.py +32 -0
  3. dashboard/backend/app/controllers/__init__.py +6 -0
  4. dashboard/backend/app/controllers/run_controller.py +86 -0
  5. dashboard/backend/app/controllers/workflow_controller.py +33 -0
  6. dashboard/backend/app/dependencies/__init__.py +5 -0
  7. dashboard/backend/app/dependencies/storage.py +50 -0
  8. dashboard/backend/app/repositories/__init__.py +6 -0
  9. dashboard/backend/app/repositories/run_repository.py +80 -0
  10. dashboard/backend/app/repositories/workflow_repository.py +27 -0
  11. dashboard/backend/app/rest/__init__.py +8 -0
  12. dashboard/backend/app/rest/v1/__init__.py +12 -0
  13. dashboard/backend/app/rest/v1/health.py +33 -0
  14. dashboard/backend/app/rest/v1/runs.py +133 -0
  15. dashboard/backend/app/rest/v1/workflows.py +41 -0
  16. dashboard/backend/app/schemas/__init__.py +23 -0
  17. dashboard/backend/app/schemas/common.py +16 -0
  18. dashboard/backend/app/schemas/event.py +24 -0
  19. dashboard/backend/app/schemas/hook.py +25 -0
  20. dashboard/backend/app/schemas/run.py +54 -0
  21. dashboard/backend/app/schemas/step.py +28 -0
  22. dashboard/backend/app/schemas/workflow.py +31 -0
  23. dashboard/backend/app/server.py +87 -0
  24. dashboard/backend/app/services/__init__.py +6 -0
  25. dashboard/backend/app/services/run_service.py +240 -0
  26. dashboard/backend/app/services/workflow_service.py +155 -0
  27. dashboard/backend/main.py +18 -0
  28. docs/concepts/cancellation.mdx +362 -0
  29. docs/concepts/continue-as-new.mdx +434 -0
  30. docs/concepts/events.mdx +266 -0
  31. docs/concepts/fault-tolerance.mdx +370 -0
  32. docs/concepts/hooks.mdx +552 -0
  33. docs/concepts/limitations.mdx +167 -0
  34. docs/concepts/schedules.mdx +775 -0
  35. docs/concepts/sleep.mdx +312 -0
  36. docs/concepts/steps.mdx +301 -0
  37. docs/concepts/workflows.mdx +255 -0
  38. docs/guides/cli.mdx +942 -0
  39. docs/guides/configuration.mdx +560 -0
  40. docs/introduction.mdx +155 -0
  41. docs/quickstart.mdx +279 -0
  42. examples/__init__.py +1 -0
  43. examples/celery/__init__.py +1 -0
  44. examples/celery/durable/docker-compose.yml +55 -0
  45. examples/celery/durable/pyworkflow.config.yaml +12 -0
  46. examples/celery/durable/workflows/__init__.py +122 -0
  47. examples/celery/durable/workflows/basic.py +87 -0
  48. examples/celery/durable/workflows/batch_processing.py +102 -0
  49. examples/celery/durable/workflows/cancellation.py +273 -0
  50. examples/celery/durable/workflows/child_workflow_patterns.py +240 -0
  51. examples/celery/durable/workflows/child_workflows.py +202 -0
  52. examples/celery/durable/workflows/continue_as_new.py +260 -0
  53. examples/celery/durable/workflows/fault_tolerance.py +210 -0
  54. examples/celery/durable/workflows/hooks.py +211 -0
  55. examples/celery/durable/workflows/idempotency.py +112 -0
  56. examples/celery/durable/workflows/long_running.py +99 -0
  57. examples/celery/durable/workflows/retries.py +101 -0
  58. examples/celery/durable/workflows/schedules.py +209 -0
  59. examples/celery/transient/01_basic_workflow.py +91 -0
  60. examples/celery/transient/02_fault_tolerance.py +257 -0
  61. examples/celery/transient/__init__.py +20 -0
  62. examples/celery/transient/pyworkflow.config.yaml +25 -0
  63. examples/local/__init__.py +1 -0
  64. examples/local/durable/01_basic_workflow.py +94 -0
  65. examples/local/durable/02_file_storage.py +132 -0
  66. examples/local/durable/03_retries.py +169 -0
  67. examples/local/durable/04_long_running.py +119 -0
  68. examples/local/durable/05_event_log.py +145 -0
  69. examples/local/durable/06_idempotency.py +148 -0
  70. examples/local/durable/07_hooks.py +334 -0
  71. examples/local/durable/08_cancellation.py +233 -0
  72. examples/local/durable/09_child_workflows.py +198 -0
  73. examples/local/durable/10_child_workflow_patterns.py +265 -0
  74. examples/local/durable/11_continue_as_new.py +249 -0
  75. examples/local/durable/12_schedules.py +198 -0
  76. examples/local/durable/__init__.py +1 -0
  77. examples/local/transient/01_quick_tasks.py +87 -0
  78. examples/local/transient/02_retries.py +130 -0
  79. examples/local/transient/03_sleep.py +141 -0
  80. examples/local/transient/__init__.py +1 -0
  81. pyworkflow/__init__.py +256 -0
  82. pyworkflow/aws/__init__.py +68 -0
  83. pyworkflow/aws/context.py +234 -0
  84. pyworkflow/aws/handler.py +184 -0
  85. pyworkflow/aws/testing.py +310 -0
  86. pyworkflow/celery/__init__.py +41 -0
  87. pyworkflow/celery/app.py +198 -0
  88. pyworkflow/celery/scheduler.py +315 -0
  89. pyworkflow/celery/tasks.py +1746 -0
  90. pyworkflow/cli/__init__.py +132 -0
  91. pyworkflow/cli/__main__.py +6 -0
  92. pyworkflow/cli/commands/__init__.py +1 -0
  93. pyworkflow/cli/commands/hooks.py +640 -0
  94. pyworkflow/cli/commands/quickstart.py +495 -0
  95. pyworkflow/cli/commands/runs.py +773 -0
  96. pyworkflow/cli/commands/scheduler.py +130 -0
  97. pyworkflow/cli/commands/schedules.py +794 -0
  98. pyworkflow/cli/commands/setup.py +703 -0
  99. pyworkflow/cli/commands/worker.py +413 -0
  100. pyworkflow/cli/commands/workflows.py +1257 -0
  101. pyworkflow/cli/output/__init__.py +1 -0
  102. pyworkflow/cli/output/formatters.py +321 -0
  103. pyworkflow/cli/output/styles.py +121 -0
  104. pyworkflow/cli/utils/__init__.py +1 -0
  105. pyworkflow/cli/utils/async_helpers.py +30 -0
  106. pyworkflow/cli/utils/config.py +130 -0
  107. pyworkflow/cli/utils/config_generator.py +344 -0
  108. pyworkflow/cli/utils/discovery.py +53 -0
  109. pyworkflow/cli/utils/docker_manager.py +651 -0
  110. pyworkflow/cli/utils/interactive.py +364 -0
  111. pyworkflow/cli/utils/storage.py +115 -0
  112. pyworkflow/config.py +329 -0
  113. pyworkflow/context/__init__.py +63 -0
  114. pyworkflow/context/aws.py +230 -0
  115. pyworkflow/context/base.py +416 -0
  116. pyworkflow/context/local.py +930 -0
  117. pyworkflow/context/mock.py +381 -0
  118. pyworkflow/core/__init__.py +0 -0
  119. pyworkflow/core/exceptions.py +353 -0
  120. pyworkflow/core/registry.py +313 -0
  121. pyworkflow/core/scheduled.py +328 -0
  122. pyworkflow/core/step.py +494 -0
  123. pyworkflow/core/workflow.py +294 -0
  124. pyworkflow/discovery.py +248 -0
  125. pyworkflow/engine/__init__.py +0 -0
  126. pyworkflow/engine/events.py +879 -0
  127. pyworkflow/engine/executor.py +682 -0
  128. pyworkflow/engine/replay.py +273 -0
  129. pyworkflow/observability/__init__.py +19 -0
  130. pyworkflow/observability/logging.py +234 -0
  131. pyworkflow/primitives/__init__.py +33 -0
  132. pyworkflow/primitives/child_handle.py +174 -0
  133. pyworkflow/primitives/child_workflow.py +372 -0
  134. pyworkflow/primitives/continue_as_new.py +101 -0
  135. pyworkflow/primitives/define_hook.py +150 -0
  136. pyworkflow/primitives/hooks.py +97 -0
  137. pyworkflow/primitives/resume_hook.py +210 -0
  138. pyworkflow/primitives/schedule.py +545 -0
  139. pyworkflow/primitives/shield.py +96 -0
  140. pyworkflow/primitives/sleep.py +100 -0
  141. pyworkflow/runtime/__init__.py +21 -0
  142. pyworkflow/runtime/base.py +179 -0
  143. pyworkflow/runtime/celery.py +310 -0
  144. pyworkflow/runtime/factory.py +101 -0
  145. pyworkflow/runtime/local.py +706 -0
  146. pyworkflow/scheduler/__init__.py +9 -0
  147. pyworkflow/scheduler/local.py +248 -0
  148. pyworkflow/serialization/__init__.py +0 -0
  149. pyworkflow/serialization/decoder.py +146 -0
  150. pyworkflow/serialization/encoder.py +162 -0
  151. pyworkflow/storage/__init__.py +54 -0
  152. pyworkflow/storage/base.py +612 -0
  153. pyworkflow/storage/config.py +185 -0
  154. pyworkflow/storage/dynamodb.py +1315 -0
  155. pyworkflow/storage/file.py +827 -0
  156. pyworkflow/storage/memory.py +549 -0
  157. pyworkflow/storage/postgres.py +1161 -0
  158. pyworkflow/storage/schemas.py +486 -0
  159. pyworkflow/storage/sqlite.py +1136 -0
  160. pyworkflow/utils/__init__.py +0 -0
  161. pyworkflow/utils/duration.py +177 -0
  162. pyworkflow/utils/schedule.py +391 -0
  163. pyworkflow_engine-0.1.7.dist-info/METADATA +687 -0
  164. pyworkflow_engine-0.1.7.dist-info/RECORD +196 -0
  165. pyworkflow_engine-0.1.7.dist-info/WHEEL +5 -0
  166. pyworkflow_engine-0.1.7.dist-info/entry_points.txt +2 -0
  167. pyworkflow_engine-0.1.7.dist-info/licenses/LICENSE +21 -0
  168. pyworkflow_engine-0.1.7.dist-info/top_level.txt +5 -0
  169. tests/examples/__init__.py +0 -0
  170. tests/integration/__init__.py +0 -0
  171. tests/integration/test_cancellation.py +330 -0
  172. tests/integration/test_child_workflows.py +439 -0
  173. tests/integration/test_continue_as_new.py +428 -0
  174. tests/integration/test_dynamodb_storage.py +1146 -0
  175. tests/integration/test_fault_tolerance.py +369 -0
  176. tests/integration/test_schedule_storage.py +484 -0
  177. tests/unit/__init__.py +0 -0
  178. tests/unit/backends/__init__.py +1 -0
  179. tests/unit/backends/test_dynamodb_storage.py +1554 -0
  180. tests/unit/backends/test_postgres_storage.py +1281 -0
  181. tests/unit/backends/test_sqlite_storage.py +1460 -0
  182. tests/unit/conftest.py +41 -0
  183. tests/unit/test_cancellation.py +364 -0
  184. tests/unit/test_child_workflows.py +680 -0
  185. tests/unit/test_continue_as_new.py +441 -0
  186. tests/unit/test_event_limits.py +316 -0
  187. tests/unit/test_executor.py +320 -0
  188. tests/unit/test_fault_tolerance.py +334 -0
  189. tests/unit/test_hooks.py +495 -0
  190. tests/unit/test_registry.py +261 -0
  191. tests/unit/test_replay.py +420 -0
  192. tests/unit/test_schedule_schemas.py +285 -0
  193. tests/unit/test_schedule_utils.py +286 -0
  194. tests/unit/test_scheduled_workflow.py +274 -0
  195. tests/unit/test_step.py +353 -0
  196. tests/unit/test_workflow.py +243 -0
@@ -0,0 +1,381 @@
1
+ """
2
+ MockContext - Testing context for workflows.
3
+
4
+ Provides a simple context implementation for testing workflows without
5
+ any side effects. Tracks all operations for verification.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import asyncio
11
+ from collections.abc import Awaitable, Callable
12
+ from typing import Any
13
+
14
+ from loguru import logger
15
+ from pydantic import BaseModel
16
+
17
+ from pyworkflow.context.base import StepFunction, WorkflowContext
18
+
19
+
20
+ class MockContext(WorkflowContext):
21
+ """
22
+ Mock context for testing workflows.
23
+
24
+ Features:
25
+ - Executes steps directly (no checkpointing)
26
+ - Skips sleeps by default (configurable)
27
+ - Tracks all operations for verification
28
+ - Supports injecting mock results
29
+
30
+ Example:
31
+ def test_my_workflow():
32
+ ctx = MockContext()
33
+
34
+ result = asyncio.run(my_workflow(ctx, "test_input"))
35
+
36
+ # Verify steps were called
37
+ assert ctx.step_count == 3
38
+ assert "validate_order" in ctx.step_names
39
+
40
+ # Verify sleeps
41
+ assert ctx.sleep_count == 1
42
+ assert ctx.total_sleep_seconds == 300
43
+ """
44
+
45
+ def __init__(
46
+ self,
47
+ run_id: str = "test_run",
48
+ workflow_name: str = "test_workflow",
49
+ skip_sleeps: bool = True,
50
+ mock_results: dict[str, Any] | None = None,
51
+ mock_events: dict[str, Any] | None = None,
52
+ mock_hooks: dict[str, Any] | None = None,
53
+ ) -> None:
54
+ """
55
+ Initialize mock context.
56
+
57
+ Args:
58
+ run_id: Run ID for the test
59
+ workflow_name: Workflow name for the test
60
+ skip_sleeps: If True, sleeps return immediately
61
+ mock_results: Dict of step_name -> result for mocking step results
62
+ mock_events: Dict of event_name -> payload for mocking events
63
+ mock_hooks: Dict of hook_name -> payload for mocking hook results
64
+ """
65
+ super().__init__(run_id=run_id, workflow_name=workflow_name)
66
+ self._skip_sleeps = skip_sleeps
67
+ self._mock_results = mock_results or {}
68
+ self._mock_events = mock_events or {}
69
+ self._mock_hooks = mock_hooks or {}
70
+
71
+ # Tracking
72
+ self._steps: list[dict[str, Any]] = []
73
+ self._sleeps: list[dict[str, Any]] = []
74
+ self._events: list[dict[str, Any]] = []
75
+ self._hooks: list[dict[str, Any]] = []
76
+ self._parallel_calls: list[int] = []
77
+
78
+ # Cancellation state
79
+ self._cancellation_requested: bool = False
80
+ self._cancellation_blocked: bool = False
81
+ self._cancellation_reason: str | None = None
82
+
83
+ # =========================================================================
84
+ # Tracking properties
85
+ # =========================================================================
86
+
87
+ @property
88
+ def steps(self) -> list[dict[str, Any]]:
89
+ """Get all step executions."""
90
+ return self._steps.copy()
91
+
92
+ @property
93
+ def step_count(self) -> int:
94
+ """Get number of steps executed."""
95
+ return len(self._steps)
96
+
97
+ @property
98
+ def step_names(self) -> list[str]:
99
+ """Get names of all executed steps."""
100
+ return [s["name"] for s in self._steps]
101
+
102
+ @property
103
+ def sleeps(self) -> list[dict[str, Any]]:
104
+ """Get all sleep calls."""
105
+ return self._sleeps.copy()
106
+
107
+ @property
108
+ def sleep_count(self) -> int:
109
+ """Get number of sleep calls."""
110
+ return len(self._sleeps)
111
+
112
+ @property
113
+ def total_sleep_seconds(self) -> int:
114
+ """Get total seconds slept."""
115
+ return sum(s["seconds"] for s in self._sleeps)
116
+
117
+ @property
118
+ def events(self) -> list[dict[str, Any]]:
119
+ """Get all event waits."""
120
+ return self._events.copy()
121
+
122
+ @property
123
+ def hooks(self) -> list[dict[str, Any]]:
124
+ """Get all hook waits."""
125
+ return self._hooks.copy()
126
+
127
+ @property
128
+ def hook_count(self) -> int:
129
+ """Get number of hook calls."""
130
+ return len(self._hooks)
131
+
132
+ @property
133
+ def hook_names(self) -> list[str]:
134
+ """Get names of all hooks."""
135
+ return [h["name"] for h in self._hooks]
136
+
137
+ # =========================================================================
138
+ # Step execution
139
+ # =========================================================================
140
+
141
+ async def run(
142
+ self,
143
+ func: StepFunction,
144
+ *args: Any,
145
+ name: str | None = None,
146
+ **kwargs: Any,
147
+ ) -> Any:
148
+ """
149
+ Execute a step function.
150
+
151
+ If a mock result is configured for this step, returns the mock.
152
+ Otherwise executes the function directly.
153
+
154
+ Args:
155
+ func: Step function to execute
156
+ *args: Arguments for the function
157
+ name: Optional step name
158
+ **kwargs: Keyword arguments
159
+
160
+ Returns:
161
+ Step result (real or mocked)
162
+ """
163
+ step_name = name or getattr(func, "__name__", "step")
164
+
165
+ # Track the call
166
+ self._steps.append(
167
+ {
168
+ "name": step_name,
169
+ "func": func,
170
+ "args": args,
171
+ "kwargs": kwargs,
172
+ }
173
+ )
174
+
175
+ logger.debug(f"[mock] Running step: {step_name}")
176
+
177
+ # Check for mock result
178
+ if step_name in self._mock_results:
179
+ logger.debug(f"[mock] Using mock result for: {step_name}")
180
+ return self._mock_results[step_name]
181
+
182
+ # Execute the function
183
+ if asyncio.iscoroutinefunction(func):
184
+ return await func(*args, **kwargs)
185
+ return func(*args, **kwargs)
186
+
187
+ # =========================================================================
188
+ # Sleep
189
+ # =========================================================================
190
+
191
+ async def sleep(self, duration: str | int | float) -> None:
192
+ """
193
+ Sleep for the specified duration.
194
+
195
+ By default, returns immediately. Set skip_sleeps=False to actually sleep.
196
+
197
+ Args:
198
+ duration: Sleep duration
199
+ """
200
+ from pyworkflow.utils.duration import parse_duration
201
+
202
+ duration_seconds = parse_duration(duration) if isinstance(duration, str) else int(duration)
203
+
204
+ # Track the call
205
+ self._sleeps.append(
206
+ {
207
+ "duration": duration,
208
+ "seconds": duration_seconds,
209
+ }
210
+ )
211
+
212
+ logger.debug(f"[mock] Sleep: {duration_seconds}s (skip={self._skip_sleeps})")
213
+
214
+ if not self._skip_sleeps:
215
+ await asyncio.sleep(duration_seconds)
216
+
217
+ # =========================================================================
218
+ # Parallel execution
219
+ # =========================================================================
220
+
221
+ async def parallel(self, *tasks: Any) -> list[Any]:
222
+ """Execute tasks in parallel (tracking the call)."""
223
+ self._parallel_calls.append(len(tasks))
224
+ return list(await asyncio.gather(*tasks))
225
+
226
+ # =========================================================================
227
+ # External events
228
+ # =========================================================================
229
+
230
+ async def wait_for_event(
231
+ self,
232
+ event_name: str,
233
+ timeout: str | int | None = None,
234
+ ) -> Any:
235
+ """
236
+ Wait for an external event.
237
+
238
+ Returns mock event data if configured, otherwise returns a default dict.
239
+
240
+ Args:
241
+ event_name: Event name
242
+ timeout: Optional timeout
243
+
244
+ Returns:
245
+ Mock event payload
246
+ """
247
+ # Track the call
248
+ self._events.append(
249
+ {
250
+ "name": event_name,
251
+ "timeout": timeout,
252
+ }
253
+ )
254
+
255
+ logger.debug(f"[mock] Waiting for event: {event_name}")
256
+
257
+ # Check for mock event
258
+ if event_name in self._mock_events:
259
+ return self._mock_events[event_name]
260
+
261
+ # Return default mock data
262
+ return {"event": event_name, "mock": True}
263
+
264
+ async def hook(
265
+ self,
266
+ name: str,
267
+ timeout: int | None = None,
268
+ on_created: Callable[[str], Awaitable[None]] | None = None,
269
+ payload_schema: type[BaseModel] | None = None,
270
+ ) -> Any:
271
+ """
272
+ Wait for an external event (hook).
273
+
274
+ Returns mock hook payload if configured, otherwise returns a default dict.
275
+
276
+ Args:
277
+ name: Hook name
278
+ timeout: Optional timeout in seconds (tracked but not enforced)
279
+ on_created: Optional callback called with token (for testing)
280
+ payload_schema: Optional Pydantic model (tracked but not enforced)
281
+
282
+ Returns:
283
+ Mock hook payload
284
+ """
285
+ # Generate mock composite token: run_id:hook_name_counter
286
+ self._hook_counter = getattr(self, "_hook_counter", 0) + 1
287
+ hook_id = f"hook_{name}_{self._hook_counter}"
288
+ actual_token = f"{self._run_id}:{hook_id}"
289
+
290
+ # Track the call
291
+ self._hooks.append(
292
+ {
293
+ "name": name,
294
+ "token": actual_token,
295
+ "timeout": timeout,
296
+ }
297
+ )
298
+
299
+ logger.debug(f"[mock] Waiting for hook: {name} (token={actual_token[:20]}...)")
300
+
301
+ # Call on_created callback if provided
302
+ if on_created is not None:
303
+ await on_created(actual_token)
304
+
305
+ # Check for mock hook payload
306
+ if name in self._mock_hooks:
307
+ return self._mock_hooks[name]
308
+
309
+ # Return default mock data
310
+ return {"hook": name, "mock": True}
311
+
312
+ # =========================================================================
313
+ # Cancellation methods
314
+ # =========================================================================
315
+
316
+ def is_cancellation_requested(self) -> bool:
317
+ """Check if cancellation has been requested."""
318
+ return self._cancellation_requested
319
+
320
+ def request_cancellation(self, reason: str | None = None) -> None:
321
+ """Request cancellation of the workflow."""
322
+ self._cancellation_requested = True
323
+ self._cancellation_reason = reason
324
+
325
+ def check_cancellation(self) -> None:
326
+ """Check if cancellation was requested and raise if not blocked."""
327
+ from pyworkflow.core.exceptions import CancellationError
328
+
329
+ if self._cancellation_requested and not self._cancellation_blocked:
330
+ raise CancellationError(
331
+ message="Workflow was cancelled",
332
+ reason=self._cancellation_reason,
333
+ )
334
+
335
+ @property
336
+ def cancellation_blocked(self) -> bool:
337
+ """Check if cancellation is currently blocked (e.g., inside shield)."""
338
+ return self._cancellation_blocked
339
+
340
+ # =========================================================================
341
+ # Utility methods
342
+ # =========================================================================
343
+
344
+ def reset(self) -> None:
345
+ """Reset all tracking data."""
346
+ self._steps.clear()
347
+ self._sleeps.clear()
348
+ self._events.clear()
349
+ self._hooks.clear()
350
+ self._parallel_calls.clear()
351
+
352
+ def assert_step_called(self, step_name: str, times: int | None = None) -> None:
353
+ """
354
+ Assert a step was called.
355
+
356
+ Args:
357
+ step_name: Name of the step
358
+ times: Optional expected call count
359
+ """
360
+ call_count = sum(1 for s in self._steps if s["name"] == step_name)
361
+
362
+ if times is not None:
363
+ assert call_count == times, (
364
+ f"Step '{step_name}' expected {times} calls, got {call_count}"
365
+ )
366
+ else:
367
+ assert call_count > 0, f"Step '{step_name}' was not called"
368
+
369
+ def assert_slept(self, total_seconds: int | None = None) -> None:
370
+ """
371
+ Assert sleep was called.
372
+
373
+ Args:
374
+ total_seconds: Optional expected total sleep time
375
+ """
376
+ assert self.sleep_count > 0, "No sleep calls recorded"
377
+
378
+ if total_seconds is not None:
379
+ assert self.total_sleep_seconds == total_seconds, (
380
+ f"Expected {total_seconds}s total sleep, got {self.total_sleep_seconds}s"
381
+ )
File without changes