pyworkflow-engine 0.1.7__py3-none-any.whl → 0.1.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. pyworkflow/__init__.py +10 -1
  2. pyworkflow/celery/tasks.py +272 -24
  3. pyworkflow/cli/__init__.py +4 -1
  4. pyworkflow/cli/commands/runs.py +4 -4
  5. pyworkflow/cli/commands/setup.py +203 -4
  6. pyworkflow/cli/utils/config_generator.py +76 -3
  7. pyworkflow/cli/utils/docker_manager.py +232 -0
  8. pyworkflow/config.py +94 -17
  9. pyworkflow/context/__init__.py +13 -0
  10. pyworkflow/context/base.py +26 -0
  11. pyworkflow/context/local.py +80 -0
  12. pyworkflow/context/step_context.py +295 -0
  13. pyworkflow/core/registry.py +6 -1
  14. pyworkflow/core/step.py +141 -0
  15. pyworkflow/core/workflow.py +56 -0
  16. pyworkflow/engine/events.py +30 -0
  17. pyworkflow/engine/replay.py +39 -0
  18. pyworkflow/primitives/child_workflow.py +1 -1
  19. pyworkflow/runtime/local.py +1 -1
  20. pyworkflow/storage/__init__.py +14 -0
  21. pyworkflow/storage/base.py +35 -0
  22. pyworkflow/storage/cassandra.py +1747 -0
  23. pyworkflow/storage/config.py +69 -0
  24. pyworkflow/storage/dynamodb.py +31 -2
  25. pyworkflow/storage/file.py +28 -0
  26. pyworkflow/storage/memory.py +18 -0
  27. pyworkflow/storage/mysql.py +1159 -0
  28. pyworkflow/storage/postgres.py +27 -2
  29. pyworkflow/storage/schemas.py +4 -3
  30. pyworkflow/storage/sqlite.py +25 -2
  31. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/METADATA +7 -4
  32. pyworkflow_engine-0.1.10.dist-info/RECORD +91 -0
  33. pyworkflow_engine-0.1.10.dist-info/top_level.txt +1 -0
  34. dashboard/backend/app/__init__.py +0 -1
  35. dashboard/backend/app/config.py +0 -32
  36. dashboard/backend/app/controllers/__init__.py +0 -6
  37. dashboard/backend/app/controllers/run_controller.py +0 -86
  38. dashboard/backend/app/controllers/workflow_controller.py +0 -33
  39. dashboard/backend/app/dependencies/__init__.py +0 -5
  40. dashboard/backend/app/dependencies/storage.py +0 -50
  41. dashboard/backend/app/repositories/__init__.py +0 -6
  42. dashboard/backend/app/repositories/run_repository.py +0 -80
  43. dashboard/backend/app/repositories/workflow_repository.py +0 -27
  44. dashboard/backend/app/rest/__init__.py +0 -8
  45. dashboard/backend/app/rest/v1/__init__.py +0 -12
  46. dashboard/backend/app/rest/v1/health.py +0 -33
  47. dashboard/backend/app/rest/v1/runs.py +0 -133
  48. dashboard/backend/app/rest/v1/workflows.py +0 -41
  49. dashboard/backend/app/schemas/__init__.py +0 -23
  50. dashboard/backend/app/schemas/common.py +0 -16
  51. dashboard/backend/app/schemas/event.py +0 -24
  52. dashboard/backend/app/schemas/hook.py +0 -25
  53. dashboard/backend/app/schemas/run.py +0 -54
  54. dashboard/backend/app/schemas/step.py +0 -28
  55. dashboard/backend/app/schemas/workflow.py +0 -31
  56. dashboard/backend/app/server.py +0 -87
  57. dashboard/backend/app/services/__init__.py +0 -6
  58. dashboard/backend/app/services/run_service.py +0 -240
  59. dashboard/backend/app/services/workflow_service.py +0 -155
  60. dashboard/backend/main.py +0 -18
  61. docs/concepts/cancellation.mdx +0 -362
  62. docs/concepts/continue-as-new.mdx +0 -434
  63. docs/concepts/events.mdx +0 -266
  64. docs/concepts/fault-tolerance.mdx +0 -370
  65. docs/concepts/hooks.mdx +0 -552
  66. docs/concepts/limitations.mdx +0 -167
  67. docs/concepts/schedules.mdx +0 -775
  68. docs/concepts/sleep.mdx +0 -312
  69. docs/concepts/steps.mdx +0 -301
  70. docs/concepts/workflows.mdx +0 -255
  71. docs/guides/cli.mdx +0 -942
  72. docs/guides/configuration.mdx +0 -560
  73. docs/introduction.mdx +0 -155
  74. docs/quickstart.mdx +0 -279
  75. examples/__init__.py +0 -1
  76. examples/celery/__init__.py +0 -1
  77. examples/celery/durable/docker-compose.yml +0 -55
  78. examples/celery/durable/pyworkflow.config.yaml +0 -12
  79. examples/celery/durable/workflows/__init__.py +0 -122
  80. examples/celery/durable/workflows/basic.py +0 -87
  81. examples/celery/durable/workflows/batch_processing.py +0 -102
  82. examples/celery/durable/workflows/cancellation.py +0 -273
  83. examples/celery/durable/workflows/child_workflow_patterns.py +0 -240
  84. examples/celery/durable/workflows/child_workflows.py +0 -202
  85. examples/celery/durable/workflows/continue_as_new.py +0 -260
  86. examples/celery/durable/workflows/fault_tolerance.py +0 -210
  87. examples/celery/durable/workflows/hooks.py +0 -211
  88. examples/celery/durable/workflows/idempotency.py +0 -112
  89. examples/celery/durable/workflows/long_running.py +0 -99
  90. examples/celery/durable/workflows/retries.py +0 -101
  91. examples/celery/durable/workflows/schedules.py +0 -209
  92. examples/celery/transient/01_basic_workflow.py +0 -91
  93. examples/celery/transient/02_fault_tolerance.py +0 -257
  94. examples/celery/transient/__init__.py +0 -20
  95. examples/celery/transient/pyworkflow.config.yaml +0 -25
  96. examples/local/__init__.py +0 -1
  97. examples/local/durable/01_basic_workflow.py +0 -94
  98. examples/local/durable/02_file_storage.py +0 -132
  99. examples/local/durable/03_retries.py +0 -169
  100. examples/local/durable/04_long_running.py +0 -119
  101. examples/local/durable/05_event_log.py +0 -145
  102. examples/local/durable/06_idempotency.py +0 -148
  103. examples/local/durable/07_hooks.py +0 -334
  104. examples/local/durable/08_cancellation.py +0 -233
  105. examples/local/durable/09_child_workflows.py +0 -198
  106. examples/local/durable/10_child_workflow_patterns.py +0 -265
  107. examples/local/durable/11_continue_as_new.py +0 -249
  108. examples/local/durable/12_schedules.py +0 -198
  109. examples/local/durable/__init__.py +0 -1
  110. examples/local/transient/01_quick_tasks.py +0 -87
  111. examples/local/transient/02_retries.py +0 -130
  112. examples/local/transient/03_sleep.py +0 -141
  113. examples/local/transient/__init__.py +0 -1
  114. pyworkflow_engine-0.1.7.dist-info/RECORD +0 -196
  115. pyworkflow_engine-0.1.7.dist-info/top_level.txt +0 -5
  116. tests/examples/__init__.py +0 -0
  117. tests/integration/__init__.py +0 -0
  118. tests/integration/test_cancellation.py +0 -330
  119. tests/integration/test_child_workflows.py +0 -439
  120. tests/integration/test_continue_as_new.py +0 -428
  121. tests/integration/test_dynamodb_storage.py +0 -1146
  122. tests/integration/test_fault_tolerance.py +0 -369
  123. tests/integration/test_schedule_storage.py +0 -484
  124. tests/unit/__init__.py +0 -0
  125. tests/unit/backends/__init__.py +0 -1
  126. tests/unit/backends/test_dynamodb_storage.py +0 -1554
  127. tests/unit/backends/test_postgres_storage.py +0 -1281
  128. tests/unit/backends/test_sqlite_storage.py +0 -1460
  129. tests/unit/conftest.py +0 -41
  130. tests/unit/test_cancellation.py +0 -364
  131. tests/unit/test_child_workflows.py +0 -680
  132. tests/unit/test_continue_as_new.py +0 -441
  133. tests/unit/test_event_limits.py +0 -316
  134. tests/unit/test_executor.py +0 -320
  135. tests/unit/test_fault_tolerance.py +0 -334
  136. tests/unit/test_hooks.py +0 -495
  137. tests/unit/test_registry.py +0 -261
  138. tests/unit/test_replay.py +0 -420
  139. tests/unit/test_schedule_schemas.py +0 -285
  140. tests/unit/test_schedule_utils.py +0 -286
  141. tests/unit/test_scheduled_workflow.py +0 -274
  142. tests/unit/test_step.py +0 -353
  143. tests/unit/test_workflow.py +0 -243
  144. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/WHEEL +0 -0
  145. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/entry_points.txt +0 -0
  146. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/licenses/LICENSE +0 -0
@@ -1,439 +0,0 @@
1
- """
2
- Integration tests for child workflow feature.
3
-
4
- Tests cover:
5
- - Basic child workflow execution
6
- - wait_for_completion=True (waiting for child)
7
- - wait_for_completion=False (fire-and-forget with handle)
8
- - Child workflow failure propagation
9
- - Max nesting depth enforcement
10
- - Parent completion cancels children (TERMINATE policy)
11
- - Event replay with child workflows
12
- """
13
-
14
- import asyncio
15
-
16
- import pytest
17
-
18
- from pyworkflow import (
19
- ChildWorkflowFailedError,
20
- ChildWorkflowHandle,
21
- RunStatus,
22
- configure,
23
- get_context,
24
- get_workflow_events,
25
- get_workflow_run,
26
- reset_config,
27
- start,
28
- start_child_workflow,
29
- step,
30
- workflow,
31
- )
32
- from pyworkflow.engine.events import EventType
33
- from pyworkflow.storage.memory import InMemoryStorageBackend
34
-
35
-
36
- @pytest.fixture(autouse=True)
37
- def setup_storage():
38
- """Setup fresh storage for each test."""
39
- reset_config()
40
- storage = InMemoryStorageBackend()
41
- configure(storage=storage, default_durable=True)
42
- yield storage
43
- reset_config()
44
-
45
-
46
- # --- Define workflows/steps at module level with unique names ---
47
-
48
-
49
- # Test 1: wait_for_completion
50
- @step()
51
- async def child_step_wait(value: int) -> int:
52
- return value * 2
53
-
54
-
55
- @workflow(durable=True)
56
- async def child_workflow_wait(value: int) -> int:
57
- return await child_step_wait(value)
58
-
59
-
60
- @workflow(durable=True)
61
- async def parent_workflow_wait(value: int) -> int:
62
- result = await start_child_workflow(child_workflow_wait, value)
63
- return result
64
-
65
-
66
- # Test 2: fire_and_forget
67
- @step()
68
- async def slow_step_fandf() -> dict:
69
- await asyncio.sleep(0.2)
70
- return {"completed": True}
71
-
72
-
73
- @workflow(durable=True)
74
- async def child_workflow_fandf() -> dict:
75
- return await slow_step_fandf()
76
-
77
-
78
- @workflow(durable=True)
79
- async def parent_workflow_fandf() -> dict:
80
- handle = await start_child_workflow(
81
- child_workflow_fandf,
82
- wait_for_completion=False,
83
- )
84
- # Parent continues immediately
85
- return {"child_run_id": handle.child_run_id}
86
-
87
-
88
- # Test 3: handle_result
89
- @step()
90
- async def process_step_handle(value: int) -> int:
91
- return value + 10
92
-
93
-
94
- @workflow(durable=True)
95
- async def child_workflow_handle(value: int) -> int:
96
- return await process_step_handle(value)
97
-
98
-
99
- @workflow(durable=True)
100
- async def parent_workflow_handle(value: int) -> int:
101
- handle: ChildWorkflowHandle = await start_child_workflow(
102
- child_workflow_handle,
103
- value,
104
- wait_for_completion=False,
105
- )
106
- # Do other work while child runs
107
- await asyncio.sleep(0.1)
108
- # Then get result
109
- result = await handle.result(timeout=5.0)
110
- return result
111
-
112
-
113
- # Test 4: failure propagation
114
- @step(max_retries=0) # No retries so failure propagates immediately
115
- async def failing_step_prop() -> None:
116
- raise ValueError("Child step failed!")
117
-
118
-
119
- @workflow(durable=True)
120
- async def failing_child_prop() -> dict:
121
- await failing_step_prop()
122
- return {"should": "not reach"}
123
-
124
-
125
- @workflow(durable=True)
126
- async def parent_workflow_failure() -> dict:
127
- try:
128
- await start_child_workflow(failing_child_prop)
129
- return {"status": "success"}
130
- except ChildWorkflowFailedError as e:
131
- return {
132
- "status": "child_failed",
133
- "error": e.error,
134
- "child_run_id": e.child_run_id,
135
- }
136
-
137
-
138
- # Test 5: nesting depth
139
- @step()
140
- async def simple_step_depth() -> dict:
141
- return {"done": True}
142
-
143
-
144
- @workflow(durable=True)
145
- async def level_2_workflow_depth() -> dict:
146
- ctx = get_context()
147
- depth = await ctx.storage.get_nesting_depth(ctx.run_id)
148
- return {"depth": depth}
149
-
150
-
151
- @workflow(durable=True)
152
- async def level_1_workflow_depth() -> dict:
153
- result = await start_child_workflow(level_2_workflow_depth)
154
- return {"child_result": result}
155
-
156
-
157
- @workflow(durable=True)
158
- async def root_workflow_depth() -> dict:
159
- result = await start_child_workflow(level_1_workflow_depth)
160
- return {"child_result": result}
161
-
162
-
163
- # Test 6: events started
164
- @step()
165
- async def child_step_events_started() -> dict:
166
- return {"done": True}
167
-
168
-
169
- @workflow(durable=True)
170
- async def child_workflow_events_started() -> dict:
171
- return await child_step_events_started()
172
-
173
-
174
- @workflow(durable=True)
175
- async def parent_workflow_events_started() -> dict:
176
- return await start_child_workflow(child_workflow_events_started)
177
-
178
-
179
- # Test 7: events completed
180
- @step()
181
- async def child_step_events_completed() -> dict:
182
- return {"done": True}
183
-
184
-
185
- @workflow(durable=True)
186
- async def child_workflow_events_completed() -> dict:
187
- return await child_step_events_completed()
188
-
189
-
190
- @workflow(durable=True)
191
- async def parent_workflow_events_completed() -> dict:
192
- return await start_child_workflow(child_workflow_events_completed)
193
-
194
-
195
- # Test 8: parent_run_id
196
- @step()
197
- async def child_step_parent_id() -> dict:
198
- return {"done": True}
199
-
200
-
201
- @workflow(durable=True)
202
- async def child_workflow_parent_id() -> dict:
203
- return await child_step_parent_id()
204
-
205
-
206
- @workflow(durable=True)
207
- async def parent_workflow_parent_id() -> dict:
208
- return await start_child_workflow(child_workflow_parent_id)
209
-
210
-
211
- # Test 9: multiple children
212
- @step()
213
- async def process_step_multi(item: str) -> dict:
214
- return {"item": item, "processed": True}
215
-
216
-
217
- @workflow(durable=True)
218
- async def item_workflow_multi(item: str) -> dict:
219
- return await process_step_multi(item)
220
-
221
-
222
- @workflow(durable=True)
223
- async def parent_workflow_multi() -> dict:
224
- # Start 3 children
225
- handles = []
226
- for i in range(3):
227
- handle = await start_child_workflow(
228
- item_workflow_multi,
229
- f"item-{i}",
230
- wait_for_completion=False,
231
- )
232
- handles.append(handle)
233
-
234
- # Wait for all
235
- results = []
236
- for handle in handles:
237
- result = await handle.result(timeout=5.0)
238
- results.append(result)
239
-
240
- return {"results": results}
241
-
242
-
243
- # Test 10: outside context
244
- @workflow(durable=True)
245
- async def some_workflow_outside() -> dict:
246
- return {"done": True}
247
-
248
-
249
- # Test 11: unregistered workflow
250
- @workflow(durable=True)
251
- async def parent_workflow_unreg() -> dict:
252
- # This function is not decorated with @workflow
253
- async def not_a_workflow() -> dict:
254
- return {"done": True}
255
-
256
- return await start_child_workflow(not_a_workflow)
257
-
258
-
259
- class TestBasicChildWorkflow:
260
- """Test basic child workflow execution."""
261
-
262
- @pytest.mark.asyncio
263
- async def test_start_child_workflow_wait_for_completion(self, setup_storage):
264
- """Test starting a child workflow and waiting for completion."""
265
- storage = setup_storage
266
-
267
- run_id = await start(parent_workflow_wait, 21)
268
-
269
- # Wait for completion
270
- await asyncio.sleep(0.5)
271
-
272
- run = await get_workflow_run(run_id, storage=storage)
273
- assert run.status == RunStatus.COMPLETED
274
- # Result is serialized
275
- assert "42" in run.result
276
-
277
- @pytest.mark.asyncio
278
- async def test_start_child_workflow_fire_and_forget(self, setup_storage):
279
- """Test starting a child workflow with fire-and-forget pattern."""
280
- storage = setup_storage
281
-
282
- run_id = await start(parent_workflow_fandf)
283
-
284
- # Parent should complete quickly (fire-and-forget)
285
- await asyncio.sleep(0.1)
286
-
287
- run = await get_workflow_run(run_id, storage=storage)
288
- assert run.status == RunStatus.COMPLETED
289
-
290
- # Wait for child to complete
291
- await asyncio.sleep(0.5)
292
-
293
- # Check children
294
- children = await storage.get_children(run_id)
295
- assert len(children) == 1
296
- assert children[0].status == RunStatus.COMPLETED
297
-
298
- @pytest.mark.asyncio
299
- async def test_child_workflow_handle_result(self, setup_storage):
300
- """Test getting result from ChildWorkflowHandle."""
301
- storage = setup_storage
302
-
303
- run_id = await start(parent_workflow_handle, 32)
304
-
305
- await asyncio.sleep(0.5)
306
-
307
- run = await get_workflow_run(run_id, storage=storage)
308
- assert run.status == RunStatus.COMPLETED
309
- assert "42" in run.result
310
-
311
-
312
- class TestChildWorkflowFailure:
313
- """Test child workflow failure handling."""
314
-
315
- @pytest.mark.asyncio
316
- async def test_child_workflow_failure_propagates(self, setup_storage):
317
- """Test that child workflow failure is propagated to parent."""
318
- storage = setup_storage
319
-
320
- run_id = await start(parent_workflow_failure)
321
-
322
- await asyncio.sleep(0.5)
323
-
324
- run = await get_workflow_run(run_id, storage=storage)
325
- assert run.status == RunStatus.COMPLETED
326
- assert "child_failed" in run.result
327
-
328
-
329
- class TestNestingDepth:
330
- """Test nesting depth enforcement."""
331
-
332
- @pytest.mark.asyncio
333
- async def test_nesting_depth_tracked(self, setup_storage):
334
- """Test that nesting depth is tracked correctly."""
335
- storage = setup_storage
336
-
337
- run_id = await start(root_workflow_depth)
338
-
339
- await asyncio.sleep(0.5)
340
-
341
- # Check children depths
342
- children = await storage.get_children(run_id)
343
- assert len(children) == 1
344
- assert children[0].nesting_depth == 1
345
-
346
-
347
- class TestChildWorkflowEvents:
348
- """Test child workflow events."""
349
-
350
- @pytest.mark.asyncio
351
- async def test_child_workflow_started_event_recorded(self, setup_storage):
352
- """Test that CHILD_WORKFLOW_STARTED event is recorded."""
353
- storage = setup_storage
354
-
355
- run_id = await start(parent_workflow_events_started)
356
-
357
- await asyncio.sleep(0.5)
358
-
359
- events = await get_workflow_events(run_id, storage=storage)
360
- event_types = [e.type for e in events]
361
-
362
- assert EventType.CHILD_WORKFLOW_STARTED in event_types
363
-
364
- @pytest.mark.asyncio
365
- async def test_child_workflow_completed_event_recorded(self, setup_storage):
366
- """Test that CHILD_WORKFLOW_COMPLETED event is recorded."""
367
- storage = setup_storage
368
-
369
- run_id = await start(parent_workflow_events_completed)
370
-
371
- await asyncio.sleep(0.5)
372
-
373
- events = await get_workflow_events(run_id, storage=storage)
374
- event_types = [e.type for e in events]
375
-
376
- assert EventType.CHILD_WORKFLOW_COMPLETED in event_types
377
-
378
-
379
- class TestParentChildLifecycle:
380
- """Test parent-child lifecycle management."""
381
-
382
- @pytest.mark.asyncio
383
- async def test_children_have_parent_run_id(self, setup_storage):
384
- """Test that children have parent_run_id set."""
385
- storage = setup_storage
386
-
387
- run_id = await start(parent_workflow_parent_id)
388
-
389
- await asyncio.sleep(0.5)
390
-
391
- children = await storage.get_children(run_id)
392
- assert len(children) == 1
393
- assert children[0].parent_run_id == run_id
394
-
395
- @pytest.mark.asyncio
396
- async def test_multiple_children(self, setup_storage):
397
- """Test parent with multiple children."""
398
- storage = setup_storage
399
-
400
- run_id = await start(parent_workflow_multi)
401
-
402
- await asyncio.sleep(1.0)
403
-
404
- children = await storage.get_children(run_id)
405
- assert len(children) == 3
406
- assert all(c.status == RunStatus.COMPLETED for c in children)
407
-
408
-
409
- class TestChildWorkflowOutsideContext:
410
- """Test start_child_workflow outside workflow context."""
411
-
412
- @pytest.mark.asyncio
413
- async def test_start_child_workflow_outside_context_raises(self, setup_storage):
414
- """Test that start_child_workflow raises outside workflow context."""
415
- with pytest.raises(RuntimeError, match="workflow context"):
416
- await start_child_workflow(some_workflow_outside)
417
-
418
-
419
- class TestChildWorkflowWithUnregisteredWorkflow:
420
- """Test start_child_workflow with unregistered workflow."""
421
-
422
- @pytest.mark.asyncio
423
- async def test_start_child_workflow_unregistered_raises(self, setup_storage):
424
- """Test that start_child_workflow raises for unregistered workflow."""
425
- # In local runtime, exceptions are raised synchronously
426
- # Catch the exception and verify the workflow was marked as failed
427
- run_id = None
428
- try:
429
- run_id = await start(parent_workflow_unreg)
430
- except ValueError as e:
431
- # Expected: ValueError for unregistered workflow
432
- assert "not registered" in str(e).lower()
433
-
434
- # The workflow should have been marked as failed in storage
435
- # Find the workflow run that was created
436
- if run_id:
437
- run = await get_workflow_run(run_id, storage=setup_storage)
438
- assert run.status == RunStatus.FAILED
439
- assert "not registered" in run.error.lower()