pyworkflow-engine 0.1.7__py3-none-any.whl → 0.1.9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. pyworkflow/__init__.py +10 -1
  2. pyworkflow/celery/tasks.py +272 -24
  3. pyworkflow/cli/__init__.py +4 -1
  4. pyworkflow/cli/commands/runs.py +4 -4
  5. pyworkflow/cli/commands/setup.py +203 -4
  6. pyworkflow/cli/utils/config_generator.py +76 -3
  7. pyworkflow/cli/utils/docker_manager.py +232 -0
  8. pyworkflow/context/__init__.py +13 -0
  9. pyworkflow/context/base.py +26 -0
  10. pyworkflow/context/local.py +80 -0
  11. pyworkflow/context/step_context.py +295 -0
  12. pyworkflow/core/registry.py +6 -1
  13. pyworkflow/core/step.py +141 -0
  14. pyworkflow/core/workflow.py +56 -0
  15. pyworkflow/engine/events.py +30 -0
  16. pyworkflow/engine/replay.py +39 -0
  17. pyworkflow/primitives/child_workflow.py +1 -1
  18. pyworkflow/runtime/local.py +1 -1
  19. pyworkflow/storage/__init__.py +14 -0
  20. pyworkflow/storage/base.py +35 -0
  21. pyworkflow/storage/cassandra.py +1747 -0
  22. pyworkflow/storage/config.py +69 -0
  23. pyworkflow/storage/dynamodb.py +31 -2
  24. pyworkflow/storage/file.py +28 -0
  25. pyworkflow/storage/memory.py +18 -0
  26. pyworkflow/storage/mysql.py +1159 -0
  27. pyworkflow/storage/postgres.py +27 -2
  28. pyworkflow/storage/schemas.py +4 -3
  29. pyworkflow/storage/sqlite.py +25 -2
  30. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/METADATA +7 -4
  31. pyworkflow_engine-0.1.9.dist-info/RECORD +91 -0
  32. pyworkflow_engine-0.1.9.dist-info/top_level.txt +1 -0
  33. dashboard/backend/app/__init__.py +0 -1
  34. dashboard/backend/app/config.py +0 -32
  35. dashboard/backend/app/controllers/__init__.py +0 -6
  36. dashboard/backend/app/controllers/run_controller.py +0 -86
  37. dashboard/backend/app/controllers/workflow_controller.py +0 -33
  38. dashboard/backend/app/dependencies/__init__.py +0 -5
  39. dashboard/backend/app/dependencies/storage.py +0 -50
  40. dashboard/backend/app/repositories/__init__.py +0 -6
  41. dashboard/backend/app/repositories/run_repository.py +0 -80
  42. dashboard/backend/app/repositories/workflow_repository.py +0 -27
  43. dashboard/backend/app/rest/__init__.py +0 -8
  44. dashboard/backend/app/rest/v1/__init__.py +0 -12
  45. dashboard/backend/app/rest/v1/health.py +0 -33
  46. dashboard/backend/app/rest/v1/runs.py +0 -133
  47. dashboard/backend/app/rest/v1/workflows.py +0 -41
  48. dashboard/backend/app/schemas/__init__.py +0 -23
  49. dashboard/backend/app/schemas/common.py +0 -16
  50. dashboard/backend/app/schemas/event.py +0 -24
  51. dashboard/backend/app/schemas/hook.py +0 -25
  52. dashboard/backend/app/schemas/run.py +0 -54
  53. dashboard/backend/app/schemas/step.py +0 -28
  54. dashboard/backend/app/schemas/workflow.py +0 -31
  55. dashboard/backend/app/server.py +0 -87
  56. dashboard/backend/app/services/__init__.py +0 -6
  57. dashboard/backend/app/services/run_service.py +0 -240
  58. dashboard/backend/app/services/workflow_service.py +0 -155
  59. dashboard/backend/main.py +0 -18
  60. docs/concepts/cancellation.mdx +0 -362
  61. docs/concepts/continue-as-new.mdx +0 -434
  62. docs/concepts/events.mdx +0 -266
  63. docs/concepts/fault-tolerance.mdx +0 -370
  64. docs/concepts/hooks.mdx +0 -552
  65. docs/concepts/limitations.mdx +0 -167
  66. docs/concepts/schedules.mdx +0 -775
  67. docs/concepts/sleep.mdx +0 -312
  68. docs/concepts/steps.mdx +0 -301
  69. docs/concepts/workflows.mdx +0 -255
  70. docs/guides/cli.mdx +0 -942
  71. docs/guides/configuration.mdx +0 -560
  72. docs/introduction.mdx +0 -155
  73. docs/quickstart.mdx +0 -279
  74. examples/__init__.py +0 -1
  75. examples/celery/__init__.py +0 -1
  76. examples/celery/durable/docker-compose.yml +0 -55
  77. examples/celery/durable/pyworkflow.config.yaml +0 -12
  78. examples/celery/durable/workflows/__init__.py +0 -122
  79. examples/celery/durable/workflows/basic.py +0 -87
  80. examples/celery/durable/workflows/batch_processing.py +0 -102
  81. examples/celery/durable/workflows/cancellation.py +0 -273
  82. examples/celery/durable/workflows/child_workflow_patterns.py +0 -240
  83. examples/celery/durable/workflows/child_workflows.py +0 -202
  84. examples/celery/durable/workflows/continue_as_new.py +0 -260
  85. examples/celery/durable/workflows/fault_tolerance.py +0 -210
  86. examples/celery/durable/workflows/hooks.py +0 -211
  87. examples/celery/durable/workflows/idempotency.py +0 -112
  88. examples/celery/durable/workflows/long_running.py +0 -99
  89. examples/celery/durable/workflows/retries.py +0 -101
  90. examples/celery/durable/workflows/schedules.py +0 -209
  91. examples/celery/transient/01_basic_workflow.py +0 -91
  92. examples/celery/transient/02_fault_tolerance.py +0 -257
  93. examples/celery/transient/__init__.py +0 -20
  94. examples/celery/transient/pyworkflow.config.yaml +0 -25
  95. examples/local/__init__.py +0 -1
  96. examples/local/durable/01_basic_workflow.py +0 -94
  97. examples/local/durable/02_file_storage.py +0 -132
  98. examples/local/durable/03_retries.py +0 -169
  99. examples/local/durable/04_long_running.py +0 -119
  100. examples/local/durable/05_event_log.py +0 -145
  101. examples/local/durable/06_idempotency.py +0 -148
  102. examples/local/durable/07_hooks.py +0 -334
  103. examples/local/durable/08_cancellation.py +0 -233
  104. examples/local/durable/09_child_workflows.py +0 -198
  105. examples/local/durable/10_child_workflow_patterns.py +0 -265
  106. examples/local/durable/11_continue_as_new.py +0 -249
  107. examples/local/durable/12_schedules.py +0 -198
  108. examples/local/durable/__init__.py +0 -1
  109. examples/local/transient/01_quick_tasks.py +0 -87
  110. examples/local/transient/02_retries.py +0 -130
  111. examples/local/transient/03_sleep.py +0 -141
  112. examples/local/transient/__init__.py +0 -1
  113. pyworkflow_engine-0.1.7.dist-info/RECORD +0 -196
  114. pyworkflow_engine-0.1.7.dist-info/top_level.txt +0 -5
  115. tests/examples/__init__.py +0 -0
  116. tests/integration/__init__.py +0 -0
  117. tests/integration/test_cancellation.py +0 -330
  118. tests/integration/test_child_workflows.py +0 -439
  119. tests/integration/test_continue_as_new.py +0 -428
  120. tests/integration/test_dynamodb_storage.py +0 -1146
  121. tests/integration/test_fault_tolerance.py +0 -369
  122. tests/integration/test_schedule_storage.py +0 -484
  123. tests/unit/__init__.py +0 -0
  124. tests/unit/backends/__init__.py +0 -1
  125. tests/unit/backends/test_dynamodb_storage.py +0 -1554
  126. tests/unit/backends/test_postgres_storage.py +0 -1281
  127. tests/unit/backends/test_sqlite_storage.py +0 -1460
  128. tests/unit/conftest.py +0 -41
  129. tests/unit/test_cancellation.py +0 -364
  130. tests/unit/test_child_workflows.py +0 -680
  131. tests/unit/test_continue_as_new.py +0 -441
  132. tests/unit/test_event_limits.py +0 -316
  133. tests/unit/test_executor.py +0 -320
  134. tests/unit/test_fault_tolerance.py +0 -334
  135. tests/unit/test_hooks.py +0 -495
  136. tests/unit/test_registry.py +0 -261
  137. tests/unit/test_replay.py +0 -420
  138. tests/unit/test_schedule_schemas.py +0 -285
  139. tests/unit/test_schedule_utils.py +0 -286
  140. tests/unit/test_scheduled_workflow.py +0 -274
  141. tests/unit/test_step.py +0 -353
  142. tests/unit/test_workflow.py +0 -243
  143. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/WHEEL +0 -0
  144. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/entry_points.txt +0 -0
  145. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/licenses/LICENSE +0 -0
@@ -1,330 +0,0 @@
1
- """
2
- Integration tests for cancellation feature.
3
-
4
- Tests cover:
5
- - cancel_workflow() function
6
- - Cancellation during step execution
7
- - Cancellation during sleep
8
- - Cancellation during hook wait
9
- - Shield prevents cancellation
10
- - Workflow catches CancellationError for cleanup
11
- """
12
-
13
- import asyncio
14
-
15
- import pytest
16
-
17
- from pyworkflow import (
18
- CancellationError,
19
- RunStatus,
20
- cancel_workflow,
21
- shield,
22
- step,
23
- )
24
- from pyworkflow.engine.events import EventType
25
- from pyworkflow.storage.memory import InMemoryStorageBackend
26
- from pyworkflow.storage.schemas import WorkflowRun
27
-
28
-
29
- class TestCancelWorkflowFunction:
30
- """Test cancel_workflow() function."""
31
-
32
- @pytest.mark.asyncio
33
- async def test_cancel_running_workflow(self):
34
- """Test cancelling a running workflow."""
35
- storage = InMemoryStorageBackend()
36
-
37
- # Create a workflow run record
38
- run = WorkflowRun(
39
- run_id="run_123",
40
- workflow_name="test_workflow",
41
- status=RunStatus.RUNNING,
42
- )
43
- await storage.create_run(run)
44
-
45
- # Cancel the workflow
46
- result = await cancel_workflow(
47
- run_id="run_123",
48
- reason="User cancelled",
49
- storage=storage,
50
- )
51
-
52
- assert result is True
53
-
54
- # Check cancellation flag was set
55
- assert await storage.check_cancellation_flag("run_123") is True
56
-
57
- # Check cancellation event was recorded
58
- events = await storage.get_events("run_123")
59
- cancellation_events = [e for e in events if e.type == EventType.CANCELLATION_REQUESTED]
60
- assert len(cancellation_events) == 1
61
- assert cancellation_events[0].data["reason"] == "User cancelled"
62
-
63
- @pytest.mark.asyncio
64
- async def test_cancel_suspended_workflow(self):
65
- """Test cancelling a suspended workflow marks it as cancelled."""
66
- storage = InMemoryStorageBackend()
67
-
68
- # Create a suspended workflow
69
- run = WorkflowRun(
70
- run_id="run_456",
71
- workflow_name="test_workflow",
72
- status=RunStatus.SUSPENDED,
73
- )
74
- await storage.create_run(run)
75
-
76
- # Cancel the workflow
77
- result = await cancel_workflow(
78
- run_id="run_456",
79
- storage=storage,
80
- )
81
-
82
- assert result is True
83
-
84
- # Suspended workflows should be marked as cancelled immediately
85
- updated_run = await storage.get_run("run_456")
86
- assert updated_run.status == RunStatus.CANCELLED
87
-
88
- @pytest.mark.asyncio
89
- async def test_cancel_completed_workflow_returns_false(self):
90
- """Test cancelling an already completed workflow returns False."""
91
- storage = InMemoryStorageBackend()
92
-
93
- # Create a completed workflow
94
- run = WorkflowRun(
95
- run_id="run_789",
96
- workflow_name="test_workflow",
97
- status=RunStatus.COMPLETED,
98
- )
99
- await storage.create_run(run)
100
-
101
- # Try to cancel
102
- result = await cancel_workflow(
103
- run_id="run_789",
104
- storage=storage,
105
- )
106
-
107
- assert result is False
108
-
109
- @pytest.mark.asyncio
110
- async def test_cancel_failed_workflow_returns_false(self):
111
- """Test cancelling an already failed workflow returns False."""
112
- storage = InMemoryStorageBackend()
113
-
114
- # Create a failed workflow
115
- run = WorkflowRun(
116
- run_id="run_abc",
117
- workflow_name="test_workflow",
118
- status=RunStatus.FAILED,
119
- )
120
- await storage.create_run(run)
121
-
122
- # Try to cancel
123
- result = await cancel_workflow(
124
- run_id="run_abc",
125
- storage=storage,
126
- )
127
-
128
- assert result is False
129
-
130
- @pytest.mark.asyncio
131
- async def test_cancel_already_cancelled_workflow_returns_false(self):
132
- """Test cancelling an already cancelled workflow returns False."""
133
- storage = InMemoryStorageBackend()
134
-
135
- # Create a cancelled workflow
136
- run = WorkflowRun(
137
- run_id="run_def",
138
- workflow_name="test_workflow",
139
- status=RunStatus.CANCELLED,
140
- )
141
- await storage.create_run(run)
142
-
143
- # Try to cancel again
144
- result = await cancel_workflow(
145
- run_id="run_def",
146
- storage=storage,
147
- )
148
-
149
- assert result is False
150
-
151
- @pytest.mark.asyncio
152
- async def test_cancel_nonexistent_workflow(self):
153
- """Test cancelling a non-existent workflow raises error."""
154
- from pyworkflow import WorkflowNotFoundError
155
-
156
- storage = InMemoryStorageBackend()
157
-
158
- with pytest.raises(WorkflowNotFoundError):
159
- await cancel_workflow(
160
- run_id="nonexistent",
161
- storage=storage,
162
- )
163
-
164
-
165
- class TestCancellationCheckPoints:
166
- """Test cancellation at various check points."""
167
-
168
- @pytest.mark.asyncio
169
- async def test_step_checks_cancellation(self):
170
- """Test that step execution checks for cancellation."""
171
- from pyworkflow.context import LocalContext, reset_context, set_context
172
-
173
- @step()
174
- async def my_step():
175
- return "result"
176
-
177
- # Create context with cancellation requested
178
- ctx = LocalContext(
179
- run_id="test_run",
180
- workflow_name="test_workflow",
181
- storage=None,
182
- durable=False,
183
- )
184
- ctx.request_cancellation(reason="Test")
185
- token = set_context(ctx)
186
-
187
- try:
188
- with pytest.raises(CancellationError):
189
- await my_step()
190
- finally:
191
- reset_context(token)
192
-
193
-
194
- class TestShieldIntegration:
195
- """Test shield() integration with workflow execution."""
196
-
197
- @pytest.mark.asyncio
198
- async def test_shield_allows_cleanup(self):
199
- """Test shield() allows cleanup operations to complete."""
200
- from pyworkflow.context import LocalContext, reset_context, set_context
201
-
202
- cleanup_completed = False
203
-
204
- async def cleanup():
205
- nonlocal cleanup_completed
206
- await asyncio.sleep(0.01) # Simulate cleanup work
207
- cleanup_completed = True
208
-
209
- ctx = LocalContext(
210
- run_id="test_run",
211
- workflow_name="test_workflow",
212
- storage=None,
213
- durable=False,
214
- )
215
- ctx.request_cancellation()
216
- token = set_context(ctx)
217
-
218
- try:
219
- async with shield():
220
- await cleanup()
221
-
222
- assert cleanup_completed is True
223
- finally:
224
- reset_context(token)
225
-
226
-
227
- class TestFileCancellationFlags:
228
- """Test file storage backend cancellation flags."""
229
-
230
- @pytest.mark.asyncio
231
- async def test_file_storage_cancellation_flags(self, tmp_path):
232
- """Test FileStorageBackend cancellation flag methods."""
233
- from pyworkflow.storage.file import FileStorageBackend
234
-
235
- storage = FileStorageBackend(base_path=str(tmp_path / "workflow_data"))
236
-
237
- # Initially not set
238
- assert await storage.check_cancellation_flag("run_123") is False
239
-
240
- # Set the flag
241
- await storage.set_cancellation_flag("run_123")
242
- assert await storage.check_cancellation_flag("run_123") is True
243
-
244
- # Clear the flag
245
- await storage.clear_cancellation_flag("run_123")
246
- assert await storage.check_cancellation_flag("run_123") is False
247
-
248
- @pytest.mark.asyncio
249
- async def test_file_storage_clear_nonexistent_flag(self, tmp_path):
250
- """Test clearing a non-existent flag does not raise."""
251
- from pyworkflow.storage.file import FileStorageBackend
252
-
253
- storage = FileStorageBackend(base_path=str(tmp_path / "workflow_data"))
254
-
255
- # Should not raise
256
- await storage.clear_cancellation_flag("run_nonexistent")
257
-
258
-
259
- class TestEventReplayCancellation:
260
- """Test cancellation state restoration during event replay."""
261
-
262
- @pytest.mark.asyncio
263
- async def test_replay_restores_cancellation_state(self):
264
- """Test that CANCELLATION_REQUESTED event sets context state during replay."""
265
- from pyworkflow.context import LocalContext
266
- from pyworkflow.engine.events import create_cancellation_requested_event
267
-
268
- storage = InMemoryStorageBackend()
269
-
270
- # Create run
271
- run = WorkflowRun(
272
- run_id="run_123",
273
- workflow_name="test_workflow",
274
- status=RunStatus.RUNNING,
275
- )
276
- await storage.create_run(run)
277
-
278
- # Record cancellation event
279
- event = create_cancellation_requested_event(
280
- run_id="run_123",
281
- reason="User cancelled",
282
- )
283
- await storage.record_event(event)
284
-
285
- # Get events
286
- events = await storage.get_events("run_123")
287
-
288
- # Create context with event log (should replay events)
289
- ctx = LocalContext(
290
- run_id="run_123",
291
- workflow_name="test_workflow",
292
- storage=storage,
293
- event_log=events,
294
- durable=True,
295
- )
296
-
297
- # Context should have cancellation requested from replay
298
- assert ctx.is_cancellation_requested() is True
299
-
300
-
301
- class TestCancellationErrorHandling:
302
- """Test CancellationError handling in workflows."""
303
-
304
- @pytest.mark.asyncio
305
- async def test_workflow_can_catch_cancellation_for_cleanup(self):
306
- """Test that workflows can catch CancellationError for cleanup."""
307
- from pyworkflow.context import LocalContext
308
-
309
- cleanup_called = False
310
-
311
- async def workflow_with_cleanup():
312
- nonlocal cleanup_called
313
- try:
314
- # Simulate work that would check cancellation
315
- ctx = LocalContext(
316
- run_id="test",
317
- workflow_name="test",
318
- storage=None,
319
- durable=False,
320
- )
321
- ctx.request_cancellation()
322
- ctx.check_cancellation()
323
- except CancellationError:
324
- cleanup_called = True
325
- raise
326
-
327
- with pytest.raises(CancellationError):
328
- await workflow_with_cleanup()
329
-
330
- assert cleanup_called is True