pyworkflow-engine 0.1.7__py3-none-any.whl → 0.1.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. pyworkflow/__init__.py +10 -1
  2. pyworkflow/celery/tasks.py +272 -24
  3. pyworkflow/cli/__init__.py +4 -1
  4. pyworkflow/cli/commands/runs.py +4 -4
  5. pyworkflow/cli/commands/setup.py +203 -4
  6. pyworkflow/cli/utils/config_generator.py +76 -3
  7. pyworkflow/cli/utils/docker_manager.py +232 -0
  8. pyworkflow/config.py +94 -17
  9. pyworkflow/context/__init__.py +13 -0
  10. pyworkflow/context/base.py +26 -0
  11. pyworkflow/context/local.py +80 -0
  12. pyworkflow/context/step_context.py +295 -0
  13. pyworkflow/core/registry.py +6 -1
  14. pyworkflow/core/step.py +141 -0
  15. pyworkflow/core/workflow.py +56 -0
  16. pyworkflow/engine/events.py +30 -0
  17. pyworkflow/engine/replay.py +39 -0
  18. pyworkflow/primitives/child_workflow.py +1 -1
  19. pyworkflow/runtime/local.py +1 -1
  20. pyworkflow/storage/__init__.py +14 -0
  21. pyworkflow/storage/base.py +35 -0
  22. pyworkflow/storage/cassandra.py +1747 -0
  23. pyworkflow/storage/config.py +69 -0
  24. pyworkflow/storage/dynamodb.py +31 -2
  25. pyworkflow/storage/file.py +28 -0
  26. pyworkflow/storage/memory.py +18 -0
  27. pyworkflow/storage/mysql.py +1159 -0
  28. pyworkflow/storage/postgres.py +27 -2
  29. pyworkflow/storage/schemas.py +4 -3
  30. pyworkflow/storage/sqlite.py +25 -2
  31. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/METADATA +7 -4
  32. pyworkflow_engine-0.1.10.dist-info/RECORD +91 -0
  33. pyworkflow_engine-0.1.10.dist-info/top_level.txt +1 -0
  34. dashboard/backend/app/__init__.py +0 -1
  35. dashboard/backend/app/config.py +0 -32
  36. dashboard/backend/app/controllers/__init__.py +0 -6
  37. dashboard/backend/app/controllers/run_controller.py +0 -86
  38. dashboard/backend/app/controllers/workflow_controller.py +0 -33
  39. dashboard/backend/app/dependencies/__init__.py +0 -5
  40. dashboard/backend/app/dependencies/storage.py +0 -50
  41. dashboard/backend/app/repositories/__init__.py +0 -6
  42. dashboard/backend/app/repositories/run_repository.py +0 -80
  43. dashboard/backend/app/repositories/workflow_repository.py +0 -27
  44. dashboard/backend/app/rest/__init__.py +0 -8
  45. dashboard/backend/app/rest/v1/__init__.py +0 -12
  46. dashboard/backend/app/rest/v1/health.py +0 -33
  47. dashboard/backend/app/rest/v1/runs.py +0 -133
  48. dashboard/backend/app/rest/v1/workflows.py +0 -41
  49. dashboard/backend/app/schemas/__init__.py +0 -23
  50. dashboard/backend/app/schemas/common.py +0 -16
  51. dashboard/backend/app/schemas/event.py +0 -24
  52. dashboard/backend/app/schemas/hook.py +0 -25
  53. dashboard/backend/app/schemas/run.py +0 -54
  54. dashboard/backend/app/schemas/step.py +0 -28
  55. dashboard/backend/app/schemas/workflow.py +0 -31
  56. dashboard/backend/app/server.py +0 -87
  57. dashboard/backend/app/services/__init__.py +0 -6
  58. dashboard/backend/app/services/run_service.py +0 -240
  59. dashboard/backend/app/services/workflow_service.py +0 -155
  60. dashboard/backend/main.py +0 -18
  61. docs/concepts/cancellation.mdx +0 -362
  62. docs/concepts/continue-as-new.mdx +0 -434
  63. docs/concepts/events.mdx +0 -266
  64. docs/concepts/fault-tolerance.mdx +0 -370
  65. docs/concepts/hooks.mdx +0 -552
  66. docs/concepts/limitations.mdx +0 -167
  67. docs/concepts/schedules.mdx +0 -775
  68. docs/concepts/sleep.mdx +0 -312
  69. docs/concepts/steps.mdx +0 -301
  70. docs/concepts/workflows.mdx +0 -255
  71. docs/guides/cli.mdx +0 -942
  72. docs/guides/configuration.mdx +0 -560
  73. docs/introduction.mdx +0 -155
  74. docs/quickstart.mdx +0 -279
  75. examples/__init__.py +0 -1
  76. examples/celery/__init__.py +0 -1
  77. examples/celery/durable/docker-compose.yml +0 -55
  78. examples/celery/durable/pyworkflow.config.yaml +0 -12
  79. examples/celery/durable/workflows/__init__.py +0 -122
  80. examples/celery/durable/workflows/basic.py +0 -87
  81. examples/celery/durable/workflows/batch_processing.py +0 -102
  82. examples/celery/durable/workflows/cancellation.py +0 -273
  83. examples/celery/durable/workflows/child_workflow_patterns.py +0 -240
  84. examples/celery/durable/workflows/child_workflows.py +0 -202
  85. examples/celery/durable/workflows/continue_as_new.py +0 -260
  86. examples/celery/durable/workflows/fault_tolerance.py +0 -210
  87. examples/celery/durable/workflows/hooks.py +0 -211
  88. examples/celery/durable/workflows/idempotency.py +0 -112
  89. examples/celery/durable/workflows/long_running.py +0 -99
  90. examples/celery/durable/workflows/retries.py +0 -101
  91. examples/celery/durable/workflows/schedules.py +0 -209
  92. examples/celery/transient/01_basic_workflow.py +0 -91
  93. examples/celery/transient/02_fault_tolerance.py +0 -257
  94. examples/celery/transient/__init__.py +0 -20
  95. examples/celery/transient/pyworkflow.config.yaml +0 -25
  96. examples/local/__init__.py +0 -1
  97. examples/local/durable/01_basic_workflow.py +0 -94
  98. examples/local/durable/02_file_storage.py +0 -132
  99. examples/local/durable/03_retries.py +0 -169
  100. examples/local/durable/04_long_running.py +0 -119
  101. examples/local/durable/05_event_log.py +0 -145
  102. examples/local/durable/06_idempotency.py +0 -148
  103. examples/local/durable/07_hooks.py +0 -334
  104. examples/local/durable/08_cancellation.py +0 -233
  105. examples/local/durable/09_child_workflows.py +0 -198
  106. examples/local/durable/10_child_workflow_patterns.py +0 -265
  107. examples/local/durable/11_continue_as_new.py +0 -249
  108. examples/local/durable/12_schedules.py +0 -198
  109. examples/local/durable/__init__.py +0 -1
  110. examples/local/transient/01_quick_tasks.py +0 -87
  111. examples/local/transient/02_retries.py +0 -130
  112. examples/local/transient/03_sleep.py +0 -141
  113. examples/local/transient/__init__.py +0 -1
  114. pyworkflow_engine-0.1.7.dist-info/RECORD +0 -196
  115. pyworkflow_engine-0.1.7.dist-info/top_level.txt +0 -5
  116. tests/examples/__init__.py +0 -0
  117. tests/integration/__init__.py +0 -0
  118. tests/integration/test_cancellation.py +0 -330
  119. tests/integration/test_child_workflows.py +0 -439
  120. tests/integration/test_continue_as_new.py +0 -428
  121. tests/integration/test_dynamodb_storage.py +0 -1146
  122. tests/integration/test_fault_tolerance.py +0 -369
  123. tests/integration/test_schedule_storage.py +0 -484
  124. tests/unit/__init__.py +0 -0
  125. tests/unit/backends/__init__.py +0 -1
  126. tests/unit/backends/test_dynamodb_storage.py +0 -1554
  127. tests/unit/backends/test_postgres_storage.py +0 -1281
  128. tests/unit/backends/test_sqlite_storage.py +0 -1460
  129. tests/unit/conftest.py +0 -41
  130. tests/unit/test_cancellation.py +0 -364
  131. tests/unit/test_child_workflows.py +0 -680
  132. tests/unit/test_continue_as_new.py +0 -441
  133. tests/unit/test_event_limits.py +0 -316
  134. tests/unit/test_executor.py +0 -320
  135. tests/unit/test_fault_tolerance.py +0 -334
  136. tests/unit/test_hooks.py +0 -495
  137. tests/unit/test_registry.py +0 -261
  138. tests/unit/test_replay.py +0 -420
  139. tests/unit/test_schedule_schemas.py +0 -285
  140. tests/unit/test_schedule_utils.py +0 -286
  141. tests/unit/test_scheduled_workflow.py +0 -274
  142. tests/unit/test_step.py +0 -353
  143. tests/unit/test_workflow.py +0 -243
  144. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/WHEEL +0 -0
  145. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/entry_points.txt +0 -0
  146. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/licenses/LICENSE +0 -0
@@ -1,330 +0,0 @@
1
- """
2
- Integration tests for cancellation feature.
3
-
4
- Tests cover:
5
- - cancel_workflow() function
6
- - Cancellation during step execution
7
- - Cancellation during sleep
8
- - Cancellation during hook wait
9
- - Shield prevents cancellation
10
- - Workflow catches CancellationError for cleanup
11
- """
12
-
13
- import asyncio
14
-
15
- import pytest
16
-
17
- from pyworkflow import (
18
- CancellationError,
19
- RunStatus,
20
- cancel_workflow,
21
- shield,
22
- step,
23
- )
24
- from pyworkflow.engine.events import EventType
25
- from pyworkflow.storage.memory import InMemoryStorageBackend
26
- from pyworkflow.storage.schemas import WorkflowRun
27
-
28
-
29
- class TestCancelWorkflowFunction:
30
- """Test cancel_workflow() function."""
31
-
32
- @pytest.mark.asyncio
33
- async def test_cancel_running_workflow(self):
34
- """Test cancelling a running workflow."""
35
- storage = InMemoryStorageBackend()
36
-
37
- # Create a workflow run record
38
- run = WorkflowRun(
39
- run_id="run_123",
40
- workflow_name="test_workflow",
41
- status=RunStatus.RUNNING,
42
- )
43
- await storage.create_run(run)
44
-
45
- # Cancel the workflow
46
- result = await cancel_workflow(
47
- run_id="run_123",
48
- reason="User cancelled",
49
- storage=storage,
50
- )
51
-
52
- assert result is True
53
-
54
- # Check cancellation flag was set
55
- assert await storage.check_cancellation_flag("run_123") is True
56
-
57
- # Check cancellation event was recorded
58
- events = await storage.get_events("run_123")
59
- cancellation_events = [e for e in events if e.type == EventType.CANCELLATION_REQUESTED]
60
- assert len(cancellation_events) == 1
61
- assert cancellation_events[0].data["reason"] == "User cancelled"
62
-
63
- @pytest.mark.asyncio
64
- async def test_cancel_suspended_workflow(self):
65
- """Test cancelling a suspended workflow marks it as cancelled."""
66
- storage = InMemoryStorageBackend()
67
-
68
- # Create a suspended workflow
69
- run = WorkflowRun(
70
- run_id="run_456",
71
- workflow_name="test_workflow",
72
- status=RunStatus.SUSPENDED,
73
- )
74
- await storage.create_run(run)
75
-
76
- # Cancel the workflow
77
- result = await cancel_workflow(
78
- run_id="run_456",
79
- storage=storage,
80
- )
81
-
82
- assert result is True
83
-
84
- # Suspended workflows should be marked as cancelled immediately
85
- updated_run = await storage.get_run("run_456")
86
- assert updated_run.status == RunStatus.CANCELLED
87
-
88
- @pytest.mark.asyncio
89
- async def test_cancel_completed_workflow_returns_false(self):
90
- """Test cancelling an already completed workflow returns False."""
91
- storage = InMemoryStorageBackend()
92
-
93
- # Create a completed workflow
94
- run = WorkflowRun(
95
- run_id="run_789",
96
- workflow_name="test_workflow",
97
- status=RunStatus.COMPLETED,
98
- )
99
- await storage.create_run(run)
100
-
101
- # Try to cancel
102
- result = await cancel_workflow(
103
- run_id="run_789",
104
- storage=storage,
105
- )
106
-
107
- assert result is False
108
-
109
- @pytest.mark.asyncio
110
- async def test_cancel_failed_workflow_returns_false(self):
111
- """Test cancelling an already failed workflow returns False."""
112
- storage = InMemoryStorageBackend()
113
-
114
- # Create a failed workflow
115
- run = WorkflowRun(
116
- run_id="run_abc",
117
- workflow_name="test_workflow",
118
- status=RunStatus.FAILED,
119
- )
120
- await storage.create_run(run)
121
-
122
- # Try to cancel
123
- result = await cancel_workflow(
124
- run_id="run_abc",
125
- storage=storage,
126
- )
127
-
128
- assert result is False
129
-
130
- @pytest.mark.asyncio
131
- async def test_cancel_already_cancelled_workflow_returns_false(self):
132
- """Test cancelling an already cancelled workflow returns False."""
133
- storage = InMemoryStorageBackend()
134
-
135
- # Create a cancelled workflow
136
- run = WorkflowRun(
137
- run_id="run_def",
138
- workflow_name="test_workflow",
139
- status=RunStatus.CANCELLED,
140
- )
141
- await storage.create_run(run)
142
-
143
- # Try to cancel again
144
- result = await cancel_workflow(
145
- run_id="run_def",
146
- storage=storage,
147
- )
148
-
149
- assert result is False
150
-
151
- @pytest.mark.asyncio
152
- async def test_cancel_nonexistent_workflow(self):
153
- """Test cancelling a non-existent workflow raises error."""
154
- from pyworkflow import WorkflowNotFoundError
155
-
156
- storage = InMemoryStorageBackend()
157
-
158
- with pytest.raises(WorkflowNotFoundError):
159
- await cancel_workflow(
160
- run_id="nonexistent",
161
- storage=storage,
162
- )
163
-
164
-
165
- class TestCancellationCheckPoints:
166
- """Test cancellation at various check points."""
167
-
168
- @pytest.mark.asyncio
169
- async def test_step_checks_cancellation(self):
170
- """Test that step execution checks for cancellation."""
171
- from pyworkflow.context import LocalContext, reset_context, set_context
172
-
173
- @step()
174
- async def my_step():
175
- return "result"
176
-
177
- # Create context with cancellation requested
178
- ctx = LocalContext(
179
- run_id="test_run",
180
- workflow_name="test_workflow",
181
- storage=None,
182
- durable=False,
183
- )
184
- ctx.request_cancellation(reason="Test")
185
- token = set_context(ctx)
186
-
187
- try:
188
- with pytest.raises(CancellationError):
189
- await my_step()
190
- finally:
191
- reset_context(token)
192
-
193
-
194
- class TestShieldIntegration:
195
- """Test shield() integration with workflow execution."""
196
-
197
- @pytest.mark.asyncio
198
- async def test_shield_allows_cleanup(self):
199
- """Test shield() allows cleanup operations to complete."""
200
- from pyworkflow.context import LocalContext, reset_context, set_context
201
-
202
- cleanup_completed = False
203
-
204
- async def cleanup():
205
- nonlocal cleanup_completed
206
- await asyncio.sleep(0.01) # Simulate cleanup work
207
- cleanup_completed = True
208
-
209
- ctx = LocalContext(
210
- run_id="test_run",
211
- workflow_name="test_workflow",
212
- storage=None,
213
- durable=False,
214
- )
215
- ctx.request_cancellation()
216
- token = set_context(ctx)
217
-
218
- try:
219
- async with shield():
220
- await cleanup()
221
-
222
- assert cleanup_completed is True
223
- finally:
224
- reset_context(token)
225
-
226
-
227
- class TestFileCancellationFlags:
228
- """Test file storage backend cancellation flags."""
229
-
230
- @pytest.mark.asyncio
231
- async def test_file_storage_cancellation_flags(self, tmp_path):
232
- """Test FileStorageBackend cancellation flag methods."""
233
- from pyworkflow.storage.file import FileStorageBackend
234
-
235
- storage = FileStorageBackend(base_path=str(tmp_path / "workflow_data"))
236
-
237
- # Initially not set
238
- assert await storage.check_cancellation_flag("run_123") is False
239
-
240
- # Set the flag
241
- await storage.set_cancellation_flag("run_123")
242
- assert await storage.check_cancellation_flag("run_123") is True
243
-
244
- # Clear the flag
245
- await storage.clear_cancellation_flag("run_123")
246
- assert await storage.check_cancellation_flag("run_123") is False
247
-
248
- @pytest.mark.asyncio
249
- async def test_file_storage_clear_nonexistent_flag(self, tmp_path):
250
- """Test clearing a non-existent flag does not raise."""
251
- from pyworkflow.storage.file import FileStorageBackend
252
-
253
- storage = FileStorageBackend(base_path=str(tmp_path / "workflow_data"))
254
-
255
- # Should not raise
256
- await storage.clear_cancellation_flag("run_nonexistent")
257
-
258
-
259
- class TestEventReplayCancellation:
260
- """Test cancellation state restoration during event replay."""
261
-
262
- @pytest.mark.asyncio
263
- async def test_replay_restores_cancellation_state(self):
264
- """Test that CANCELLATION_REQUESTED event sets context state during replay."""
265
- from pyworkflow.context import LocalContext
266
- from pyworkflow.engine.events import create_cancellation_requested_event
267
-
268
- storage = InMemoryStorageBackend()
269
-
270
- # Create run
271
- run = WorkflowRun(
272
- run_id="run_123",
273
- workflow_name="test_workflow",
274
- status=RunStatus.RUNNING,
275
- )
276
- await storage.create_run(run)
277
-
278
- # Record cancellation event
279
- event = create_cancellation_requested_event(
280
- run_id="run_123",
281
- reason="User cancelled",
282
- )
283
- await storage.record_event(event)
284
-
285
- # Get events
286
- events = await storage.get_events("run_123")
287
-
288
- # Create context with event log (should replay events)
289
- ctx = LocalContext(
290
- run_id="run_123",
291
- workflow_name="test_workflow",
292
- storage=storage,
293
- event_log=events,
294
- durable=True,
295
- )
296
-
297
- # Context should have cancellation requested from replay
298
- assert ctx.is_cancellation_requested() is True
299
-
300
-
301
- class TestCancellationErrorHandling:
302
- """Test CancellationError handling in workflows."""
303
-
304
- @pytest.mark.asyncio
305
- async def test_workflow_can_catch_cancellation_for_cleanup(self):
306
- """Test that workflows can catch CancellationError for cleanup."""
307
- from pyworkflow.context import LocalContext
308
-
309
- cleanup_called = False
310
-
311
- async def workflow_with_cleanup():
312
- nonlocal cleanup_called
313
- try:
314
- # Simulate work that would check cancellation
315
- ctx = LocalContext(
316
- run_id="test",
317
- workflow_name="test",
318
- storage=None,
319
- durable=False,
320
- )
321
- ctx.request_cancellation()
322
- ctx.check_cancellation()
323
- except CancellationError:
324
- cleanup_called = True
325
- raise
326
-
327
- with pytest.raises(CancellationError):
328
- await workflow_with_cleanup()
329
-
330
- assert cleanup_called is True