pyworkflow-engine 0.1.7__py3-none-any.whl → 0.1.9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. pyworkflow/__init__.py +10 -1
  2. pyworkflow/celery/tasks.py +272 -24
  3. pyworkflow/cli/__init__.py +4 -1
  4. pyworkflow/cli/commands/runs.py +4 -4
  5. pyworkflow/cli/commands/setup.py +203 -4
  6. pyworkflow/cli/utils/config_generator.py +76 -3
  7. pyworkflow/cli/utils/docker_manager.py +232 -0
  8. pyworkflow/context/__init__.py +13 -0
  9. pyworkflow/context/base.py +26 -0
  10. pyworkflow/context/local.py +80 -0
  11. pyworkflow/context/step_context.py +295 -0
  12. pyworkflow/core/registry.py +6 -1
  13. pyworkflow/core/step.py +141 -0
  14. pyworkflow/core/workflow.py +56 -0
  15. pyworkflow/engine/events.py +30 -0
  16. pyworkflow/engine/replay.py +39 -0
  17. pyworkflow/primitives/child_workflow.py +1 -1
  18. pyworkflow/runtime/local.py +1 -1
  19. pyworkflow/storage/__init__.py +14 -0
  20. pyworkflow/storage/base.py +35 -0
  21. pyworkflow/storage/cassandra.py +1747 -0
  22. pyworkflow/storage/config.py +69 -0
  23. pyworkflow/storage/dynamodb.py +31 -2
  24. pyworkflow/storage/file.py +28 -0
  25. pyworkflow/storage/memory.py +18 -0
  26. pyworkflow/storage/mysql.py +1159 -0
  27. pyworkflow/storage/postgres.py +27 -2
  28. pyworkflow/storage/schemas.py +4 -3
  29. pyworkflow/storage/sqlite.py +25 -2
  30. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/METADATA +7 -4
  31. pyworkflow_engine-0.1.9.dist-info/RECORD +91 -0
  32. pyworkflow_engine-0.1.9.dist-info/top_level.txt +1 -0
  33. dashboard/backend/app/__init__.py +0 -1
  34. dashboard/backend/app/config.py +0 -32
  35. dashboard/backend/app/controllers/__init__.py +0 -6
  36. dashboard/backend/app/controllers/run_controller.py +0 -86
  37. dashboard/backend/app/controllers/workflow_controller.py +0 -33
  38. dashboard/backend/app/dependencies/__init__.py +0 -5
  39. dashboard/backend/app/dependencies/storage.py +0 -50
  40. dashboard/backend/app/repositories/__init__.py +0 -6
  41. dashboard/backend/app/repositories/run_repository.py +0 -80
  42. dashboard/backend/app/repositories/workflow_repository.py +0 -27
  43. dashboard/backend/app/rest/__init__.py +0 -8
  44. dashboard/backend/app/rest/v1/__init__.py +0 -12
  45. dashboard/backend/app/rest/v1/health.py +0 -33
  46. dashboard/backend/app/rest/v1/runs.py +0 -133
  47. dashboard/backend/app/rest/v1/workflows.py +0 -41
  48. dashboard/backend/app/schemas/__init__.py +0 -23
  49. dashboard/backend/app/schemas/common.py +0 -16
  50. dashboard/backend/app/schemas/event.py +0 -24
  51. dashboard/backend/app/schemas/hook.py +0 -25
  52. dashboard/backend/app/schemas/run.py +0 -54
  53. dashboard/backend/app/schemas/step.py +0 -28
  54. dashboard/backend/app/schemas/workflow.py +0 -31
  55. dashboard/backend/app/server.py +0 -87
  56. dashboard/backend/app/services/__init__.py +0 -6
  57. dashboard/backend/app/services/run_service.py +0 -240
  58. dashboard/backend/app/services/workflow_service.py +0 -155
  59. dashboard/backend/main.py +0 -18
  60. docs/concepts/cancellation.mdx +0 -362
  61. docs/concepts/continue-as-new.mdx +0 -434
  62. docs/concepts/events.mdx +0 -266
  63. docs/concepts/fault-tolerance.mdx +0 -370
  64. docs/concepts/hooks.mdx +0 -552
  65. docs/concepts/limitations.mdx +0 -167
  66. docs/concepts/schedules.mdx +0 -775
  67. docs/concepts/sleep.mdx +0 -312
  68. docs/concepts/steps.mdx +0 -301
  69. docs/concepts/workflows.mdx +0 -255
  70. docs/guides/cli.mdx +0 -942
  71. docs/guides/configuration.mdx +0 -560
  72. docs/introduction.mdx +0 -155
  73. docs/quickstart.mdx +0 -279
  74. examples/__init__.py +0 -1
  75. examples/celery/__init__.py +0 -1
  76. examples/celery/durable/docker-compose.yml +0 -55
  77. examples/celery/durable/pyworkflow.config.yaml +0 -12
  78. examples/celery/durable/workflows/__init__.py +0 -122
  79. examples/celery/durable/workflows/basic.py +0 -87
  80. examples/celery/durable/workflows/batch_processing.py +0 -102
  81. examples/celery/durable/workflows/cancellation.py +0 -273
  82. examples/celery/durable/workflows/child_workflow_patterns.py +0 -240
  83. examples/celery/durable/workflows/child_workflows.py +0 -202
  84. examples/celery/durable/workflows/continue_as_new.py +0 -260
  85. examples/celery/durable/workflows/fault_tolerance.py +0 -210
  86. examples/celery/durable/workflows/hooks.py +0 -211
  87. examples/celery/durable/workflows/idempotency.py +0 -112
  88. examples/celery/durable/workflows/long_running.py +0 -99
  89. examples/celery/durable/workflows/retries.py +0 -101
  90. examples/celery/durable/workflows/schedules.py +0 -209
  91. examples/celery/transient/01_basic_workflow.py +0 -91
  92. examples/celery/transient/02_fault_tolerance.py +0 -257
  93. examples/celery/transient/__init__.py +0 -20
  94. examples/celery/transient/pyworkflow.config.yaml +0 -25
  95. examples/local/__init__.py +0 -1
  96. examples/local/durable/01_basic_workflow.py +0 -94
  97. examples/local/durable/02_file_storage.py +0 -132
  98. examples/local/durable/03_retries.py +0 -169
  99. examples/local/durable/04_long_running.py +0 -119
  100. examples/local/durable/05_event_log.py +0 -145
  101. examples/local/durable/06_idempotency.py +0 -148
  102. examples/local/durable/07_hooks.py +0 -334
  103. examples/local/durable/08_cancellation.py +0 -233
  104. examples/local/durable/09_child_workflows.py +0 -198
  105. examples/local/durable/10_child_workflow_patterns.py +0 -265
  106. examples/local/durable/11_continue_as_new.py +0 -249
  107. examples/local/durable/12_schedules.py +0 -198
  108. examples/local/durable/__init__.py +0 -1
  109. examples/local/transient/01_quick_tasks.py +0 -87
  110. examples/local/transient/02_retries.py +0 -130
  111. examples/local/transient/03_sleep.py +0 -141
  112. examples/local/transient/__init__.py +0 -1
  113. pyworkflow_engine-0.1.7.dist-info/RECORD +0 -196
  114. pyworkflow_engine-0.1.7.dist-info/top_level.txt +0 -5
  115. tests/examples/__init__.py +0 -0
  116. tests/integration/__init__.py +0 -0
  117. tests/integration/test_cancellation.py +0 -330
  118. tests/integration/test_child_workflows.py +0 -439
  119. tests/integration/test_continue_as_new.py +0 -428
  120. tests/integration/test_dynamodb_storage.py +0 -1146
  121. tests/integration/test_fault_tolerance.py +0 -369
  122. tests/integration/test_schedule_storage.py +0 -484
  123. tests/unit/__init__.py +0 -0
  124. tests/unit/backends/__init__.py +0 -1
  125. tests/unit/backends/test_dynamodb_storage.py +0 -1554
  126. tests/unit/backends/test_postgres_storage.py +0 -1281
  127. tests/unit/backends/test_sqlite_storage.py +0 -1460
  128. tests/unit/conftest.py +0 -41
  129. tests/unit/test_cancellation.py +0 -364
  130. tests/unit/test_child_workflows.py +0 -680
  131. tests/unit/test_continue_as_new.py +0 -441
  132. tests/unit/test_event_limits.py +0 -316
  133. tests/unit/test_executor.py +0 -320
  134. tests/unit/test_fault_tolerance.py +0 -334
  135. tests/unit/test_hooks.py +0 -495
  136. tests/unit/test_registry.py +0 -261
  137. tests/unit/test_replay.py +0 -420
  138. tests/unit/test_schedule_schemas.py +0 -285
  139. tests/unit/test_schedule_utils.py +0 -286
  140. tests/unit/test_scheduled_workflow.py +0 -274
  141. tests/unit/test_step.py +0 -353
  142. tests/unit/test_workflow.py +0 -243
  143. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/WHEEL +0 -0
  144. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/entry_points.txt +0 -0
  145. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/licenses/LICENSE +0 -0
tests/unit/conftest.py DELETED
@@ -1,41 +0,0 @@
1
- """
2
- Test configuration and fixtures for unit tests.
3
- """
4
-
5
- import pytest
6
-
7
-
8
- @pytest.fixture(autouse=True)
9
- def reset_global_registry():
10
- """Reset the global registry before each test to ensure test isolation."""
11
- from pyworkflow.core.registry import _registry
12
-
13
- # Store original state
14
- original_workflows = _registry._workflows.copy()
15
- original_steps = _registry._steps.copy()
16
-
17
- # Clear registry for test
18
- _registry._workflows.clear()
19
- _registry._steps.clear()
20
-
21
- yield
22
-
23
- # Restore original state (or clear again)
24
- _registry._workflows.clear()
25
- _registry._steps.clear()
26
- _registry._workflows.update(original_workflows)
27
- _registry._steps.update(original_steps)
28
-
29
-
30
- @pytest.fixture(autouse=True)
31
- def reset_workflow_context():
32
- """Ensure no workflow context leaks between tests."""
33
- from pyworkflow.context import set_context
34
-
35
- # Clear any existing context before test
36
- set_context(None)
37
-
38
- yield
39
-
40
- # Clear context after test
41
- set_context(None)
@@ -1,364 +0,0 @@
1
- """
2
- Unit tests for cancellation feature.
3
-
4
- Tests cover:
5
- - CancellationError exception
6
- - Context cancellation state methods
7
- - shield() context manager
8
- - Storage cancellation flag methods
9
- - Cancellation events
10
- """
11
-
12
- import pytest
13
-
14
- from pyworkflow import (
15
- CancellationError,
16
- LocalContext,
17
- MockContext,
18
- set_context,
19
- shield,
20
- )
21
- from pyworkflow.engine.events import (
22
- EventType,
23
- create_cancellation_requested_event,
24
- create_step_cancelled_event,
25
- create_workflow_cancelled_event,
26
- )
27
- from pyworkflow.storage.memory import InMemoryStorageBackend
28
-
29
-
30
- class TestCancellationError:
31
- """Test CancellationError exception."""
32
-
33
- def test_cancellation_error_default_message(self):
34
- """Test CancellationError has default message."""
35
- error = CancellationError()
36
- assert str(error) == "Workflow was cancelled"
37
- assert error.reason is None
38
-
39
- def test_cancellation_error_custom_message(self):
40
- """Test CancellationError with custom message."""
41
- error = CancellationError("Custom cancellation message")
42
- assert str(error) == "Custom cancellation message"
43
-
44
- def test_cancellation_error_with_reason(self):
45
- """Test CancellationError with reason."""
46
- error = CancellationError("Cancelled", reason="User requested")
47
- assert error.reason == "User requested"
48
-
49
- def test_cancellation_error_is_workflow_error(self):
50
- """Test CancellationError inherits from WorkflowError."""
51
- from pyworkflow import WorkflowError
52
-
53
- error = CancellationError()
54
- assert isinstance(error, WorkflowError)
55
-
56
-
57
- class TestContextCancellationState:
58
- """Test context cancellation state methods."""
59
-
60
- def test_local_context_cancellation_not_requested_by_default(self):
61
- """Test LocalContext starts with cancellation not requested."""
62
- ctx = LocalContext(
63
- run_id="test_run",
64
- workflow_name="test_workflow",
65
- storage=None,
66
- durable=False,
67
- )
68
- assert ctx.is_cancellation_requested() is False
69
-
70
- def test_local_context_request_cancellation(self):
71
- """Test LocalContext can request cancellation."""
72
- ctx = LocalContext(
73
- run_id="test_run",
74
- workflow_name="test_workflow",
75
- storage=None,
76
- durable=False,
77
- )
78
- ctx.request_cancellation()
79
- assert ctx.is_cancellation_requested() is True
80
-
81
- def test_local_context_request_cancellation_with_reason(self):
82
- """Test LocalContext stores cancellation reason."""
83
- ctx = LocalContext(
84
- run_id="test_run",
85
- workflow_name="test_workflow",
86
- storage=None,
87
- durable=False,
88
- )
89
- ctx.request_cancellation(reason="User clicked cancel")
90
- assert ctx.is_cancellation_requested() is True
91
- assert ctx._cancellation_reason == "User clicked cancel"
92
-
93
- def test_local_context_check_cancellation_raises_when_requested(self):
94
- """Test check_cancellation raises CancellationError when requested."""
95
- ctx = LocalContext(
96
- run_id="test_run",
97
- workflow_name="test_workflow",
98
- storage=None,
99
- durable=False,
100
- )
101
- ctx.request_cancellation(reason="Test")
102
-
103
- with pytest.raises(CancellationError) as exc_info:
104
- ctx.check_cancellation()
105
-
106
- assert exc_info.value.reason == "Test"
107
-
108
- def test_local_context_check_cancellation_does_not_raise_when_not_requested(self):
109
- """Test check_cancellation does not raise when not requested."""
110
- ctx = LocalContext(
111
- run_id="test_run",
112
- workflow_name="test_workflow",
113
- storage=None,
114
- durable=False,
115
- )
116
- # Should not raise
117
- ctx.check_cancellation()
118
-
119
- def test_local_context_cancellation_blocked_property(self):
120
- """Test cancellation_blocked property."""
121
- ctx = LocalContext(
122
- run_id="test_run",
123
- workflow_name="test_workflow",
124
- storage=None,
125
- durable=False,
126
- )
127
- assert ctx.cancellation_blocked is False
128
-
129
- ctx._cancellation_blocked = True
130
- assert ctx.cancellation_blocked is True
131
-
132
- def test_local_context_check_cancellation_blocked(self):
133
- """Test check_cancellation does not raise when blocked."""
134
- ctx = LocalContext(
135
- run_id="test_run",
136
- workflow_name="test_workflow",
137
- storage=None,
138
- durable=False,
139
- )
140
- ctx.request_cancellation()
141
- ctx._cancellation_blocked = True
142
-
143
- # Should not raise even though cancellation is requested
144
- ctx.check_cancellation()
145
-
146
-
147
- class TestShieldContextManager:
148
- """Test shield() context manager."""
149
-
150
- @pytest.mark.asyncio
151
- async def test_shield_blocks_cancellation(self):
152
- """Test shield() blocks cancellation check."""
153
- ctx = LocalContext(
154
- run_id="test_run",
155
- workflow_name="test_workflow",
156
- storage=None,
157
- durable=False,
158
- )
159
- set_context(ctx)
160
-
161
- try:
162
- ctx.request_cancellation()
163
-
164
- async with shield():
165
- # Should not raise while shielded
166
- ctx.check_cancellation()
167
-
168
- # Should raise after shield
169
- with pytest.raises(CancellationError):
170
- ctx.check_cancellation()
171
- finally:
172
- set_context(None)
173
-
174
- @pytest.mark.asyncio
175
- async def test_shield_restores_previous_state(self):
176
- """Test shield() restores previous blocked state."""
177
- ctx = LocalContext(
178
- run_id="test_run",
179
- workflow_name="test_workflow",
180
- storage=None,
181
- durable=False,
182
- )
183
- set_context(ctx)
184
-
185
- try:
186
- assert ctx.cancellation_blocked is False
187
-
188
- async with shield():
189
- assert ctx.cancellation_blocked is True
190
-
191
- assert ctx.cancellation_blocked is False
192
- finally:
193
- set_context(None)
194
-
195
- @pytest.mark.asyncio
196
- async def test_shield_nested(self):
197
- """Test nested shield() calls work correctly."""
198
- ctx = LocalContext(
199
- run_id="test_run",
200
- workflow_name="test_workflow",
201
- storage=None,
202
- durable=False,
203
- )
204
- set_context(ctx)
205
-
206
- try:
207
- ctx.request_cancellation()
208
-
209
- async with shield():
210
- assert ctx.cancellation_blocked is True
211
- ctx.check_cancellation() # Should not raise
212
-
213
- async with shield():
214
- assert ctx.cancellation_blocked is True
215
- ctx.check_cancellation() # Should not raise
216
-
217
- # Still blocked after inner shield
218
- assert ctx.cancellation_blocked is True
219
- ctx.check_cancellation() # Should not raise
220
-
221
- # Now should raise
222
- with pytest.raises(CancellationError):
223
- ctx.check_cancellation()
224
- finally:
225
- set_context(None)
226
-
227
- @pytest.mark.asyncio
228
- async def test_shield_without_context(self):
229
- """Test shield() works without workflow context (no-op)."""
230
- set_context(None)
231
-
232
- # Should not raise
233
- async with shield():
234
- pass
235
-
236
-
237
- class TestStorageCancellationFlags:
238
- """Test storage backend cancellation flag methods."""
239
-
240
- @pytest.mark.asyncio
241
- async def test_memory_storage_set_cancellation_flag(self):
242
- """Test InMemoryStorageBackend set_cancellation_flag."""
243
- storage = InMemoryStorageBackend()
244
-
245
- await storage.set_cancellation_flag("run_123")
246
-
247
- assert await storage.check_cancellation_flag("run_123") is True
248
-
249
- @pytest.mark.asyncio
250
- async def test_memory_storage_check_cancellation_flag_not_set(self):
251
- """Test InMemoryStorageBackend returns False when flag not set."""
252
- storage = InMemoryStorageBackend()
253
-
254
- assert await storage.check_cancellation_flag("run_123") is False
255
-
256
- @pytest.mark.asyncio
257
- async def test_memory_storage_clear_cancellation_flag(self):
258
- """Test InMemoryStorageBackend clear_cancellation_flag."""
259
- storage = InMemoryStorageBackend()
260
-
261
- await storage.set_cancellation_flag("run_123")
262
- assert await storage.check_cancellation_flag("run_123") is True
263
-
264
- await storage.clear_cancellation_flag("run_123")
265
- assert await storage.check_cancellation_flag("run_123") is False
266
-
267
- @pytest.mark.asyncio
268
- async def test_memory_storage_clear_nonexistent_flag(self):
269
- """Test clearing a non-existent flag does not raise."""
270
- storage = InMemoryStorageBackend()
271
-
272
- # Should not raise
273
- await storage.clear_cancellation_flag("run_nonexistent")
274
-
275
-
276
- class TestCancellationEvents:
277
- """Test cancellation event creation."""
278
-
279
- def test_create_cancellation_requested_event(self):
280
- """Test create_cancellation_requested_event."""
281
- event = create_cancellation_requested_event(
282
- run_id="run_123",
283
- reason="User requested",
284
- requested_by="admin",
285
- )
286
-
287
- assert event.run_id == "run_123"
288
- assert event.type == EventType.CANCELLATION_REQUESTED
289
- assert event.data["reason"] == "User requested"
290
- assert event.data["requested_by"] == "admin"
291
-
292
- def test_create_cancellation_requested_event_minimal(self):
293
- """Test create_cancellation_requested_event with minimal params."""
294
- event = create_cancellation_requested_event(run_id="run_123")
295
-
296
- assert event.run_id == "run_123"
297
- assert event.type == EventType.CANCELLATION_REQUESTED
298
- assert event.data.get("reason") is None
299
- assert event.data.get("requested_by") is None
300
-
301
- def test_create_workflow_cancelled_event(self):
302
- """Test create_workflow_cancelled_event."""
303
- event = create_workflow_cancelled_event(
304
- run_id="run_123",
305
- reason="Test cancellation",
306
- cleanup_completed=True,
307
- )
308
-
309
- assert event.run_id == "run_123"
310
- assert event.type == EventType.WORKFLOW_CANCELLED
311
- assert event.data["reason"] == "Test cancellation"
312
- assert event.data["cleanup_completed"] is True
313
-
314
- def test_create_workflow_cancelled_event_minimal(self):
315
- """Test create_workflow_cancelled_event with minimal params."""
316
- event = create_workflow_cancelled_event(run_id="run_123")
317
-
318
- assert event.run_id == "run_123"
319
- assert event.type == EventType.WORKFLOW_CANCELLED
320
- assert event.data.get("cleanup_completed") is False
321
-
322
- def test_create_step_cancelled_event(self):
323
- """Test create_step_cancelled_event."""
324
- event = create_step_cancelled_event(
325
- run_id="run_123",
326
- step_id="step_456",
327
- step_name="my_step",
328
- )
329
-
330
- assert event.run_id == "run_123"
331
- assert event.type == EventType.STEP_CANCELLED
332
- assert event.data["step_id"] == "step_456"
333
- assert event.data["step_name"] == "my_step"
334
-
335
-
336
- class TestMockContextCancellation:
337
- """Test MockContext cancellation support."""
338
-
339
- def test_mock_context_cancellation_not_requested_by_default(self):
340
- """Test MockContext starts with cancellation not requested."""
341
- ctx = MockContext(run_id="test", workflow_name="test")
342
- assert ctx.is_cancellation_requested() is False
343
-
344
- def test_mock_context_request_cancellation(self):
345
- """Test MockContext can request cancellation."""
346
- ctx = MockContext(run_id="test", workflow_name="test")
347
- ctx.request_cancellation()
348
- assert ctx.is_cancellation_requested() is True
349
-
350
- def test_mock_context_check_cancellation(self):
351
- """Test MockContext check_cancellation raises when requested."""
352
- ctx = MockContext(run_id="test", workflow_name="test")
353
- ctx.request_cancellation(reason="Test")
354
-
355
- with pytest.raises(CancellationError):
356
- ctx.check_cancellation()
357
-
358
- def test_mock_context_cancellation_blocked(self):
359
- """Test MockContext cancellation blocked property."""
360
- ctx = MockContext(run_id="test", workflow_name="test")
361
- assert ctx.cancellation_blocked is False
362
-
363
- ctx._cancellation_blocked = True
364
- assert ctx.cancellation_blocked is True