pyworkflow-engine 0.1.7__py3-none-any.whl → 0.1.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. pyworkflow/__init__.py +10 -1
  2. pyworkflow/celery/tasks.py +272 -24
  3. pyworkflow/cli/__init__.py +4 -1
  4. pyworkflow/cli/commands/runs.py +4 -4
  5. pyworkflow/cli/commands/setup.py +203 -4
  6. pyworkflow/cli/utils/config_generator.py +76 -3
  7. pyworkflow/cli/utils/docker_manager.py +232 -0
  8. pyworkflow/config.py +94 -17
  9. pyworkflow/context/__init__.py +13 -0
  10. pyworkflow/context/base.py +26 -0
  11. pyworkflow/context/local.py +80 -0
  12. pyworkflow/context/step_context.py +295 -0
  13. pyworkflow/core/registry.py +6 -1
  14. pyworkflow/core/step.py +141 -0
  15. pyworkflow/core/workflow.py +56 -0
  16. pyworkflow/engine/events.py +30 -0
  17. pyworkflow/engine/replay.py +39 -0
  18. pyworkflow/primitives/child_workflow.py +1 -1
  19. pyworkflow/runtime/local.py +1 -1
  20. pyworkflow/storage/__init__.py +14 -0
  21. pyworkflow/storage/base.py +35 -0
  22. pyworkflow/storage/cassandra.py +1747 -0
  23. pyworkflow/storage/config.py +69 -0
  24. pyworkflow/storage/dynamodb.py +31 -2
  25. pyworkflow/storage/file.py +28 -0
  26. pyworkflow/storage/memory.py +18 -0
  27. pyworkflow/storage/mysql.py +1159 -0
  28. pyworkflow/storage/postgres.py +27 -2
  29. pyworkflow/storage/schemas.py +4 -3
  30. pyworkflow/storage/sqlite.py +25 -2
  31. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/METADATA +7 -4
  32. pyworkflow_engine-0.1.10.dist-info/RECORD +91 -0
  33. pyworkflow_engine-0.1.10.dist-info/top_level.txt +1 -0
  34. dashboard/backend/app/__init__.py +0 -1
  35. dashboard/backend/app/config.py +0 -32
  36. dashboard/backend/app/controllers/__init__.py +0 -6
  37. dashboard/backend/app/controllers/run_controller.py +0 -86
  38. dashboard/backend/app/controllers/workflow_controller.py +0 -33
  39. dashboard/backend/app/dependencies/__init__.py +0 -5
  40. dashboard/backend/app/dependencies/storage.py +0 -50
  41. dashboard/backend/app/repositories/__init__.py +0 -6
  42. dashboard/backend/app/repositories/run_repository.py +0 -80
  43. dashboard/backend/app/repositories/workflow_repository.py +0 -27
  44. dashboard/backend/app/rest/__init__.py +0 -8
  45. dashboard/backend/app/rest/v1/__init__.py +0 -12
  46. dashboard/backend/app/rest/v1/health.py +0 -33
  47. dashboard/backend/app/rest/v1/runs.py +0 -133
  48. dashboard/backend/app/rest/v1/workflows.py +0 -41
  49. dashboard/backend/app/schemas/__init__.py +0 -23
  50. dashboard/backend/app/schemas/common.py +0 -16
  51. dashboard/backend/app/schemas/event.py +0 -24
  52. dashboard/backend/app/schemas/hook.py +0 -25
  53. dashboard/backend/app/schemas/run.py +0 -54
  54. dashboard/backend/app/schemas/step.py +0 -28
  55. dashboard/backend/app/schemas/workflow.py +0 -31
  56. dashboard/backend/app/server.py +0 -87
  57. dashboard/backend/app/services/__init__.py +0 -6
  58. dashboard/backend/app/services/run_service.py +0 -240
  59. dashboard/backend/app/services/workflow_service.py +0 -155
  60. dashboard/backend/main.py +0 -18
  61. docs/concepts/cancellation.mdx +0 -362
  62. docs/concepts/continue-as-new.mdx +0 -434
  63. docs/concepts/events.mdx +0 -266
  64. docs/concepts/fault-tolerance.mdx +0 -370
  65. docs/concepts/hooks.mdx +0 -552
  66. docs/concepts/limitations.mdx +0 -167
  67. docs/concepts/schedules.mdx +0 -775
  68. docs/concepts/sleep.mdx +0 -312
  69. docs/concepts/steps.mdx +0 -301
  70. docs/concepts/workflows.mdx +0 -255
  71. docs/guides/cli.mdx +0 -942
  72. docs/guides/configuration.mdx +0 -560
  73. docs/introduction.mdx +0 -155
  74. docs/quickstart.mdx +0 -279
  75. examples/__init__.py +0 -1
  76. examples/celery/__init__.py +0 -1
  77. examples/celery/durable/docker-compose.yml +0 -55
  78. examples/celery/durable/pyworkflow.config.yaml +0 -12
  79. examples/celery/durable/workflows/__init__.py +0 -122
  80. examples/celery/durable/workflows/basic.py +0 -87
  81. examples/celery/durable/workflows/batch_processing.py +0 -102
  82. examples/celery/durable/workflows/cancellation.py +0 -273
  83. examples/celery/durable/workflows/child_workflow_patterns.py +0 -240
  84. examples/celery/durable/workflows/child_workflows.py +0 -202
  85. examples/celery/durable/workflows/continue_as_new.py +0 -260
  86. examples/celery/durable/workflows/fault_tolerance.py +0 -210
  87. examples/celery/durable/workflows/hooks.py +0 -211
  88. examples/celery/durable/workflows/idempotency.py +0 -112
  89. examples/celery/durable/workflows/long_running.py +0 -99
  90. examples/celery/durable/workflows/retries.py +0 -101
  91. examples/celery/durable/workflows/schedules.py +0 -209
  92. examples/celery/transient/01_basic_workflow.py +0 -91
  93. examples/celery/transient/02_fault_tolerance.py +0 -257
  94. examples/celery/transient/__init__.py +0 -20
  95. examples/celery/transient/pyworkflow.config.yaml +0 -25
  96. examples/local/__init__.py +0 -1
  97. examples/local/durable/01_basic_workflow.py +0 -94
  98. examples/local/durable/02_file_storage.py +0 -132
  99. examples/local/durable/03_retries.py +0 -169
  100. examples/local/durable/04_long_running.py +0 -119
  101. examples/local/durable/05_event_log.py +0 -145
  102. examples/local/durable/06_idempotency.py +0 -148
  103. examples/local/durable/07_hooks.py +0 -334
  104. examples/local/durable/08_cancellation.py +0 -233
  105. examples/local/durable/09_child_workflows.py +0 -198
  106. examples/local/durable/10_child_workflow_patterns.py +0 -265
  107. examples/local/durable/11_continue_as_new.py +0 -249
  108. examples/local/durable/12_schedules.py +0 -198
  109. examples/local/durable/__init__.py +0 -1
  110. examples/local/transient/01_quick_tasks.py +0 -87
  111. examples/local/transient/02_retries.py +0 -130
  112. examples/local/transient/03_sleep.py +0 -141
  113. examples/local/transient/__init__.py +0 -1
  114. pyworkflow_engine-0.1.7.dist-info/RECORD +0 -196
  115. pyworkflow_engine-0.1.7.dist-info/top_level.txt +0 -5
  116. tests/examples/__init__.py +0 -0
  117. tests/integration/__init__.py +0 -0
  118. tests/integration/test_cancellation.py +0 -330
  119. tests/integration/test_child_workflows.py +0 -439
  120. tests/integration/test_continue_as_new.py +0 -428
  121. tests/integration/test_dynamodb_storage.py +0 -1146
  122. tests/integration/test_fault_tolerance.py +0 -369
  123. tests/integration/test_schedule_storage.py +0 -484
  124. tests/unit/__init__.py +0 -0
  125. tests/unit/backends/__init__.py +0 -1
  126. tests/unit/backends/test_dynamodb_storage.py +0 -1554
  127. tests/unit/backends/test_postgres_storage.py +0 -1281
  128. tests/unit/backends/test_sqlite_storage.py +0 -1460
  129. tests/unit/conftest.py +0 -41
  130. tests/unit/test_cancellation.py +0 -364
  131. tests/unit/test_child_workflows.py +0 -680
  132. tests/unit/test_continue_as_new.py +0 -441
  133. tests/unit/test_event_limits.py +0 -316
  134. tests/unit/test_executor.py +0 -320
  135. tests/unit/test_fault_tolerance.py +0 -334
  136. tests/unit/test_hooks.py +0 -495
  137. tests/unit/test_registry.py +0 -261
  138. tests/unit/test_replay.py +0 -420
  139. tests/unit/test_schedule_schemas.py +0 -285
  140. tests/unit/test_schedule_utils.py +0 -286
  141. tests/unit/test_scheduled_workflow.py +0 -274
  142. tests/unit/test_step.py +0 -353
  143. tests/unit/test_workflow.py +0 -243
  144. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/WHEEL +0 -0
  145. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/entry_points.txt +0 -0
  146. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/licenses/LICENSE +0 -0
tests/unit/conftest.py DELETED
@@ -1,41 +0,0 @@
1
- """
2
- Test configuration and fixtures for unit tests.
3
- """
4
-
5
- import pytest
6
-
7
-
8
- @pytest.fixture(autouse=True)
9
- def reset_global_registry():
10
- """Reset the global registry before each test to ensure test isolation."""
11
- from pyworkflow.core.registry import _registry
12
-
13
- # Store original state
14
- original_workflows = _registry._workflows.copy()
15
- original_steps = _registry._steps.copy()
16
-
17
- # Clear registry for test
18
- _registry._workflows.clear()
19
- _registry._steps.clear()
20
-
21
- yield
22
-
23
- # Restore original state (or clear again)
24
- _registry._workflows.clear()
25
- _registry._steps.clear()
26
- _registry._workflows.update(original_workflows)
27
- _registry._steps.update(original_steps)
28
-
29
-
30
- @pytest.fixture(autouse=True)
31
- def reset_workflow_context():
32
- """Ensure no workflow context leaks between tests."""
33
- from pyworkflow.context import set_context
34
-
35
- # Clear any existing context before test
36
- set_context(None)
37
-
38
- yield
39
-
40
- # Clear context after test
41
- set_context(None)
@@ -1,364 +0,0 @@
1
- """
2
- Unit tests for cancellation feature.
3
-
4
- Tests cover:
5
- - CancellationError exception
6
- - Context cancellation state methods
7
- - shield() context manager
8
- - Storage cancellation flag methods
9
- - Cancellation events
10
- """
11
-
12
- import pytest
13
-
14
- from pyworkflow import (
15
- CancellationError,
16
- LocalContext,
17
- MockContext,
18
- set_context,
19
- shield,
20
- )
21
- from pyworkflow.engine.events import (
22
- EventType,
23
- create_cancellation_requested_event,
24
- create_step_cancelled_event,
25
- create_workflow_cancelled_event,
26
- )
27
- from pyworkflow.storage.memory import InMemoryStorageBackend
28
-
29
-
30
- class TestCancellationError:
31
- """Test CancellationError exception."""
32
-
33
- def test_cancellation_error_default_message(self):
34
- """Test CancellationError has default message."""
35
- error = CancellationError()
36
- assert str(error) == "Workflow was cancelled"
37
- assert error.reason is None
38
-
39
- def test_cancellation_error_custom_message(self):
40
- """Test CancellationError with custom message."""
41
- error = CancellationError("Custom cancellation message")
42
- assert str(error) == "Custom cancellation message"
43
-
44
- def test_cancellation_error_with_reason(self):
45
- """Test CancellationError with reason."""
46
- error = CancellationError("Cancelled", reason="User requested")
47
- assert error.reason == "User requested"
48
-
49
- def test_cancellation_error_is_workflow_error(self):
50
- """Test CancellationError inherits from WorkflowError."""
51
- from pyworkflow import WorkflowError
52
-
53
- error = CancellationError()
54
- assert isinstance(error, WorkflowError)
55
-
56
-
57
- class TestContextCancellationState:
58
- """Test context cancellation state methods."""
59
-
60
- def test_local_context_cancellation_not_requested_by_default(self):
61
- """Test LocalContext starts with cancellation not requested."""
62
- ctx = LocalContext(
63
- run_id="test_run",
64
- workflow_name="test_workflow",
65
- storage=None,
66
- durable=False,
67
- )
68
- assert ctx.is_cancellation_requested() is False
69
-
70
- def test_local_context_request_cancellation(self):
71
- """Test LocalContext can request cancellation."""
72
- ctx = LocalContext(
73
- run_id="test_run",
74
- workflow_name="test_workflow",
75
- storage=None,
76
- durable=False,
77
- )
78
- ctx.request_cancellation()
79
- assert ctx.is_cancellation_requested() is True
80
-
81
- def test_local_context_request_cancellation_with_reason(self):
82
- """Test LocalContext stores cancellation reason."""
83
- ctx = LocalContext(
84
- run_id="test_run",
85
- workflow_name="test_workflow",
86
- storage=None,
87
- durable=False,
88
- )
89
- ctx.request_cancellation(reason="User clicked cancel")
90
- assert ctx.is_cancellation_requested() is True
91
- assert ctx._cancellation_reason == "User clicked cancel"
92
-
93
- def test_local_context_check_cancellation_raises_when_requested(self):
94
- """Test check_cancellation raises CancellationError when requested."""
95
- ctx = LocalContext(
96
- run_id="test_run",
97
- workflow_name="test_workflow",
98
- storage=None,
99
- durable=False,
100
- )
101
- ctx.request_cancellation(reason="Test")
102
-
103
- with pytest.raises(CancellationError) as exc_info:
104
- ctx.check_cancellation()
105
-
106
- assert exc_info.value.reason == "Test"
107
-
108
- def test_local_context_check_cancellation_does_not_raise_when_not_requested(self):
109
- """Test check_cancellation does not raise when not requested."""
110
- ctx = LocalContext(
111
- run_id="test_run",
112
- workflow_name="test_workflow",
113
- storage=None,
114
- durable=False,
115
- )
116
- # Should not raise
117
- ctx.check_cancellation()
118
-
119
- def test_local_context_cancellation_blocked_property(self):
120
- """Test cancellation_blocked property."""
121
- ctx = LocalContext(
122
- run_id="test_run",
123
- workflow_name="test_workflow",
124
- storage=None,
125
- durable=False,
126
- )
127
- assert ctx.cancellation_blocked is False
128
-
129
- ctx._cancellation_blocked = True
130
- assert ctx.cancellation_blocked is True
131
-
132
- def test_local_context_check_cancellation_blocked(self):
133
- """Test check_cancellation does not raise when blocked."""
134
- ctx = LocalContext(
135
- run_id="test_run",
136
- workflow_name="test_workflow",
137
- storage=None,
138
- durable=False,
139
- )
140
- ctx.request_cancellation()
141
- ctx._cancellation_blocked = True
142
-
143
- # Should not raise even though cancellation is requested
144
- ctx.check_cancellation()
145
-
146
-
147
- class TestShieldContextManager:
148
- """Test shield() context manager."""
149
-
150
- @pytest.mark.asyncio
151
- async def test_shield_blocks_cancellation(self):
152
- """Test shield() blocks cancellation check."""
153
- ctx = LocalContext(
154
- run_id="test_run",
155
- workflow_name="test_workflow",
156
- storage=None,
157
- durable=False,
158
- )
159
- set_context(ctx)
160
-
161
- try:
162
- ctx.request_cancellation()
163
-
164
- async with shield():
165
- # Should not raise while shielded
166
- ctx.check_cancellation()
167
-
168
- # Should raise after shield
169
- with pytest.raises(CancellationError):
170
- ctx.check_cancellation()
171
- finally:
172
- set_context(None)
173
-
174
- @pytest.mark.asyncio
175
- async def test_shield_restores_previous_state(self):
176
- """Test shield() restores previous blocked state."""
177
- ctx = LocalContext(
178
- run_id="test_run",
179
- workflow_name="test_workflow",
180
- storage=None,
181
- durable=False,
182
- )
183
- set_context(ctx)
184
-
185
- try:
186
- assert ctx.cancellation_blocked is False
187
-
188
- async with shield():
189
- assert ctx.cancellation_blocked is True
190
-
191
- assert ctx.cancellation_blocked is False
192
- finally:
193
- set_context(None)
194
-
195
- @pytest.mark.asyncio
196
- async def test_shield_nested(self):
197
- """Test nested shield() calls work correctly."""
198
- ctx = LocalContext(
199
- run_id="test_run",
200
- workflow_name="test_workflow",
201
- storage=None,
202
- durable=False,
203
- )
204
- set_context(ctx)
205
-
206
- try:
207
- ctx.request_cancellation()
208
-
209
- async with shield():
210
- assert ctx.cancellation_blocked is True
211
- ctx.check_cancellation() # Should not raise
212
-
213
- async with shield():
214
- assert ctx.cancellation_blocked is True
215
- ctx.check_cancellation() # Should not raise
216
-
217
- # Still blocked after inner shield
218
- assert ctx.cancellation_blocked is True
219
- ctx.check_cancellation() # Should not raise
220
-
221
- # Now should raise
222
- with pytest.raises(CancellationError):
223
- ctx.check_cancellation()
224
- finally:
225
- set_context(None)
226
-
227
- @pytest.mark.asyncio
228
- async def test_shield_without_context(self):
229
- """Test shield() works without workflow context (no-op)."""
230
- set_context(None)
231
-
232
- # Should not raise
233
- async with shield():
234
- pass
235
-
236
-
237
- class TestStorageCancellationFlags:
238
- """Test storage backend cancellation flag methods."""
239
-
240
- @pytest.mark.asyncio
241
- async def test_memory_storage_set_cancellation_flag(self):
242
- """Test InMemoryStorageBackend set_cancellation_flag."""
243
- storage = InMemoryStorageBackend()
244
-
245
- await storage.set_cancellation_flag("run_123")
246
-
247
- assert await storage.check_cancellation_flag("run_123") is True
248
-
249
- @pytest.mark.asyncio
250
- async def test_memory_storage_check_cancellation_flag_not_set(self):
251
- """Test InMemoryStorageBackend returns False when flag not set."""
252
- storage = InMemoryStorageBackend()
253
-
254
- assert await storage.check_cancellation_flag("run_123") is False
255
-
256
- @pytest.mark.asyncio
257
- async def test_memory_storage_clear_cancellation_flag(self):
258
- """Test InMemoryStorageBackend clear_cancellation_flag."""
259
- storage = InMemoryStorageBackend()
260
-
261
- await storage.set_cancellation_flag("run_123")
262
- assert await storage.check_cancellation_flag("run_123") is True
263
-
264
- await storage.clear_cancellation_flag("run_123")
265
- assert await storage.check_cancellation_flag("run_123") is False
266
-
267
- @pytest.mark.asyncio
268
- async def test_memory_storage_clear_nonexistent_flag(self):
269
- """Test clearing a non-existent flag does not raise."""
270
- storage = InMemoryStorageBackend()
271
-
272
- # Should not raise
273
- await storage.clear_cancellation_flag("run_nonexistent")
274
-
275
-
276
- class TestCancellationEvents:
277
- """Test cancellation event creation."""
278
-
279
- def test_create_cancellation_requested_event(self):
280
- """Test create_cancellation_requested_event."""
281
- event = create_cancellation_requested_event(
282
- run_id="run_123",
283
- reason="User requested",
284
- requested_by="admin",
285
- )
286
-
287
- assert event.run_id == "run_123"
288
- assert event.type == EventType.CANCELLATION_REQUESTED
289
- assert event.data["reason"] == "User requested"
290
- assert event.data["requested_by"] == "admin"
291
-
292
- def test_create_cancellation_requested_event_minimal(self):
293
- """Test create_cancellation_requested_event with minimal params."""
294
- event = create_cancellation_requested_event(run_id="run_123")
295
-
296
- assert event.run_id == "run_123"
297
- assert event.type == EventType.CANCELLATION_REQUESTED
298
- assert event.data.get("reason") is None
299
- assert event.data.get("requested_by") is None
300
-
301
- def test_create_workflow_cancelled_event(self):
302
- """Test create_workflow_cancelled_event."""
303
- event = create_workflow_cancelled_event(
304
- run_id="run_123",
305
- reason="Test cancellation",
306
- cleanup_completed=True,
307
- )
308
-
309
- assert event.run_id == "run_123"
310
- assert event.type == EventType.WORKFLOW_CANCELLED
311
- assert event.data["reason"] == "Test cancellation"
312
- assert event.data["cleanup_completed"] is True
313
-
314
- def test_create_workflow_cancelled_event_minimal(self):
315
- """Test create_workflow_cancelled_event with minimal params."""
316
- event = create_workflow_cancelled_event(run_id="run_123")
317
-
318
- assert event.run_id == "run_123"
319
- assert event.type == EventType.WORKFLOW_CANCELLED
320
- assert event.data.get("cleanup_completed") is False
321
-
322
- def test_create_step_cancelled_event(self):
323
- """Test create_step_cancelled_event."""
324
- event = create_step_cancelled_event(
325
- run_id="run_123",
326
- step_id="step_456",
327
- step_name="my_step",
328
- )
329
-
330
- assert event.run_id == "run_123"
331
- assert event.type == EventType.STEP_CANCELLED
332
- assert event.data["step_id"] == "step_456"
333
- assert event.data["step_name"] == "my_step"
334
-
335
-
336
- class TestMockContextCancellation:
337
- """Test MockContext cancellation support."""
338
-
339
- def test_mock_context_cancellation_not_requested_by_default(self):
340
- """Test MockContext starts with cancellation not requested."""
341
- ctx = MockContext(run_id="test", workflow_name="test")
342
- assert ctx.is_cancellation_requested() is False
343
-
344
- def test_mock_context_request_cancellation(self):
345
- """Test MockContext can request cancellation."""
346
- ctx = MockContext(run_id="test", workflow_name="test")
347
- ctx.request_cancellation()
348
- assert ctx.is_cancellation_requested() is True
349
-
350
- def test_mock_context_check_cancellation(self):
351
- """Test MockContext check_cancellation raises when requested."""
352
- ctx = MockContext(run_id="test", workflow_name="test")
353
- ctx.request_cancellation(reason="Test")
354
-
355
- with pytest.raises(CancellationError):
356
- ctx.check_cancellation()
357
-
358
- def test_mock_context_cancellation_blocked(self):
359
- """Test MockContext cancellation blocked property."""
360
- ctx = MockContext(run_id="test", workflow_name="test")
361
- assert ctx.cancellation_blocked is False
362
-
363
- ctx._cancellation_blocked = True
364
- assert ctx.cancellation_blocked is True