pyworkflow-engine 0.1.7__py3-none-any.whl → 0.1.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyworkflow/__init__.py +10 -1
- pyworkflow/celery/tasks.py +272 -24
- pyworkflow/cli/__init__.py +4 -1
- pyworkflow/cli/commands/runs.py +4 -4
- pyworkflow/cli/commands/setup.py +203 -4
- pyworkflow/cli/utils/config_generator.py +76 -3
- pyworkflow/cli/utils/docker_manager.py +232 -0
- pyworkflow/context/__init__.py +13 -0
- pyworkflow/context/base.py +26 -0
- pyworkflow/context/local.py +80 -0
- pyworkflow/context/step_context.py +295 -0
- pyworkflow/core/registry.py +6 -1
- pyworkflow/core/step.py +141 -0
- pyworkflow/core/workflow.py +56 -0
- pyworkflow/engine/events.py +30 -0
- pyworkflow/engine/replay.py +39 -0
- pyworkflow/primitives/child_workflow.py +1 -1
- pyworkflow/runtime/local.py +1 -1
- pyworkflow/storage/__init__.py +14 -0
- pyworkflow/storage/base.py +35 -0
- pyworkflow/storage/cassandra.py +1747 -0
- pyworkflow/storage/config.py +69 -0
- pyworkflow/storage/dynamodb.py +31 -2
- pyworkflow/storage/file.py +28 -0
- pyworkflow/storage/memory.py +18 -0
- pyworkflow/storage/mysql.py +1159 -0
- pyworkflow/storage/postgres.py +27 -2
- pyworkflow/storage/schemas.py +4 -3
- pyworkflow/storage/sqlite.py +25 -2
- {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/METADATA +7 -4
- pyworkflow_engine-0.1.9.dist-info/RECORD +91 -0
- pyworkflow_engine-0.1.9.dist-info/top_level.txt +1 -0
- dashboard/backend/app/__init__.py +0 -1
- dashboard/backend/app/config.py +0 -32
- dashboard/backend/app/controllers/__init__.py +0 -6
- dashboard/backend/app/controllers/run_controller.py +0 -86
- dashboard/backend/app/controllers/workflow_controller.py +0 -33
- dashboard/backend/app/dependencies/__init__.py +0 -5
- dashboard/backend/app/dependencies/storage.py +0 -50
- dashboard/backend/app/repositories/__init__.py +0 -6
- dashboard/backend/app/repositories/run_repository.py +0 -80
- dashboard/backend/app/repositories/workflow_repository.py +0 -27
- dashboard/backend/app/rest/__init__.py +0 -8
- dashboard/backend/app/rest/v1/__init__.py +0 -12
- dashboard/backend/app/rest/v1/health.py +0 -33
- dashboard/backend/app/rest/v1/runs.py +0 -133
- dashboard/backend/app/rest/v1/workflows.py +0 -41
- dashboard/backend/app/schemas/__init__.py +0 -23
- dashboard/backend/app/schemas/common.py +0 -16
- dashboard/backend/app/schemas/event.py +0 -24
- dashboard/backend/app/schemas/hook.py +0 -25
- dashboard/backend/app/schemas/run.py +0 -54
- dashboard/backend/app/schemas/step.py +0 -28
- dashboard/backend/app/schemas/workflow.py +0 -31
- dashboard/backend/app/server.py +0 -87
- dashboard/backend/app/services/__init__.py +0 -6
- dashboard/backend/app/services/run_service.py +0 -240
- dashboard/backend/app/services/workflow_service.py +0 -155
- dashboard/backend/main.py +0 -18
- docs/concepts/cancellation.mdx +0 -362
- docs/concepts/continue-as-new.mdx +0 -434
- docs/concepts/events.mdx +0 -266
- docs/concepts/fault-tolerance.mdx +0 -370
- docs/concepts/hooks.mdx +0 -552
- docs/concepts/limitations.mdx +0 -167
- docs/concepts/schedules.mdx +0 -775
- docs/concepts/sleep.mdx +0 -312
- docs/concepts/steps.mdx +0 -301
- docs/concepts/workflows.mdx +0 -255
- docs/guides/cli.mdx +0 -942
- docs/guides/configuration.mdx +0 -560
- docs/introduction.mdx +0 -155
- docs/quickstart.mdx +0 -279
- examples/__init__.py +0 -1
- examples/celery/__init__.py +0 -1
- examples/celery/durable/docker-compose.yml +0 -55
- examples/celery/durable/pyworkflow.config.yaml +0 -12
- examples/celery/durable/workflows/__init__.py +0 -122
- examples/celery/durable/workflows/basic.py +0 -87
- examples/celery/durable/workflows/batch_processing.py +0 -102
- examples/celery/durable/workflows/cancellation.py +0 -273
- examples/celery/durable/workflows/child_workflow_patterns.py +0 -240
- examples/celery/durable/workflows/child_workflows.py +0 -202
- examples/celery/durable/workflows/continue_as_new.py +0 -260
- examples/celery/durable/workflows/fault_tolerance.py +0 -210
- examples/celery/durable/workflows/hooks.py +0 -211
- examples/celery/durable/workflows/idempotency.py +0 -112
- examples/celery/durable/workflows/long_running.py +0 -99
- examples/celery/durable/workflows/retries.py +0 -101
- examples/celery/durable/workflows/schedules.py +0 -209
- examples/celery/transient/01_basic_workflow.py +0 -91
- examples/celery/transient/02_fault_tolerance.py +0 -257
- examples/celery/transient/__init__.py +0 -20
- examples/celery/transient/pyworkflow.config.yaml +0 -25
- examples/local/__init__.py +0 -1
- examples/local/durable/01_basic_workflow.py +0 -94
- examples/local/durable/02_file_storage.py +0 -132
- examples/local/durable/03_retries.py +0 -169
- examples/local/durable/04_long_running.py +0 -119
- examples/local/durable/05_event_log.py +0 -145
- examples/local/durable/06_idempotency.py +0 -148
- examples/local/durable/07_hooks.py +0 -334
- examples/local/durable/08_cancellation.py +0 -233
- examples/local/durable/09_child_workflows.py +0 -198
- examples/local/durable/10_child_workflow_patterns.py +0 -265
- examples/local/durable/11_continue_as_new.py +0 -249
- examples/local/durable/12_schedules.py +0 -198
- examples/local/durable/__init__.py +0 -1
- examples/local/transient/01_quick_tasks.py +0 -87
- examples/local/transient/02_retries.py +0 -130
- examples/local/transient/03_sleep.py +0 -141
- examples/local/transient/__init__.py +0 -1
- pyworkflow_engine-0.1.7.dist-info/RECORD +0 -196
- pyworkflow_engine-0.1.7.dist-info/top_level.txt +0 -5
- tests/examples/__init__.py +0 -0
- tests/integration/__init__.py +0 -0
- tests/integration/test_cancellation.py +0 -330
- tests/integration/test_child_workflows.py +0 -439
- tests/integration/test_continue_as_new.py +0 -428
- tests/integration/test_dynamodb_storage.py +0 -1146
- tests/integration/test_fault_tolerance.py +0 -369
- tests/integration/test_schedule_storage.py +0 -484
- tests/unit/__init__.py +0 -0
- tests/unit/backends/__init__.py +0 -1
- tests/unit/backends/test_dynamodb_storage.py +0 -1554
- tests/unit/backends/test_postgres_storage.py +0 -1281
- tests/unit/backends/test_sqlite_storage.py +0 -1460
- tests/unit/conftest.py +0 -41
- tests/unit/test_cancellation.py +0 -364
- tests/unit/test_child_workflows.py +0 -680
- tests/unit/test_continue_as_new.py +0 -441
- tests/unit/test_event_limits.py +0 -316
- tests/unit/test_executor.py +0 -320
- tests/unit/test_fault_tolerance.py +0 -334
- tests/unit/test_hooks.py +0 -495
- tests/unit/test_registry.py +0 -261
- tests/unit/test_replay.py +0 -420
- tests/unit/test_schedule_schemas.py +0 -285
- tests/unit/test_schedule_utils.py +0 -286
- tests/unit/test_scheduled_workflow.py +0 -274
- tests/unit/test_step.py +0 -353
- tests/unit/test_workflow.py +0 -243
- {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/WHEEL +0 -0
- {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/entry_points.txt +0 -0
- {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,330 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Integration tests for cancellation feature.
|
|
3
|
-
|
|
4
|
-
Tests cover:
|
|
5
|
-
- cancel_workflow() function
|
|
6
|
-
- Cancellation during step execution
|
|
7
|
-
- Cancellation during sleep
|
|
8
|
-
- Cancellation during hook wait
|
|
9
|
-
- Shield prevents cancellation
|
|
10
|
-
- Workflow catches CancellationError for cleanup
|
|
11
|
-
"""
|
|
12
|
-
|
|
13
|
-
import asyncio
|
|
14
|
-
|
|
15
|
-
import pytest
|
|
16
|
-
|
|
17
|
-
from pyworkflow import (
|
|
18
|
-
CancellationError,
|
|
19
|
-
RunStatus,
|
|
20
|
-
cancel_workflow,
|
|
21
|
-
shield,
|
|
22
|
-
step,
|
|
23
|
-
)
|
|
24
|
-
from pyworkflow.engine.events import EventType
|
|
25
|
-
from pyworkflow.storage.memory import InMemoryStorageBackend
|
|
26
|
-
from pyworkflow.storage.schemas import WorkflowRun
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
class TestCancelWorkflowFunction:
|
|
30
|
-
"""Test cancel_workflow() function."""
|
|
31
|
-
|
|
32
|
-
@pytest.mark.asyncio
|
|
33
|
-
async def test_cancel_running_workflow(self):
|
|
34
|
-
"""Test cancelling a running workflow."""
|
|
35
|
-
storage = InMemoryStorageBackend()
|
|
36
|
-
|
|
37
|
-
# Create a workflow run record
|
|
38
|
-
run = WorkflowRun(
|
|
39
|
-
run_id="run_123",
|
|
40
|
-
workflow_name="test_workflow",
|
|
41
|
-
status=RunStatus.RUNNING,
|
|
42
|
-
)
|
|
43
|
-
await storage.create_run(run)
|
|
44
|
-
|
|
45
|
-
# Cancel the workflow
|
|
46
|
-
result = await cancel_workflow(
|
|
47
|
-
run_id="run_123",
|
|
48
|
-
reason="User cancelled",
|
|
49
|
-
storage=storage,
|
|
50
|
-
)
|
|
51
|
-
|
|
52
|
-
assert result is True
|
|
53
|
-
|
|
54
|
-
# Check cancellation flag was set
|
|
55
|
-
assert await storage.check_cancellation_flag("run_123") is True
|
|
56
|
-
|
|
57
|
-
# Check cancellation event was recorded
|
|
58
|
-
events = await storage.get_events("run_123")
|
|
59
|
-
cancellation_events = [e for e in events if e.type == EventType.CANCELLATION_REQUESTED]
|
|
60
|
-
assert len(cancellation_events) == 1
|
|
61
|
-
assert cancellation_events[0].data["reason"] == "User cancelled"
|
|
62
|
-
|
|
63
|
-
@pytest.mark.asyncio
|
|
64
|
-
async def test_cancel_suspended_workflow(self):
|
|
65
|
-
"""Test cancelling a suspended workflow marks it as cancelled."""
|
|
66
|
-
storage = InMemoryStorageBackend()
|
|
67
|
-
|
|
68
|
-
# Create a suspended workflow
|
|
69
|
-
run = WorkflowRun(
|
|
70
|
-
run_id="run_456",
|
|
71
|
-
workflow_name="test_workflow",
|
|
72
|
-
status=RunStatus.SUSPENDED,
|
|
73
|
-
)
|
|
74
|
-
await storage.create_run(run)
|
|
75
|
-
|
|
76
|
-
# Cancel the workflow
|
|
77
|
-
result = await cancel_workflow(
|
|
78
|
-
run_id="run_456",
|
|
79
|
-
storage=storage,
|
|
80
|
-
)
|
|
81
|
-
|
|
82
|
-
assert result is True
|
|
83
|
-
|
|
84
|
-
# Suspended workflows should be marked as cancelled immediately
|
|
85
|
-
updated_run = await storage.get_run("run_456")
|
|
86
|
-
assert updated_run.status == RunStatus.CANCELLED
|
|
87
|
-
|
|
88
|
-
@pytest.mark.asyncio
|
|
89
|
-
async def test_cancel_completed_workflow_returns_false(self):
|
|
90
|
-
"""Test cancelling an already completed workflow returns False."""
|
|
91
|
-
storage = InMemoryStorageBackend()
|
|
92
|
-
|
|
93
|
-
# Create a completed workflow
|
|
94
|
-
run = WorkflowRun(
|
|
95
|
-
run_id="run_789",
|
|
96
|
-
workflow_name="test_workflow",
|
|
97
|
-
status=RunStatus.COMPLETED,
|
|
98
|
-
)
|
|
99
|
-
await storage.create_run(run)
|
|
100
|
-
|
|
101
|
-
# Try to cancel
|
|
102
|
-
result = await cancel_workflow(
|
|
103
|
-
run_id="run_789",
|
|
104
|
-
storage=storage,
|
|
105
|
-
)
|
|
106
|
-
|
|
107
|
-
assert result is False
|
|
108
|
-
|
|
109
|
-
@pytest.mark.asyncio
|
|
110
|
-
async def test_cancel_failed_workflow_returns_false(self):
|
|
111
|
-
"""Test cancelling an already failed workflow returns False."""
|
|
112
|
-
storage = InMemoryStorageBackend()
|
|
113
|
-
|
|
114
|
-
# Create a failed workflow
|
|
115
|
-
run = WorkflowRun(
|
|
116
|
-
run_id="run_abc",
|
|
117
|
-
workflow_name="test_workflow",
|
|
118
|
-
status=RunStatus.FAILED,
|
|
119
|
-
)
|
|
120
|
-
await storage.create_run(run)
|
|
121
|
-
|
|
122
|
-
# Try to cancel
|
|
123
|
-
result = await cancel_workflow(
|
|
124
|
-
run_id="run_abc",
|
|
125
|
-
storage=storage,
|
|
126
|
-
)
|
|
127
|
-
|
|
128
|
-
assert result is False
|
|
129
|
-
|
|
130
|
-
@pytest.mark.asyncio
|
|
131
|
-
async def test_cancel_already_cancelled_workflow_returns_false(self):
|
|
132
|
-
"""Test cancelling an already cancelled workflow returns False."""
|
|
133
|
-
storage = InMemoryStorageBackend()
|
|
134
|
-
|
|
135
|
-
# Create a cancelled workflow
|
|
136
|
-
run = WorkflowRun(
|
|
137
|
-
run_id="run_def",
|
|
138
|
-
workflow_name="test_workflow",
|
|
139
|
-
status=RunStatus.CANCELLED,
|
|
140
|
-
)
|
|
141
|
-
await storage.create_run(run)
|
|
142
|
-
|
|
143
|
-
# Try to cancel again
|
|
144
|
-
result = await cancel_workflow(
|
|
145
|
-
run_id="run_def",
|
|
146
|
-
storage=storage,
|
|
147
|
-
)
|
|
148
|
-
|
|
149
|
-
assert result is False
|
|
150
|
-
|
|
151
|
-
@pytest.mark.asyncio
|
|
152
|
-
async def test_cancel_nonexistent_workflow(self):
|
|
153
|
-
"""Test cancelling a non-existent workflow raises error."""
|
|
154
|
-
from pyworkflow import WorkflowNotFoundError
|
|
155
|
-
|
|
156
|
-
storage = InMemoryStorageBackend()
|
|
157
|
-
|
|
158
|
-
with pytest.raises(WorkflowNotFoundError):
|
|
159
|
-
await cancel_workflow(
|
|
160
|
-
run_id="nonexistent",
|
|
161
|
-
storage=storage,
|
|
162
|
-
)
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
class TestCancellationCheckPoints:
|
|
166
|
-
"""Test cancellation at various check points."""
|
|
167
|
-
|
|
168
|
-
@pytest.mark.asyncio
|
|
169
|
-
async def test_step_checks_cancellation(self):
|
|
170
|
-
"""Test that step execution checks for cancellation."""
|
|
171
|
-
from pyworkflow.context import LocalContext, reset_context, set_context
|
|
172
|
-
|
|
173
|
-
@step()
|
|
174
|
-
async def my_step():
|
|
175
|
-
return "result"
|
|
176
|
-
|
|
177
|
-
# Create context with cancellation requested
|
|
178
|
-
ctx = LocalContext(
|
|
179
|
-
run_id="test_run",
|
|
180
|
-
workflow_name="test_workflow",
|
|
181
|
-
storage=None,
|
|
182
|
-
durable=False,
|
|
183
|
-
)
|
|
184
|
-
ctx.request_cancellation(reason="Test")
|
|
185
|
-
token = set_context(ctx)
|
|
186
|
-
|
|
187
|
-
try:
|
|
188
|
-
with pytest.raises(CancellationError):
|
|
189
|
-
await my_step()
|
|
190
|
-
finally:
|
|
191
|
-
reset_context(token)
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
class TestShieldIntegration:
|
|
195
|
-
"""Test shield() integration with workflow execution."""
|
|
196
|
-
|
|
197
|
-
@pytest.mark.asyncio
|
|
198
|
-
async def test_shield_allows_cleanup(self):
|
|
199
|
-
"""Test shield() allows cleanup operations to complete."""
|
|
200
|
-
from pyworkflow.context import LocalContext, reset_context, set_context
|
|
201
|
-
|
|
202
|
-
cleanup_completed = False
|
|
203
|
-
|
|
204
|
-
async def cleanup():
|
|
205
|
-
nonlocal cleanup_completed
|
|
206
|
-
await asyncio.sleep(0.01) # Simulate cleanup work
|
|
207
|
-
cleanup_completed = True
|
|
208
|
-
|
|
209
|
-
ctx = LocalContext(
|
|
210
|
-
run_id="test_run",
|
|
211
|
-
workflow_name="test_workflow",
|
|
212
|
-
storage=None,
|
|
213
|
-
durable=False,
|
|
214
|
-
)
|
|
215
|
-
ctx.request_cancellation()
|
|
216
|
-
token = set_context(ctx)
|
|
217
|
-
|
|
218
|
-
try:
|
|
219
|
-
async with shield():
|
|
220
|
-
await cleanup()
|
|
221
|
-
|
|
222
|
-
assert cleanup_completed is True
|
|
223
|
-
finally:
|
|
224
|
-
reset_context(token)
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
class TestFileCancellationFlags:
|
|
228
|
-
"""Test file storage backend cancellation flags."""
|
|
229
|
-
|
|
230
|
-
@pytest.mark.asyncio
|
|
231
|
-
async def test_file_storage_cancellation_flags(self, tmp_path):
|
|
232
|
-
"""Test FileStorageBackend cancellation flag methods."""
|
|
233
|
-
from pyworkflow.storage.file import FileStorageBackend
|
|
234
|
-
|
|
235
|
-
storage = FileStorageBackend(base_path=str(tmp_path / "workflow_data"))
|
|
236
|
-
|
|
237
|
-
# Initially not set
|
|
238
|
-
assert await storage.check_cancellation_flag("run_123") is False
|
|
239
|
-
|
|
240
|
-
# Set the flag
|
|
241
|
-
await storage.set_cancellation_flag("run_123")
|
|
242
|
-
assert await storage.check_cancellation_flag("run_123") is True
|
|
243
|
-
|
|
244
|
-
# Clear the flag
|
|
245
|
-
await storage.clear_cancellation_flag("run_123")
|
|
246
|
-
assert await storage.check_cancellation_flag("run_123") is False
|
|
247
|
-
|
|
248
|
-
@pytest.mark.asyncio
|
|
249
|
-
async def test_file_storage_clear_nonexistent_flag(self, tmp_path):
|
|
250
|
-
"""Test clearing a non-existent flag does not raise."""
|
|
251
|
-
from pyworkflow.storage.file import FileStorageBackend
|
|
252
|
-
|
|
253
|
-
storage = FileStorageBackend(base_path=str(tmp_path / "workflow_data"))
|
|
254
|
-
|
|
255
|
-
# Should not raise
|
|
256
|
-
await storage.clear_cancellation_flag("run_nonexistent")
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
class TestEventReplayCancellation:
|
|
260
|
-
"""Test cancellation state restoration during event replay."""
|
|
261
|
-
|
|
262
|
-
@pytest.mark.asyncio
|
|
263
|
-
async def test_replay_restores_cancellation_state(self):
|
|
264
|
-
"""Test that CANCELLATION_REQUESTED event sets context state during replay."""
|
|
265
|
-
from pyworkflow.context import LocalContext
|
|
266
|
-
from pyworkflow.engine.events import create_cancellation_requested_event
|
|
267
|
-
|
|
268
|
-
storage = InMemoryStorageBackend()
|
|
269
|
-
|
|
270
|
-
# Create run
|
|
271
|
-
run = WorkflowRun(
|
|
272
|
-
run_id="run_123",
|
|
273
|
-
workflow_name="test_workflow",
|
|
274
|
-
status=RunStatus.RUNNING,
|
|
275
|
-
)
|
|
276
|
-
await storage.create_run(run)
|
|
277
|
-
|
|
278
|
-
# Record cancellation event
|
|
279
|
-
event = create_cancellation_requested_event(
|
|
280
|
-
run_id="run_123",
|
|
281
|
-
reason="User cancelled",
|
|
282
|
-
)
|
|
283
|
-
await storage.record_event(event)
|
|
284
|
-
|
|
285
|
-
# Get events
|
|
286
|
-
events = await storage.get_events("run_123")
|
|
287
|
-
|
|
288
|
-
# Create context with event log (should replay events)
|
|
289
|
-
ctx = LocalContext(
|
|
290
|
-
run_id="run_123",
|
|
291
|
-
workflow_name="test_workflow",
|
|
292
|
-
storage=storage,
|
|
293
|
-
event_log=events,
|
|
294
|
-
durable=True,
|
|
295
|
-
)
|
|
296
|
-
|
|
297
|
-
# Context should have cancellation requested from replay
|
|
298
|
-
assert ctx.is_cancellation_requested() is True
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
class TestCancellationErrorHandling:
|
|
302
|
-
"""Test CancellationError handling in workflows."""
|
|
303
|
-
|
|
304
|
-
@pytest.mark.asyncio
|
|
305
|
-
async def test_workflow_can_catch_cancellation_for_cleanup(self):
|
|
306
|
-
"""Test that workflows can catch CancellationError for cleanup."""
|
|
307
|
-
from pyworkflow.context import LocalContext
|
|
308
|
-
|
|
309
|
-
cleanup_called = False
|
|
310
|
-
|
|
311
|
-
async def workflow_with_cleanup():
|
|
312
|
-
nonlocal cleanup_called
|
|
313
|
-
try:
|
|
314
|
-
# Simulate work that would check cancellation
|
|
315
|
-
ctx = LocalContext(
|
|
316
|
-
run_id="test",
|
|
317
|
-
workflow_name="test",
|
|
318
|
-
storage=None,
|
|
319
|
-
durable=False,
|
|
320
|
-
)
|
|
321
|
-
ctx.request_cancellation()
|
|
322
|
-
ctx.check_cancellation()
|
|
323
|
-
except CancellationError:
|
|
324
|
-
cleanup_called = True
|
|
325
|
-
raise
|
|
326
|
-
|
|
327
|
-
with pytest.raises(CancellationError):
|
|
328
|
-
await workflow_with_cleanup()
|
|
329
|
-
|
|
330
|
-
assert cleanup_called is True
|