pyworkflow-engine 0.1.7__py3-none-any.whl → 0.1.10__py3-none-any.whl

tests/integration/test_fault_tolerance.py

@@ -1,369 +0,0 @@
-"""
-Integration tests for fault tolerance features.
-
-Tests cover the full workflow recovery flow after simulated worker failures.
-"""
-
-from datetime import UTC, datetime
-
-import pytest
-
-from pyworkflow import (
-    reset_config,
-    workflow,
-)
-from pyworkflow.engine.events import (
-    EventType,
-    create_step_completed_event,
-    create_workflow_interrupted_event,
-    create_workflow_started_event,
-)
-from pyworkflow.serialization.encoder import serialize, serialize_args, serialize_kwargs
-from pyworkflow.storage.memory import InMemoryStorageBackend
-from pyworkflow.storage.schemas import RunStatus, WorkflowRun
-
-
-@pytest.fixture
-def storage():
-    """Provide a clean in-memory storage backend for each test."""
-    return InMemoryStorageBackend()
-
-
-@pytest.fixture(autouse=True)
-def reset_pyworkflow_config():
-    """Reset configuration before and after each test."""
-    reset_config()
-    yield
-    reset_config()
-
-
-class TestRecoveryDetection:
-    """Tests for detecting recovery scenarios."""
-
-    @pytest.mark.asyncio
-    async def test_detect_running_workflow_as_recovery_scenario(self, storage):
-        """A workflow in RUNNING status should be detected as recovery scenario."""
-        # Create a workflow run that's stuck in RUNNING (simulates worker crash)
-        run = WorkflowRun(
-            run_id="test_run",
-            workflow_name="test_workflow",
-            status=RunStatus.RUNNING,
-            created_at=datetime.now(UTC),
-            started_at=datetime.now(UTC),
-            input_args=serialize_args(),
-            input_kwargs=serialize_kwargs(),
-            recovery_attempts=0,
-            max_recovery_attempts=3,
-            recover_on_worker_loss=True,
-        )
-        await storage.create_run(run)
-
-        # Verify the run is in RUNNING status
-        retrieved_run = await storage.get_run("test_run")
-        assert retrieved_run.status == RunStatus.RUNNING
-        assert retrieved_run.recover_on_worker_loss is True
-
-    @pytest.mark.asyncio
-    async def test_recovery_disabled_workflow(self, storage):
-        """Workflow with recover_on_worker_loss=False should not auto-recover."""
-        run = WorkflowRun(
-            run_id="test_run",
-            workflow_name="test_workflow",
-            status=RunStatus.RUNNING,
-            created_at=datetime.now(UTC),
-            input_args=serialize_args(),
-            input_kwargs=serialize_kwargs(),
-            recovery_attempts=0,
-            max_recovery_attempts=3,
-            recover_on_worker_loss=False,  # Disabled
-        )
-        await storage.create_run(run)
-
-        retrieved_run = await storage.get_run("test_run")
-        assert retrieved_run.recover_on_worker_loss is False
-
-
-class TestInterruptedEventRecording:
-    """Tests for recording WORKFLOW_INTERRUPTED events."""
-
-    @pytest.mark.asyncio
-    async def test_record_interrupted_event(self, storage):
-        """Should record WORKFLOW_INTERRUPTED event on recovery."""
-        # Create workflow run
-        run = WorkflowRun(
-            run_id="test_run",
-            workflow_name="test_workflow",
-            status=RunStatus.RUNNING,
-            created_at=datetime.now(UTC),
-            input_args=serialize_args(),
-            input_kwargs=serialize_kwargs(),
-        )
-        await storage.create_run(run)
-
-        # Record workflow started event
-        start_event = create_workflow_started_event(
-            run_id="test_run",
-            workflow_name="test_workflow",
-            args=serialize_args(),
-            kwargs=serialize_kwargs(),
-        )
-        await storage.record_event(start_event)
-
-        # Record step completed event
-        step_event = create_step_completed_event(
-            run_id="test_run",
-            step_id="step_1",
-            result=serialize(42),
-            step_name="test_step",
-        )
-        await storage.record_event(step_event)
-
-        # Simulate worker crash - record interrupted event
-        interrupted_event = create_workflow_interrupted_event(
-            run_id="test_run",
-            reason="worker_lost",
-            worker_id="worker_1",
-            last_event_sequence=2,
-            error="Worker process terminated",
-            recovery_attempt=1,
-            recoverable=True,
-        )
-        await storage.record_event(interrupted_event)
-
-        # Verify events
-        events = await storage.get_events("test_run")
-        assert len(events) == 3
-
-        # Check interrupted event
-        interrupted = [e for e in events if e.type == EventType.WORKFLOW_INTERRUPTED]
-        assert len(interrupted) == 1
-        assert interrupted[0].data["reason"] == "worker_lost"
-        assert interrupted[0].data["recovery_attempt"] == 1
-
-
-class TestRecoveryAttemptTracking:
-    """Tests for tracking recovery attempts."""
-
-    @pytest.mark.asyncio
-    async def test_increment_recovery_attempts(self, storage):
-        """Should increment recovery_attempts on each recovery."""
-        run = WorkflowRun(
-            run_id="test_run",
-            workflow_name="test_workflow",
-            status=RunStatus.RUNNING,
-            created_at=datetime.now(UTC),
-            input_args=serialize_args(),
-            input_kwargs=serialize_kwargs(),
-            recovery_attempts=0,
-            max_recovery_attempts=3,
-        )
-        await storage.create_run(run)
-
-        # First recovery attempt
-        await storage.update_run_recovery_attempts("test_run", 1)
-        run1 = await storage.get_run("test_run")
-        assert run1.recovery_attempts == 1
-
-        # Second recovery attempt
-        await storage.update_run_recovery_attempts("test_run", 2)
-        run2 = await storage.get_run("test_run")
-        assert run2.recovery_attempts == 2
-
-        # Third recovery attempt
-        await storage.update_run_recovery_attempts("test_run", 3)
-        run3 = await storage.get_run("test_run")
-        assert run3.recovery_attempts == 3
-
-    @pytest.mark.asyncio
-    async def test_max_recovery_attempts_exceeded(self, storage):
-        """Should mark workflow as FAILED when max attempts exceeded."""
-        run = WorkflowRun(
-            run_id="test_run",
-            workflow_name="test_workflow",
-            status=RunStatus.RUNNING,
-            created_at=datetime.now(UTC),
-            input_args=serialize_args(),
-            input_kwargs=serialize_kwargs(),
-            recovery_attempts=3,  # Already at max
-            max_recovery_attempts=3,
-        )
-        await storage.create_run(run)
-
-        # Simulating what would happen when max exceeded
-        await storage.update_run_status(
-            run_id="test_run",
-            status=RunStatus.FAILED,
-            error="Exceeded max recovery attempts (3)",
-        )
-
-        run = await storage.get_run("test_run")
-        assert run.status == RunStatus.FAILED
-        assert "max recovery attempts" in run.error.lower()
-
-
-class TestEventReplayWithInterruption:
-    """Tests for event replay after interruption."""
-
-    @pytest.mark.asyncio
-    async def test_replay_preserves_step_results_after_interruption(self, storage):
-        """Step results should be preserved and replayable after interruption."""
-        from pyworkflow.context import LocalContext
-        from pyworkflow.engine.replay import replay_events
-
-        # Create events simulating a workflow that was interrupted
-        events = [
-            create_workflow_started_event(
-                run_id="test_run",
-                workflow_name="test_workflow",
-                args=serialize_args("arg1"),
-                kwargs=serialize_kwargs(key="value"),
-            ),
-            create_step_completed_event(
-                run_id="test_run",
-                step_id="step_1",
-                result=serialize({"processed": True}),
-                step_name="step_1",
-            ),
-            create_step_completed_event(
-                run_id="test_run",
-                step_id="step_2",
-                result=serialize(100),
-                step_name="step_2",
-            ),
-            create_workflow_interrupted_event(
-                run_id="test_run",
-                reason="worker_lost",
-                recovery_attempt=1,
-            ),
-        ]
-
-        # Assign sequence numbers
-        for i, event in enumerate(events):
-            event.sequence = i + 1
-            await storage.record_event(event)
-
-        # Load events and replay
-        loaded_events = await storage.get_events("test_run")
-
-        ctx = LocalContext(
-            run_id="test_run",
-            workflow_name="test_workflow",
-            storage=storage,
-            event_log=loaded_events,
-            durable=True,
-        )
-
-        await replay_events(ctx, loaded_events)
-
-        # Verify step results are available
-        assert ctx.get_step_result("step_1") == {"processed": True}
-        assert ctx.get_step_result("step_2") == 100
-
-
-class TestStatusTransitions:
-    """Tests for workflow status transitions during recovery."""
-
-    @pytest.mark.asyncio
-    async def test_running_to_interrupted(self, storage):
-        """RUNNING -> INTERRUPTED transition."""
-        run = WorkflowRun(
-            run_id="test_run",
-            workflow_name="test_workflow",
-            status=RunStatus.RUNNING,
-            created_at=datetime.now(UTC),
-            input_args=serialize_args(),
-            input_kwargs=serialize_kwargs(),
-        )
-        await storage.create_run(run)
-
-        await storage.update_run_status(
-            run_id="test_run",
-            status=RunStatus.INTERRUPTED,
-        )
-
-        run = await storage.get_run("test_run")
-        assert run.status == RunStatus.INTERRUPTED
-
-    @pytest.mark.asyncio
-    async def test_interrupted_to_running_on_recovery(self, storage):
-        """INTERRUPTED -> RUNNING transition on recovery."""
-        run = WorkflowRun(
-            run_id="test_run",
-            workflow_name="test_workflow",
-            status=RunStatus.INTERRUPTED,
-            created_at=datetime.now(UTC),
-            input_args=serialize_args(),
-            input_kwargs=serialize_kwargs(),
-            recovery_attempts=1,
-        )
-        await storage.create_run(run)
-
-        # Recovery starts
-        await storage.update_run_status(
-            run_id="test_run",
-            status=RunStatus.RUNNING,
-        )
-
-        run = await storage.get_run("test_run")
-        assert run.status == RunStatus.RUNNING
-
-    @pytest.mark.asyncio
-    async def test_interrupted_to_failed_on_max_attempts(self, storage):
-        """INTERRUPTED -> FAILED when max attempts exceeded."""
-        run = WorkflowRun(
-            run_id="test_run",
-            workflow_name="test_workflow",
-            status=RunStatus.INTERRUPTED,
-            created_at=datetime.now(UTC),
-            input_args=serialize_args(),
-            input_kwargs=serialize_kwargs(),
-            recovery_attempts=3,
-            max_recovery_attempts=3,
-        )
-        await storage.create_run(run)
-
-        await storage.update_run_status(
-            run_id="test_run",
-            status=RunStatus.FAILED,
-            error="Exceeded max recovery attempts",
-        )
-
-        run = await storage.get_run("test_run")
-        assert run.status == RunStatus.FAILED
-
-
-class TestWorkflowDecoratorRecoveryConfig:
-    """Tests for workflow decorator recovery configuration."""
-
-    def test_workflow_decorator_stores_recovery_config(self):
-        """@workflow decorator should store recovery config on wrapper."""
-
-        @workflow(
-            name="test_recovery_config_1",
-            recover_on_worker_loss=True,
-            max_recovery_attempts=5,
-        )
-        async def my_workflow():
-            pass
-
-        assert my_workflow.__workflow_recover_on_worker_loss__ is True
-        assert my_workflow.__workflow_max_recovery_attempts__ == 5
-
-    def test_workflow_decorator_defaults_none(self):
-        """@workflow decorator should default recovery config to None when called with ()."""
-
-        @workflow(name="test_recovery_config_2")
-        async def my_workflow():
-            pass
-
-        assert my_workflow.__workflow_recover_on_worker_loss__ is None
-        assert my_workflow.__workflow_max_recovery_attempts__ is None
-
-    def test_workflow_decorator_disable_recovery(self):
-        """@workflow decorator can disable recovery."""
-
-        @workflow(name="test_recovery_config_3", recover_on_worker_loss=False)
-        async def my_workflow():
-            pass
-
-        assert my_workflow.__workflow_recover_on_worker_loss__ is False