pyworkflow-engine 0.1.7__py3-none-any.whl

examples/local/durable/01_basic_workflow.py

@@ -0,0 +1,94 @@
+"""
+Durable Workflow - Basic Example
+
+This example demonstrates a simple event-sourced workflow using InMemoryStorageBackend.
+- 3-step order processing workflow
+- Events recorded for each step
+- Event log inspection after completion
+- Basic @workflow and @step decorators
+
+Run: python examples/local/durable/01_basic_workflow.py 2>/dev/null
+"""
+
+import asyncio
+
+from pyworkflow import (
+    configure,
+    get_workflow_events,
+    get_workflow_run,
+    reset_config,
+    start,
+    step,
+    workflow,
+)
+from pyworkflow.storage import InMemoryStorageBackend
+
+
+# --- Steps ---
+@step()
+async def process_order(order_id: str) -> dict:
+    """Process the order and validate it."""
+    print(f"  Processing order {order_id}...")
+    return {"order_id": order_id, "status": "processed"}
+
+
+@step()
+async def charge_payment(order: dict, amount: float) -> dict:
+    """Charge the payment for the order."""
+    print(f"  Charging payment: ${amount:.2f}...")
+    return {**order, "charged": amount}
+
+
+@step()
+async def send_notification(order: dict) -> dict:
+    """Send order confirmation notification."""
+    print(f"  Sending notification for order {order['order_id']}...")
+    return {**order, "notified": True}
+
+
+# --- Workflow ---
+@workflow(durable=True, tags=["local", "durable"])
+async def order_workflow(order_id: str, amount: float) -> dict:
+    """Complete order processing workflow."""
+    order = await process_order(order_id)
+    order = await charge_payment(order, amount)
+    order = await send_notification(order)
+    return order
+
+
+async def main():
+    # Configure with InMemoryStorageBackend
+    reset_config()
+    storage = InMemoryStorageBackend()
+    configure(storage=storage, default_durable=True)
+
+    print("=== Durable Workflow - Basic Example ===\n")
+    print("Running order workflow...")
+
+    # Start workflow
+    run_id = await start(order_workflow, "order-123", 99.99)
+    print(f"\nWorkflow completed: {run_id}\n")
+
+    # Check workflow status
+    run = await get_workflow_run(run_id)
+    print(f"Status: {run.status.value}")
+    print(f"Result: {run.result}")
+
+    # Inspect event log
+    events = await get_workflow_events(run_id)
+    print(f"\n=== Event Log ({len(events)} events) ===")
+    for event in events:
+        print(f"  {event.sequence}: {event.type.value}")
+        if event.type.value == "step_completed":
+            step_name = event.data.get("step_name", "unknown")
+            print(f"     Step: {step_name}")
+
+    print("\n=== Key Takeaways ===")
+    print("✓ Workflow executed with event sourcing")
+    print("✓ Each step recorded as an event")
+    print("✓ InMemoryStorageBackend used (data lost on exit)")
+    print("✓ Try 02_file_storage.py for persistence!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/local/durable/02_file_storage.py

@@ -0,0 +1,132 @@
+"""
+Durable Workflow - File Storage
+
+This example demonstrates persistent workflow storage using FileStorageBackend.
+- Same 3-step workflow as 01_basic_workflow.py
+- Data persists to filesystem in workflow_data/ directory
+- Human-readable JSON files
+- Inspect stored file structure and JSONL event log
+
+Run: python examples/local/durable/02_file_storage.py 2>/dev/null
+"""
+
+import asyncio
+import json
+import os
+from pathlib import Path
+
+from pyworkflow import (
+    configure,
+    get_workflow_events,
+    get_workflow_run,
+    reset_config,
+    start,
+    step,
+    workflow,
+)
+from pyworkflow.storage import FileStorageBackend
+
+
+# --- Steps ---
+@step()
+async def process_order(order_id: str) -> dict:
+    """Process the order and validate it."""
+    print(f"  Processing order {order_id}...")
+    return {"order_id": order_id, "status": "processed"}
+
+
+@step()
+async def charge_payment(order: dict, amount: float) -> dict:
+    """Charge the payment for the order."""
+    print(f"  Charging payment: ${amount:.2f}...")
+    return {**order, "charged": amount}
+
+
+@step()
+async def send_notification(order: dict) -> dict:
+    """Send order confirmation notification."""
+    print(f"  Sending notification for order {order['order_id']}...")
+    return {**order, "notified": True}
+
+
+# --- Workflow ---
+@workflow(durable=True, tags=["local", "durable"])
+async def order_workflow(order_id: str, amount: float) -> dict:
+    """Complete order processing workflow."""
+    order = await process_order(order_id)
+    order = await charge_payment(order, amount)
+    order = await send_notification(order)
+    return order
+
+
+async def main():
+    # Use local directory for persistence (added to .gitignore)
+    data_dir = Path(__file__).parent / "workflow_data"
+    data_dir.mkdir(exist_ok=True)
+
+    print("=== Durable Workflow - File Storage ===\n")
+    print(f"Storage directory: {data_dir}\n")
+
+    # Configure with FileStorageBackend
+    reset_config()
+    storage = FileStorageBackend(base_path=str(data_dir))
+    configure(storage=storage, default_durable=True)
+
+    print("Running order workflow...")
+
+    # Start workflow
+    run_id = await start(order_workflow, "order-456", 149.99)
+    print(f"\nWorkflow completed: {run_id}\n")
+
+    # Check workflow status
+    run = await get_workflow_run(run_id)
+    print(f"Status: {run.status.value}")
+    print(f"Result: {run.result}")
+
+    # Show events
+    events = await get_workflow_events(run_id)
+    print(f"\n=== Event Log ({len(events)} events) ===")
+    for event in events:
+        print(f"  {event.sequence}: {event.type.value}")
+
+    # Show stored files
+    print("\n=== Stored Files ===")
+    for root, dirs, files in os.walk(data_dir):
+        # Skip hidden directories (.locks)
+        dirs[:] = [d for d in dirs if not d.startswith(".")]
+        for f in files:
+            path = os.path.join(root, f)
+            rel_path = os.path.relpath(path, data_dir)
+            size = os.path.getsize(path)
+            print(f"  {rel_path} ({size} bytes)")
+
+    # Show JSONL event log contents
+    event_log_path = data_dir / "events" / f"{run_id}.jsonl"
+    if event_log_path.exists():
+        print(f"\n=== Event Log File Contents ({event_log_path.name}) ===")
+        with open(event_log_path) as f:
+            for i, line in enumerate(f, 1):
+                event_data = json.loads(line.strip())
+                event_type = event_data.get("type", "unknown")
+                print(f"  Line {i}: {event_type}")
+                # Show full data for first event
+                if i == 1:
+                    print(f"    Full data: {json.dumps(event_data, indent=6)}")
+
+    print("\n=== Directory Structure ===")
+    print("  runs/       - Workflow run metadata (JSON)")
+    print("  events/     - Event log (JSONL, append-only)")
+    print("  steps/      - Step execution records (JSON)")
+    print("  .locks/     - Internal file locks")
+
+    print("\n=== Key Takeaways ===")
+    print("✓ Data persists to filesystem in workflow_data/")
+    print("✓ Human-readable JSON format")
+    print("✓ JSONL (JSON Lines) for event log (one event per line)")
+    print("✓ Survives process restarts")
+    print("✓ Good for development and single-machine deployments")
+    print(f"\nℹ  Storage persisted at: {data_dir.absolute()}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/local/durable/03_retries.py

@@ -0,0 +1,169 @@
+"""
+Durable Workflow - Automatic Retries with Suspension
+
+This example demonstrates automatic retry behavior with workflow suspension/resumption.
+- Simulates flaky external API (fails 2x, succeeds on 3rd try)
+- Workflow suspends between retry attempts
+- Retry events recorded in event log for audit trail
+- Demonstrates manual resume after each retry delay
+
+IMPORTANT: Manual resumption (await resume(run_id)) is ONLY for local development/CI.
+In production, use Celery runtime for automatic scheduled resumption.
+See examples/celery/ for production-ready distributed execution.
+
+Run: python examples/local/durable/03_retries.py 2>/dev/null
+"""
+
+import asyncio
+
+from pyworkflow import (
+    configure,
+    get_workflow_events,
+    get_workflow_run,
+    reset_config,
+    resume,
+    start,
+    step,
+    workflow,
+)
+from pyworkflow.storage import InMemoryStorageBackend
+
+# Simulate API call counter
+attempt_count = 0
+
+
+# --- Steps ---
+@step()
+async def validate_order(order_id: str) -> dict:
+    """Validate the order."""
+    print(f"  Validating order {order_id}...")
+    return {"order_id": order_id, "valid": True}
+
+
+@step(max_retries=3, retry_delay=1)
+async def call_flaky_api(order: dict) -> dict:
+    """Simulate unreliable external API - fails twice then succeeds."""
+    global attempt_count
+    attempt_count += 1
+
+    print(f"  Calling external API (attempt {attempt_count})...")
+
+    if attempt_count < 3:
+        # Simulate temporary failure
+        raise Exception(f"API timeout - connection refused (attempt {attempt_count})")
+
+    # Third attempt succeeds
+    print(f"  ✓ API call successful on attempt {attempt_count}!")
+    return {**order, "api_response": "payment_approved", "attempts": attempt_count}
+
+
+@step()
+async def finalize_order(order: dict) -> dict:
+    """Finalize the order after successful API call."""
+    print(f"  Finalizing order {order['order_id']}...")
+    return {**order, "finalized": True}
+
+
+# --- Workflow ---
+@workflow(durable=True, tags=["local", "durable"])
+async def order_workflow(order_id: str) -> dict:
+    """Complete order processing with retry logic."""
+    order = await validate_order(order_id)
+    order = await call_flaky_api(order)  # Will retry on failure
+    order = await finalize_order(order)
+    return order
+
+
+async def main():
+    global attempt_count
+
+    # Configure with InMemoryStorageBackend
+    reset_config()
+    storage = InMemoryStorageBackend()
+    configure(storage=storage, default_durable=True)
+
+    print("=== Durable Workflow - Automatic Retries with Suspension ===\n")
+    print("Simulating flaky API (fails 2x, succeeds on 3rd try)...\n")
+
+    # Reset counter
+    attempt_count = 0
+
+    # Start workflow
+    print("Starting workflow...")
+    run_id = await start(order_workflow, "order-789")
+
+    # Check status after first attempt
+    run = await get_workflow_run(run_id)
+    print(f"\nStatus after attempt 1: {run.status.value}")
+
+    if run.status.value == "suspended":
+        print("→ Workflow suspended for retry (waiting 1 second...)")
+
+        # Show events so far
+        events = await get_workflow_events(run_id)
+        print(f"\n=== Event Log (After Attempt 1) - {len(events)} events ===")
+        for event in events:
+            event_type = event.type.value
+            attempt = event.data.get("attempt", "?")
+            print(f"  {event.sequence}: {event_type} (attempt={attempt})")
+            if event_type == "step_retrying":
+                next_attempt = event.data.get("attempt")
+                print(f"     → Will retry as attempt {next_attempt}")
+
+        # Wait for retry delay and resume
+        await asyncio.sleep(1.5)
+        print("\nResuming workflow for attempt 2...")
+        await resume(run_id)
+
+        # Check status again
+        run = await get_workflow_run(run_id)
+        print(f"Status after attempt 2: {run.status.value}")
+
+        if run.status.value == "suspended":
+            print("→ Workflow suspended for retry again (waiting 1 second...)")
+
+            # Wait and resume for attempt 3
+            await asyncio.sleep(1.5)
+            print("\nResuming workflow for attempt 3...")
+            result = await resume(run_id)
+
+            print("\n✓ Workflow completed successfully!")
+            print(f"Result: {result}")
+
+    # Final status
+    run = await get_workflow_run(run_id)
+    print(f"\nFinal status: {run.status.value}")
+
+    # Show complete event log
+    events = await get_workflow_events(run_id)
+    print(f"\n=== Complete Event Log ({len(events)} events) ===")
+
+    for event in events:
+        event_type = event.type.value
+        attempt = event.data.get("attempt", "")
+        step_name = event.data.get("step_name", "")
+
+        if attempt:
+            print(f"  {event.sequence}: {event_type} (attempt={attempt}, step={step_name})")
+        else:
+            print(f"  {event.sequence}: {event_type}")
+
+        if event_type == "step_failed":
+            error = event.data.get("error", "")[:50]
+            print(f"     Error: {error}...")
+        elif event_type == "step_retrying":
+            retry_after = event.data.get("retry_after")
+            resume_at = event.data.get("resume_at", "")[:19]
+            print(f"     Retry after: {retry_after}s, resume at: {resume_at}")
+
+    print("\n=== Key Takeaways ===")
+    print("✓ Workflow suspends between retry attempts (releases resources)")
+    print("✓ Each retry requires manual resume() or automatic Celery scheduling")
+    print("✓ Event log shows STEP_FAILED + STEP_RETRYING for each retry")
+    print("✓ Resume restores state via event replay and continues from retry")
+    print("✓ max_retries=3, retry_delay=1 (1 initial + 3 retries = 4 total attempts)")
+    print(f"✓ Total attempts in this run: {attempt_count}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/local/durable/04_long_running.py

@@ -0,0 +1,119 @@
+"""
+Durable Workflow - Long Running with Sleep
+
+This example demonstrates workflow suspension and resumption with sleep().
+- Workflow suspends during sleep (releases resources)
+- Can be resumed after sleep completes
+- Uses FileStorageBackend for persistence across process restarts
+- Demonstrates manual resumption pattern
+
+IMPORTANT: Manual resumption (await resume(run_id)) is ONLY for local development/CI.
+In production, use Celery runtime for automatic scheduled resumption.
+See examples/celery/ for production-ready distributed execution.
+
+Run: python examples/local/durable/04_long_running.py 2>/dev/null
+"""
+
+import asyncio
+import tempfile
+
+from pyworkflow import (
+    configure,
+    get_workflow_run,
+    reset_config,
+    resume,
+    sleep,
+    start,
+    step,
+    workflow,
+)
+from pyworkflow.storage import FileStorageBackend
+
+
+# --- Steps ---
+@step()
+async def prepare_batch(batch_id: str) -> dict:
+    """Prepare the batch for processing."""
+    print(f"  Preparing batch {batch_id}...")
+    return {"batch_id": batch_id, "status": "prepared"}
+
+
+@step()
+async def process_batch(batch: dict) -> dict:
+    """Process the batch after sleep completes."""
+    print(f"  Processing batch {batch['batch_id']}...")
+    return {**batch, "status": "processed", "items": 1000}
+
+
+@step()
+async def finalize_batch(batch: dict) -> dict:
+    """Finalize the batch."""
+    print(f"  Finalizing batch {batch['batch_id']}...")
+    return {**batch, "status": "completed"}
+
+
+# --- Workflow ---
+@workflow(durable=True, tags=["local", "durable"])
+async def batch_workflow(batch_id: str) -> dict:
+    """Long-running batch processing workflow with sleep."""
+    batch = await prepare_batch(batch_id)
+
+    print("  Sleeping for 5 seconds (workflow will suspend)...")
+    await sleep("5s")  # Suspends workflow here
+
+    print("  Resuming after sleep...")
+    batch = await process_batch(batch)
+    batch = await finalize_batch(batch)
+    return batch
+
+
+async def main():
+    # Use temp directory (use real path like "./workflow_data" for production)
+    with tempfile.TemporaryDirectory() as tmpdir:
+        print("=== Durable Workflow - Long Running ===\n")
+
+        # Configure with FileStorageBackend (for persistence)
+        reset_config()
+        storage = FileStorageBackend(base_path=tmpdir)
+        configure(storage=storage, default_durable=True)
+
+        print("Starting batch workflow...\n")
+
+        # Start workflow
+        run_id = await start(batch_workflow, "batch-001")
+
+        # Check status after start
+        run = await get_workflow_run(run_id)
+        print(f"\nWorkflow status after sleep: {run.status.value}")
+
+        if run.status.value == "suspended":
+            print("Workflow is suspended (waiting for sleep to complete)")
+
+            # Wait for sleep duration, then resume
+            print("\nWaiting 5 seconds for sleep to complete...")
+            await asyncio.sleep(5)
+
+            print(f"Resuming workflow {run_id}...\n")
+            result = await resume(run_id)
+
+            print("\nWorkflow completed!")
+            print(f"Result: {result}")
+        else:
+            # Workflow already completed (sleep was short enough)
+            print("Workflow completed without suspension")
+            print(f"Result: {run.result}")
+
+        # Final status check
+        run = await get_workflow_run(run_id)
+        print(f"\nFinal status: {run.status.value}")
+
+        print("\n=== Key Takeaways ===")
+        print("✓ Workflow suspends during sleep() (releases resources)")
+        print("✓ FileStorageBackend persists state during suspension")
+        print("✓ Can resume after sleep completes (even after process restart)")
+        print("✓ Perfect for rate limiting, scheduled tasks, waiting for events")
+        print("\nℹ  For production: use real storage path, implement auto-resume logic")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/local/durable/05_event_log.py

@@ -0,0 +1,145 @@
+"""
+Durable Workflow - Event Log Deep Dive
+
+This example demonstrates detailed event sourcing inspection.
+- Multiple workflows to show different event sequences
+- Deep dive into event structure (sequence, type, timestamp, data)
+- Understanding event types and their meaning
+- Event replay concepts
+
+Run: python examples/local/durable/05_event_log.py 2>/dev/null
+"""
+
+import asyncio
+
+from pyworkflow import (
+    configure,
+    get_workflow_events,
+    get_workflow_run,
+    reset_config,
+    start,
+    step,
+    workflow,
+)
+from pyworkflow.storage import InMemoryStorageBackend
+
+
+# --- Steps ---
+@step()
+async def step_a(value: int) -> int:
+    """Simple step that doubles the value."""
+    return value * 2
+
+
+@step()
+async def step_b(value: int) -> int:
+    """Simple step that adds 10."""
+    return value + 10
+
+
+@step()
+async def step_c(value: int) -> int:
+    """Simple step that subtracts 5."""
+    return value - 5
+
+
+# --- Workflows ---
+@workflow(durable=True, tags=["local", "durable"])
+async def simple_workflow(value: int) -> int:
+    """Simple 2-step workflow."""
+    result = await step_a(value)
+    result = await step_b(result)
+    return result
+
+
+@workflow(durable=True, tags=["local", "durable"])
+async def complex_workflow(value: int) -> int:
+    """More complex 3-step workflow."""
+    result = await step_a(value)
+    result = await step_b(result)
+    result = await step_c(result)
+    return result
+
+
+def print_event_details(event, index: int):
+    """Pretty print event details."""
+    print(f"\nEvent #{index + 1}:")
+    print(f"  Sequence: {event.sequence}")
+    print(f"  Type: {event.type.value}")
+    print(f"  Timestamp: {event.timestamp.strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]}")
+
+    if event.data:
+        print("  Data:")
+        for key, value in event.data.items():
+            # Format value nicely
+            if isinstance(value, str) and len(value) > 50:
+                value = value[:50] + "..."
+            print(f"    {key}: {value}")
+
+
+async def main():
+    # Configure with InMemoryStorageBackend
+    reset_config()
+    storage = InMemoryStorageBackend()
+    configure(storage=storage, default_durable=True)
+
+    print("=== Durable Workflow - Event Log Deep Dive ===\n")
+
+    # Run first workflow
+    print("Running simple_workflow(5)...\n")
+    run_id_1 = await start(simple_workflow, 5)
+
+    run = await get_workflow_run(run_id_1)
+    print(f"Result: {run.result}")
+    print(f"Status: {run.status.value}")
+
+    # Inspect events
+    events = await get_workflow_events(run_id_1)
+    print(f"\n=== Event Log for simple_workflow ({len(events)} events) ===")
+
+    for i, event in enumerate(events):
+        print_event_details(event, i)
+
+    # Run second workflow
+    print("\n" + "=" * 60)
+    print("\nRunning complex_workflow(10)...\n")
+    run_id_2 = await start(complex_workflow, 10)
+
+    run = await get_workflow_run(run_id_2)
+    print(f"Result: {run.result}")
+    print(f"Status: {run.status.value}")
+
+    # Inspect events
+    events = await get_workflow_events(run_id_2)
+    print(f"\n=== Event Log for complex_workflow ({len(events)} events) ===")
+
+    for i, event in enumerate(events):
+        print_event_details(event, i)
+
+    # Event type summary
+    print("\n" + "=" * 60)
+    print("\n=== Event Types Explained ===")
+    print("workflow_started   - Workflow execution begins")
+    print("step_completed     - Step successfully executed (result cached)")
+    print("step_failed        - Step failed (will retry if configured)")
+    print("sleep_started      - Workflow suspended (sleep begins)")
+    print("sleep_completed    - Workflow resumed (sleep ends)")
+    print("workflow_completed - Workflow finished successfully")
+    print("workflow_failed    - Workflow failed permanently")
+
+    print("\n=== Event Replay Concepts ===")
+    print("✓ Events are immutable - never modified, only appended")
+    print("✓ On crash/restart, events replayed to restore state")
+    print("✓ step_completed events: result cached, step not re-executed")
+    print("✓ Sequence numbers ensure deterministic ordering")
+    print("✓ Timestamps enable time-travel debugging")
+
+    print("\n=== Key Takeaways ===")
+    print("✓ Every state change recorded as an event")
+    print("✓ Events contain sequence, type, timestamp, and data")
+    print("✓ Event log enables crash recovery via replay")
+    print("✓ Complete audit trail for compliance and debugging")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())