pyworkflow-engine 0.1.7__py3-none-any.whl

examples/celery/durable/workflows/child_workflows.py

@@ -0,0 +1,202 @@
+"""
+Celery Durable Workflow - Child Workflows Basic Example
+
+This example demonstrates how to spawn child workflows from a parent workflow.
+- Parent workflow spawns child workflows using start_child_workflow()
+- Children have their own run_id and event history
+- wait_for_completion=True (default) waits for child to complete
+- wait_for_completion=False returns a ChildWorkflowHandle immediately
+- TERMINATE policy: when parent completes/fails/cancels, children are cancelled
+
+Prerequisites:
+    1. Start Redis: docker run -d -p 6379:6379 redis:7-alpine
+    2. Start worker: pyworkflow --module examples.celery.durable.09_child_workflows worker run
+
+Run with CLI:
+    pyworkflow --module examples.celery.durable.09_child_workflows workflows run order_fulfillment_workflow \
+        --arg order_id=order-456 --arg amount=149.99 --arg customer_email=customer@example.com
+
+Check status:
+    pyworkflow runs list
+    pyworkflow runs status <run_id>
+    pyworkflow runs children <run_id>
+"""
+
+from pyworkflow import (
+    ChildWorkflowHandle,
+    start_child_workflow,
+    step,
+    workflow,
+)
+
+
+# --- Steps ---
+@step(name="child_demo_validate_order")
+async def validate_order(order_id: str) -> dict:
+    """Validate order details."""
+    print(f"    Validating order {order_id}...")
+    return {"order_id": order_id, "valid": True}
+
+
+@step(name="child_demo_process_payment")
+async def process_payment(order_id: str, amount: float) -> dict:
+    """Process payment for order."""
+    print(f"    Processing payment ${amount:.2f} for {order_id}...")
+    return {"order_id": order_id, "amount": amount, "paid": True}
+
+
+@step(name="child_demo_ship_order")
+async def ship_order(order_id: str) -> dict:
+    """Ship the order."""
+    print(f"    Shipping order {order_id}...")
+    return {"order_id": order_id, "shipped": True}
+
+
+@step(name="child_demo_send_email")
+async def send_email(recipient: str, subject: str) -> dict:
+    """Send an email notification."""
+    print(f"    Sending email to {recipient}: {subject}")
+    return {"recipient": recipient, "subject": subject, "sent": True}
+
+
+# --- Child Workflows ---
+@workflow(name="child_demo_payment_workflow", tags=["celery", "durable"])
+async def payment_workflow(order_id: str, amount: float) -> dict:
+    """Child workflow for payment processing."""
+    print(f"  [PaymentWorkflow] Starting for order {order_id}")
+    result = await process_payment(order_id, amount)
+    print(f"  [PaymentWorkflow] Completed for order {order_id}")
+    return result
+
+
+@workflow(name="child_demo_shipping_workflow", tags=["celery", "durable"])
+async def shipping_workflow(order_id: str) -> dict:
+    """Child workflow for shipping."""
+    print(f"  [ShippingWorkflow] Starting for order {order_id}")
+    result = await ship_order(order_id)
+    print(f"  [ShippingWorkflow] Completed for order {order_id}")
+    return result
+
+
+@workflow(name="child_demo_notification_workflow", tags=["celery", "durable"])
+async def notification_workflow(email: str, order_id: str) -> dict:
+    """Child workflow for sending notifications."""
+    print(f"  [NotificationWorkflow] Starting for {email}")
+    result = await send_email(email, f"Order {order_id} update")
+    print(f"  [NotificationWorkflow] Completed for {email}")
+    return result
+
+
+# --- Parent Workflow ---
+@workflow(tags=["celery", "durable"])
+async def order_fulfillment_workflow(
+    order_id: str,
+    amount: float,
+    customer_email: str,
+) -> dict:
+    """
+    Parent workflow that orchestrates order fulfillment using child workflows.
+
+    This demonstrates:
+    1. wait_for_completion=True - Wait for child to complete (default)
+    2. wait_for_completion=False - Fire-and-forget with handle
+    """
+    print(f"[OrderFulfillment] Starting for order {order_id}")
+
+    # Step 1: Validate order (regular step)
+    validation = await validate_order(order_id)
+    if not validation["valid"]:
+        return {"order_id": order_id, "status": "invalid"}
+
+    # Step 2: Process payment via child workflow (wait for completion)
+    print("[OrderFulfillment] Starting payment child workflow...")
+    payment_result = await start_child_workflow(
+        payment_workflow,
+        order_id,
+        amount,
+        wait_for_completion=True,  # Default: wait for child to complete
+    )
+    print(f"[OrderFulfillment] Payment completed: {payment_result}")
+
+    # Step 3: Ship order via child workflow (wait for completion)
+    print("[OrderFulfillment] Starting shipping child workflow...")
+    shipping_result = await start_child_workflow(
+        shipping_workflow,
+        order_id,
+        wait_for_completion=True,
+    )
+    print(f"[OrderFulfillment] Shipping completed: {shipping_result}")
+
+    # Step 4: Send notification via fire-and-forget child workflow
+    # This returns immediately with a handle, parent continues
+    print("[OrderFulfillment] Starting notification child workflow (fire-and-forget)...")
+    notification_handle: ChildWorkflowHandle = await start_child_workflow(
+        notification_workflow,
+        customer_email,
+        order_id,
+        wait_for_completion=False,  # Fire-and-forget
+    )
+    print(f"[OrderFulfillment] Notification child started: {notification_handle.child_run_id}")
+
+    result = {
+        "order_id": order_id,
+        "status": "fulfilled",
+        "payment": payment_result,
+        "shipping": shipping_result,
+        "notification_run_id": notification_handle.child_run_id,
+    }
+
+    print(f"[OrderFulfillment] Completed for order {order_id}")
+    return result
+
+
+async def main() -> None:
+    """Run the order fulfillment workflow example."""
+    import argparse
+    import asyncio
+
+    import pyworkflow
+    from pyworkflow import get_workflow_run
+
+    parser = argparse.ArgumentParser(description="Order Fulfillment Workflow with Child Workflows")
+    parser.add_argument("--order-id", default="order-456", help="Order ID to process")
+    parser.add_argument("--amount", type=float, default=149.99, help="Order amount")
+    parser.add_argument("--email", default="customer@example.com", help="Customer email")
+    args = parser.parse_args()
+
+    print("=== Child Workflows - Basic Example ===\n")
+    print("Running order fulfillment workflow with child workflows...\n")
+
+    # Start parent workflow
+    run_id = await pyworkflow.start(
+        order_fulfillment_workflow,
+        args.order_id,
+        args.amount,
+        args.email,
+    )
+
+    print(f"\nWorkflow started with run_id: {run_id}")
+    print("\nCheck status:")
+    print(f"  pyworkflow runs status {run_id}")
+    print(f"  pyworkflow runs children {run_id}")
+
+    # Poll for completion
+    print("\nWaiting for workflow to complete...")
+    for _ in range(30):
+        await asyncio.sleep(1)
+        run = await get_workflow_run(run_id)
+        if run.status.value in ("completed", "failed", "cancelled"):
+            print(f"\nWorkflow {run.status.value}!")
+            if run.result:
+                print(f"Result: {run.result}")
+            if run.error:
+                print(f"Error: {run.error}")
+            break
+    else:
+        print("\nTimeout waiting for workflow completion")
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())

examples/celery/durable/workflows/continue_as_new.py

@@ -0,0 +1,260 @@
+"""
+Durable Workflow (Celery) - Continue-As-New
+
+This example demonstrates continue_as_new() with Celery workers:
+- Polling workflows that need fresh event history
+- Batch processing with continuation
+- Tracking workflow chains across distributed workers
+
+Prerequisites:
+1. Redis running: docker run -d -p 6379:6379 redis
+2. Start Celery worker:
+   celery -A pyworkflow.celery.tasks worker -Q pyworkflow.workflows,pyworkflow.steps,pyworkflow.schedules -l info
+
+Run: python examples/celery/durable/11_continue_as_new.py
+"""
+
+import asyncio
+
+from pyworkflow import (
+    configure,
+    continue_as_new,
+    get_workflow_chain,
+    reset_config,
+    start,
+    step,
+    workflow,
+)
+from pyworkflow.storage import FileStorageBackend
+
+
+# --- Steps ---
+@step()
+async def fetch_batch(offset: int, batch_size: int) -> list:
+    """Fetch a batch of items to process."""
+    # Simulate fetching items - returns empty when done
+    total_items = 50  # Simulate 50 total items
+    if offset >= total_items:
+        return []
+    end = min(offset + batch_size, total_items)
+    items = list(range(offset, end))
+    print(f"  [Step] Fetched items {offset} to {end - 1}")
+    await asyncio.sleep(0.1)  # Simulate I/O
+    return items
+
+
+@step(name="continue_process_item")
+async def process_item(item: int) -> dict:
+    """Process a single item."""
+    await asyncio.sleep(0.05)  # Simulate work
+    return {"item": item, "processed": True}
+
+
+@step()
+async def poll_for_messages(cursor: str | None) -> tuple[str | None, list]:
+    """Poll message queue for new messages."""
+    # Simulate message queue polling
+    await asyncio.sleep(0.1)
+
+    if cursor is None:
+        return "msg_batch_1", [{"id": 1, "type": "order"}, {"id": 2, "type": "payment"}]
+    elif cursor == "msg_batch_1":
+        return "msg_batch_2", [{"id": 3, "type": "shipment"}]
+    elif cursor == "msg_batch_2":
+        return "msg_batch_3", [{"id": 4, "type": "notification"}]
+    else:
+        return None, []  # No more messages
+
+
+@step()
+async def handle_message(message: dict) -> dict:
+    """Handle a single message."""
+    await asyncio.sleep(0.05)
+    return {"message_id": message["id"], "handled": True}
+
+
+# --- Batch Processing Workflow ---
+@workflow(durable=True, tags=["celery", "durable"])
+async def batch_processor(offset: int = 0, batch_size: int = 10) -> str:
+    """
+    Process items in batches using continue_as_new.
+
+    Each batch runs as a separate workflow execution with fresh
+    event history, preventing unbounded history growth.
+
+    This pattern is ideal for:
+    - ETL pipelines processing millions of records
+    - Data migration jobs
+    - Bulk update operations
+    """
+    print(f"\n  [Batch] Starting at offset {offset}")
+
+    items = await fetch_batch(offset, batch_size)
+
+    if not items:
+        return f"Batch processing complete! Total items: {offset}"
+
+    # Process items
+    for item in items:
+        await process_item(item)
+
+    print(f"  [Batch] Processed {len(items)} items")
+
+    # Continue with next batch
+    continue_as_new(offset=offset + batch_size, batch_size=batch_size)
+
+
+# --- Message Consumer Workflow ---
+@workflow(durable=True, tags=["celery", "durable"])
+async def message_consumer(cursor: str | None = None, messages_processed: int = 0) -> str:
+    """
+    Consume messages from a queue, continuing as new after each batch.
+
+    This pattern is useful for:
+    - Queue consumers that run indefinitely
+    - Event stream processors
+    - Real-time data ingestion
+    """
+    print(f"\n  [Consumer] Polling with cursor: {cursor}")
+
+    # Poll for messages
+    new_cursor, messages = await poll_for_messages(cursor)
+
+    if not messages and new_cursor is None:
+        return f"Consumer complete! Processed {messages_processed} messages"
+
+    # Handle each message
+    count = 0
+    for message in messages:
+        await handle_message(message)
+        count += 1
+
+    total = messages_processed + count
+    print(f"  [Consumer] Handled {count} messages (total: {total})")
+
+    # Continue with new cursor
+    continue_as_new(cursor=new_cursor, messages_processed=total)
+
+
+# --- Recurring Task Workflow ---
+@workflow(durable=True, tags=["celery", "durable"])
+async def recurring_report(iteration: int = 1, max_iterations: int = 3) -> str:
+    """
+    Generate reports on a schedule, continuing as new for each iteration.
+
+    This demonstrates a pattern for:
+    - Daily/weekly reports
+    - Scheduled cleanup tasks
+    - Periodic sync operations
+
+    In production, you might add sleep() between iterations.
+    """
+    print(f"\n  [Report] Generating report #{iteration}")
+
+    # Simulate report generation
+    await asyncio.sleep(0.1)
+    print(f"  [Report] Report #{iteration} complete")
+
+    if iteration >= max_iterations:
+        return f"All {max_iterations} reports generated!"
+
+    # Continue with next iteration
+    continue_as_new(iteration=iteration + 1, max_iterations=max_iterations)
+
+
+async def run_examples():
+    """Run all continue-as-new examples."""
+    print("\n=== Continue-As-New Examples (Celery) ===\n")
+
+    # Example 1: Batch Processing
+    print("--- Example 1: Batch Processing ---")
+    print("Processing 50 items in batches of 10...")
+
+    run_id = await start(batch_processor, offset=0, batch_size=10)
+    print(f"Started workflow: {run_id}")
+
+    # Wait for completion (in production, use webhooks or polling)
+    print("Waiting for completion...")
+    await asyncio.sleep(5)
+
+    # Check the chain
+    from pyworkflow import get_storage
+
+    storage = get_storage()
+    chain = await get_workflow_chain(run_id, storage=storage)
+    print(f"\nWorkflow chain has {len(chain)} runs:")
+    for i, run in enumerate(chain):
+        marker = "  <- started here" if run.run_id == run_id else ""
+        print(f"  {i + 1}. {run.run_id[:20]}... [{run.status.value}]{marker}")
+
+    # Example 2: Message Consumer
+    print("\n--- Example 2: Message Consumer ---")
+    print("Consuming messages until queue is empty...")
+
+    run_id2 = await start(message_consumer)
+    print(f"Started workflow: {run_id2}")
+
+    await asyncio.sleep(3)
+
+    chain2 = await get_workflow_chain(run_id2, storage=storage)
+    print(f"\nConsumer chain has {len(chain2)} runs")
+    if chain2:
+        final = chain2[-1]
+        if final.result:
+            print(f"Final result: {final.result}")
+
+    # Example 3: Recurring Task
+    print("\n--- Example 3: Recurring Report ---")
+    print("Running 3 report iterations...")
+
+    run_id3 = await start(recurring_report)
+    print(f"Started workflow: {run_id3}")
+
+    await asyncio.sleep(2)
+
+    chain3 = await get_workflow_chain(run_id3, storage=storage)
+    print(f"\nReport chain has {len(chain3)} runs")
+
+    # Summary
+    print("\n=== Summary ===")
+    print(f"  Batch processor: {len(chain)} workflow executions")
+    print(f"  Message consumer: {len(chain2)} workflow executions")
+    print(f"  Recurring report: {len(chain3)} workflow executions")
+
+
+def main():
+    """Configure and run examples."""
+    print("Configuring PyWorkflow with Celery runtime...")
+
+    # Reset any existing config
+    reset_config()
+
+    # Configure storage
+    storage = FileStorageBackend(base_path=".workflow_data")
+
+    # Configure pyworkflow
+    configure(
+        storage=storage,
+        default_runtime="celery",
+        default_durable=True,
+    )
+
+    print("Configuration complete!")
+    print("\nMake sure Celery worker is running:")
+    print(
+        "  celery -A pyworkflow.celery.tasks worker -Q pyworkflow.workflows,pyworkflow.steps,pyworkflow.schedules -l info\n"
+    )
+
+    # Run examples
+    asyncio.run(run_examples())
+
+    print("\n=== Key Takeaways ===")
+    print("  - continue_as_new() works across distributed Celery workers")
+    print("  - Each continuation is a new Celery task execution")
+    print("  - Event history is reset, preventing unbounded growth")
+    print("  - Chains can be tracked with get_workflow_chain()")
+    print("  - Useful for long-running polling, batch processing, recurring tasks")
+
+
+if __name__ == "__main__":
+    main()

examples/celery/durable/workflows/fault_tolerance.py

@@ -0,0 +1,210 @@
+"""
+Celery Durable Workflow - Fault Tolerance Example
+
+This example demonstrates automatic recovery from worker failures for durable workflows.
+
+Key features:
+- Automatic recovery when workers crash mid-execution
+- Event replay continues from the last completed step
+- Configurable recovery attempts limit
+- WORKFLOW_INTERRUPTED events recorded for auditing
+
+When a worker crashes:
+1. The task is automatically requeued (task_reject_on_worker_lost=True)
+2. Another worker detects the RUNNING workflow and initiates recovery
+3. WORKFLOW_INTERRUPTED event is recorded
+4. Events are replayed to restore state
+5. Workflow continues from the last checkpoint
+
+Prerequisites:
+    1. Start Redis: docker run -d -p 6379:6379 redis:7-alpine
+    2. Start worker: pyworkflow --module examples.celery.durable.06_fault_tolerance worker run
+
+Run with CLI:
+    pyworkflow --module examples.celery.durable.06_fault_tolerance workflows run data_pipeline \
+        --arg data_id=data-123
+
+To test fault tolerance:
+    1. Start the workflow
+    2. While it's running, kill the worker (Ctrl+C)
+    3. Start a new worker
+    4. Watch the workflow recover and continue from the last completed step
+
+Check status:
+    pyworkflow runs list
+    pyworkflow runs status <run_id>
+    pyworkflow runs logs <run_id>  # Will show WORKFLOW_INTERRUPTED events
+"""
+
+import asyncio
+
+from pyworkflow import sleep, step, workflow
+
+
+@step()
+async def fetch_data(data_id: str) -> dict:
+    """Fetch data from external source."""
+    print(f"[Step 1] Fetching data for {data_id}...")
+    await asyncio.sleep(2)  # Simulate network delay
+    return {"data_id": data_id, "records": 1000, "fetched": True}
+
+
+@step()
+async def validate_data(data: dict) -> dict:
+    """Validate the fetched data."""
+    print(f"[Step 2] Validating {data['records']} records...")
+    await asyncio.sleep(2)  # Simulate validation time
+    return {**data, "valid_records": 950, "validated": True}
+
+
+@step()
+async def transform_data(data: dict) -> dict:
+    """Transform data for processing."""
+    print(f"[Step 3] Transforming {data['valid_records']} records...")
+    await asyncio.sleep(3)  # Simulate CPU-intensive work
+    return {**data, "transformed_records": 950, "transformed": True}
+
+
+@step()
+async def load_data(data: dict) -> dict:
+    """Load transformed data into destination."""
+    print(f"[Step 4] Loading {data['transformed_records']} records...")
+    await asyncio.sleep(2)  # Simulate database writes
+    return {**data, "loaded": True}
+
+
+@step()
+async def send_notification(data: dict) -> dict:
+    """Send completion notification."""
+    print(f"[Step 5] Sending notification for {data['data_id']}...")
+    return {**data, "notified": True}
+
+
+@workflow(
+    recover_on_worker_loss=True,  # Enable automatic recovery (default for durable)
+    max_recovery_attempts=5,  # Allow up to 5 recovery attempts
+    tags=["celery", "durable"],
+)
+async def data_pipeline(data_id: str) -> dict:
+    """
+    Data processing pipeline with fault tolerance.
+
+    This workflow demonstrates automatic recovery from worker failures:
+
+    1. Fetch data from external source
+    2. Validate the data
+    3. Transform data for processing
+    4. Load into destination
+    5. Send completion notification
+
+    If a worker crashes during any step:
+    - The workflow will be automatically recovered by another worker
+    - Already completed steps will be skipped (results from event replay)
+    - Execution continues from where it left off
+    - Up to 5 recovery attempts are allowed
+
+    Test fault tolerance:
+    - Kill the worker during step 3 (transform_data) which takes longest
+    - Start a new worker and watch it recover
+    """
+    print(f"\n{'=' * 60}")
+    print(f"Starting data pipeline for {data_id}")
+    print(f"{'=' * 60}\n")
+
+    data = await fetch_data(data_id)
+    print(f"  -> Fetch complete: {data['records']} records\n")
+
+    print("  [Sleeping 10s before validation - kill worker now to test recovery!]")
+    await sleep("10s")
+
+    data = await validate_data(data)
+    print(f"  -> Validation complete: {data['valid_records']} valid records\n")
+
+    print("  [Sleeping 10s before transform - kill worker now to test recovery!]")
+    await sleep("10s")
+
+    data = await transform_data(data)
+    print(f"  -> Transform complete: {data['transformed_records']} records\n")
+
+    print("  [Sleeping 10s before load - kill worker now to test recovery!]")
+    await sleep("10s")
+
+    data = await load_data(data)
+    print("  -> Load complete\n")
+
+    data = await send_notification(data)
+    print("  -> Notification sent\n")
+
+    print(f"{'=' * 60}")
+    print("Pipeline completed successfully!")
+    print(f"{'=' * 60}\n")
+
+    return data
+
+
+@workflow(
+    recover_on_worker_loss=False,  # Disable recovery for this workflow
+    max_recovery_attempts=0,
+    tags=["celery", "durable"],
+)
+async def critical_pipeline(data_id: str) -> dict:
+    """
+    Critical pipeline that should NOT auto-recover.
+
+    Some workflows should fail completely on worker loss rather than
+    recover, for example when:
+    - Steps have side effects that can't be safely repeated
+    - Human intervention is required after failures
+    - The workflow interacts with non-idempotent external systems
+
+    If a worker crashes during this workflow:
+    - The workflow will be marked as FAILED
+    - No automatic recovery will be attempted
+    - Manual intervention is required
+
+    Usage:
+        pyworkflow workflows run critical_pipeline --arg data_id=critical-001
+    """
+    print(f"[Critical] Processing {data_id} - NO AUTO-RECOVERY")
+
+    data = await fetch_data(data_id)
+    data = await validate_data(data)
+    data = await transform_data(data)
+
+    print(f"[Critical] Completed {data_id}")
+    return data
+
+
+async def main() -> None:
+    """Run the fault tolerance example."""
+    import argparse
+
+    import pyworkflow
+
+    parser = argparse.ArgumentParser(description="Data Pipeline with Fault Tolerance")
+    parser.add_argument("--data-id", default="data-123", help="Data ID to process")
+    parser.add_argument(
+        "--critical",
+        action="store_true",
+        help="Run the critical pipeline (no auto-recovery)",
+    )
+    args = parser.parse_args()
+
+    if args.critical:
+        print(f"Starting CRITICAL pipeline for {args.data_id}...")
+        print("NOTE: This workflow will NOT auto-recover from worker failures")
+        run_id = await pyworkflow.start(critical_pipeline, args.data_id)
+    else:
+        print(f"Starting data pipeline for {args.data_id}...")
+        print("NOTE: This workflow WILL auto-recover from worker failures")
+        print("      Kill the worker during execution to test recovery")
+        run_id = await pyworkflow.start(data_pipeline, args.data_id)
+
+    print(f"\nWorkflow started with run_id: {run_id}")
+    print("\nMonitor with:")
+    print(f"  pyworkflow runs status {run_id}")
+    print(f"  pyworkflow runs logs {run_id}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())