pyworkflow-engine 0.1.7__py3-none-any.whl → 0.1.10__py3-none-any.whl

examples/celery/durable/workflows/child_workflows.py

@@ -1,202 +0,0 @@
-"""
-Celery Durable Workflow - Child Workflows Basic Example
-
-This example demonstrates how to spawn child workflows from a parent workflow.
-- Parent workflow spawns child workflows using start_child_workflow()
-- Children have their own run_id and event history
-- wait_for_completion=True (default) waits for child to complete
-- wait_for_completion=False returns a ChildWorkflowHandle immediately
-- TERMINATE policy: when parent completes/fails/cancels, children are cancelled
-
-Prerequisites:
-    1. Start Redis: docker run -d -p 6379:6379 redis:7-alpine
-    2. Start worker: pyworkflow --module examples.celery.durable.09_child_workflows worker run
-
-Run with CLI:
-    pyworkflow --module examples.celery.durable.09_child_workflows workflows run order_fulfillment_workflow \
-        --arg order_id=order-456 --arg amount=149.99 --arg customer_email=customer@example.com
-
-Check status:
-    pyworkflow runs list
-    pyworkflow runs status <run_id>
-    pyworkflow runs children <run_id>
-"""
-
-from pyworkflow import (
-    ChildWorkflowHandle,
-    start_child_workflow,
-    step,
-    workflow,
-)
-
-
-# --- Steps ---
-@step(name="child_demo_validate_order")
-async def validate_order(order_id: str) -> dict:
-    """Validate order details."""
-    print(f"    Validating order {order_id}...")
-    return {"order_id": order_id, "valid": True}
-
-
-@step(name="child_demo_process_payment")
-async def process_payment(order_id: str, amount: float) -> dict:
-    """Process payment for order."""
-    print(f"    Processing payment ${amount:.2f} for {order_id}...")
-    return {"order_id": order_id, "amount": amount, "paid": True}
-
-
-@step(name="child_demo_ship_order")
-async def ship_order(order_id: str) -> dict:
-    """Ship the order."""
-    print(f"    Shipping order {order_id}...")
-    return {"order_id": order_id, "shipped": True}
-
-
-@step(name="child_demo_send_email")
-async def send_email(recipient: str, subject: str) -> dict:
-    """Send an email notification."""
-    print(f"    Sending email to {recipient}: {subject}")
-    return {"recipient": recipient, "subject": subject, "sent": True}
-
-
-# --- Child Workflows ---
-@workflow(name="child_demo_payment_workflow", tags=["celery", "durable"])
-async def payment_workflow(order_id: str, amount: float) -> dict:
-    """Child workflow for payment processing."""
-    print(f"  [PaymentWorkflow] Starting for order {order_id}")
-    result = await process_payment(order_id, amount)
-    print(f"  [PaymentWorkflow] Completed for order {order_id}")
-    return result
-
-
-@workflow(name="child_demo_shipping_workflow", tags=["celery", "durable"])
-async def shipping_workflow(order_id: str) -> dict:
-    """Child workflow for shipping."""
-    print(f"  [ShippingWorkflow] Starting for order {order_id}")
-    result = await ship_order(order_id)
-    print(f"  [ShippingWorkflow] Completed for order {order_id}")
-    return result
-
-
-@workflow(name="child_demo_notification_workflow", tags=["celery", "durable"])
-async def notification_workflow(email: str, order_id: str) -> dict:
-    """Child workflow for sending notifications."""
-    print(f"  [NotificationWorkflow] Starting for {email}")
-    result = await send_email(email, f"Order {order_id} update")
-    print(f"  [NotificationWorkflow] Completed for {email}")
-    return result
-
-
-# --- Parent Workflow ---
-@workflow(tags=["celery", "durable"])
-async def order_fulfillment_workflow(
-    order_id: str,
-    amount: float,
-    customer_email: str,
-) -> dict:
-    """
-    Parent workflow that orchestrates order fulfillment using child workflows.
-
-    This demonstrates:
-    1. wait_for_completion=True - Wait for child to complete (default)
-    2. wait_for_completion=False - Fire-and-forget with handle
-    """
-    print(f"[OrderFulfillment] Starting for order {order_id}")
-
-    # Step 1: Validate order (regular step)
-    validation = await validate_order(order_id)
-    if not validation["valid"]:
-        return {"order_id": order_id, "status": "invalid"}
-
-    # Step 2: Process payment via child workflow (wait for completion)
-    print("[OrderFulfillment] Starting payment child workflow...")
-    payment_result = await start_child_workflow(
-        payment_workflow,
-        order_id,
-        amount,
-        wait_for_completion=True,  # Default: wait for child to complete
-    )
-    print(f"[OrderFulfillment] Payment completed: {payment_result}")
-
-    # Step 3: Ship order via child workflow (wait for completion)
-    print("[OrderFulfillment] Starting shipping child workflow...")
-    shipping_result = await start_child_workflow(
-        shipping_workflow,
-        order_id,
-        wait_for_completion=True,
-    )
-    print(f"[OrderFulfillment] Shipping completed: {shipping_result}")
-
-    # Step 4: Send notification via fire-and-forget child workflow
-    # This returns immediately with a handle, parent continues
-    print("[OrderFulfillment] Starting notification child workflow (fire-and-forget)...")
-    notification_handle: ChildWorkflowHandle = await start_child_workflow(
-        notification_workflow,
-        customer_email,
-        order_id,
-        wait_for_completion=False,  # Fire-and-forget
-    )
-    print(f"[OrderFulfillment] Notification child started: {notification_handle.child_run_id}")
-
-    result = {
-        "order_id": order_id,
-        "status": "fulfilled",
-        "payment": payment_result,
-        "shipping": shipping_result,
-        "notification_run_id": notification_handle.child_run_id,
-    }
-
-    print(f"[OrderFulfillment] Completed for order {order_id}")
-    return result
-
-
-async def main() -> None:
-    """Run the order fulfillment workflow example."""
-    import argparse
-    import asyncio
-
-    import pyworkflow
-    from pyworkflow import get_workflow_run
-
-    parser = argparse.ArgumentParser(description="Order Fulfillment Workflow with Child Workflows")
-    parser.add_argument("--order-id", default="order-456", help="Order ID to process")
-    parser.add_argument("--amount", type=float, default=149.99, help="Order amount")
-    parser.add_argument("--email", default="customer@example.com", help="Customer email")
-    args = parser.parse_args()
-
-    print("=== Child Workflows - Basic Example ===\n")
-    print("Running order fulfillment workflow with child workflows...\n")
-
-    # Start parent workflow
-    run_id = await pyworkflow.start(
-        order_fulfillment_workflow,
-        args.order_id,
-        args.amount,
-        args.email,
-    )
-
-    print(f"\nWorkflow started with run_id: {run_id}")
-    print("\nCheck status:")
-    print(f"  pyworkflow runs status {run_id}")
-    print(f"  pyworkflow runs children {run_id}")
-
-    # Poll for completion
-    print("\nWaiting for workflow to complete...")
-    for _ in range(30):
-        await asyncio.sleep(1)
-        run = await get_workflow_run(run_id)
-        if run.status.value in ("completed", "failed", "cancelled"):
-            print(f"\nWorkflow {run.status.value}!")
-            if run.result:
-                print(f"Result: {run.result}")
-            if run.error:
-                print(f"Error: {run.error}")
-            break
-    else:
-        print("\nTimeout waiting for workflow completion")
-
-
-if __name__ == "__main__":
-    import asyncio
-
-    asyncio.run(main())

examples/celery/durable/workflows/continue_as_new.py

@@ -1,260 +0,0 @@
-"""
-Durable Workflow (Celery) - Continue-As-New
-
-This example demonstrates continue_as_new() with Celery workers:
-- Polling workflows that need fresh event history
-- Batch processing with continuation
-- Tracking workflow chains across distributed workers
-
-Prerequisites:
-1. Redis running: docker run -d -p 6379:6379 redis
-2. Start Celery worker:
-   celery -A pyworkflow.celery.tasks worker -Q pyworkflow.workflows,pyworkflow.steps,pyworkflow.schedules -l info
-
-Run: python examples/celery/durable/11_continue_as_new.py
-"""
-
-import asyncio
-
-from pyworkflow import (
-    configure,
-    continue_as_new,
-    get_workflow_chain,
-    reset_config,
-    start,
-    step,
-    workflow,
-)
-from pyworkflow.storage import FileStorageBackend
-
-
-# --- Steps ---
-@step()
-async def fetch_batch(offset: int, batch_size: int) -> list:
-    """Fetch a batch of items to process."""
-    # Simulate fetching items - returns empty when done
-    total_items = 50  # Simulate 50 total items
-    if offset >= total_items:
-        return []
-    end = min(offset + batch_size, total_items)
-    items = list(range(offset, end))
-    print(f"  [Step] Fetched items {offset} to {end - 1}")
-    await asyncio.sleep(0.1)  # Simulate I/O
-    return items
-
-
-@step(name="continue_process_item")
-async def process_item(item: int) -> dict:
-    """Process a single item."""
-    await asyncio.sleep(0.05)  # Simulate work
-    return {"item": item, "processed": True}
-
-
-@step()
-async def poll_for_messages(cursor: str | None) -> tuple[str | None, list]:
-    """Poll message queue for new messages."""
-    # Simulate message queue polling
-    await asyncio.sleep(0.1)
-
-    if cursor is None:
-        return "msg_batch_1", [{"id": 1, "type": "order"}, {"id": 2, "type": "payment"}]
-    elif cursor == "msg_batch_1":
-        return "msg_batch_2", [{"id": 3, "type": "shipment"}]
-    elif cursor == "msg_batch_2":
-        return "msg_batch_3", [{"id": 4, "type": "notification"}]
-    else:
-        return None, []  # No more messages
-
-
-@step()
-async def handle_message(message: dict) -> dict:
-    """Handle a single message."""
-    await asyncio.sleep(0.05)
-    return {"message_id": message["id"], "handled": True}
-
-
-# --- Batch Processing Workflow ---
-@workflow(durable=True, tags=["celery", "durable"])
-async def batch_processor(offset: int = 0, batch_size: int = 10) -> str:
-    """
-    Process items in batches using continue_as_new.
-
-    Each batch runs as a separate workflow execution with fresh
-    event history, preventing unbounded history growth.
-
-    This pattern is ideal for:
-    - ETL pipelines processing millions of records
-    - Data migration jobs
-    - Bulk update operations
-    """
-    print(f"\n  [Batch] Starting at offset {offset}")
-
-    items = await fetch_batch(offset, batch_size)
-
-    if not items:
-        return f"Batch processing complete! Total items: {offset}"
-
-    # Process items
-    for item in items:
-        await process_item(item)
-
-    print(f"  [Batch] Processed {len(items)} items")
-
-    # Continue with next batch
-    continue_as_new(offset=offset + batch_size, batch_size=batch_size)
-
-
-# --- Message Consumer Workflow ---
-@workflow(durable=True, tags=["celery", "durable"])
-async def message_consumer(cursor: str | None = None, messages_processed: int = 0) -> str:
-    """
-    Consume messages from a queue, continuing as new after each batch.
-
-    This pattern is useful for:
-    - Queue consumers that run indefinitely
-    - Event stream processors
-    - Real-time data ingestion
-    """
-    print(f"\n  [Consumer] Polling with cursor: {cursor}")
-
-    # Poll for messages
-    new_cursor, messages = await poll_for_messages(cursor)
-
-    if not messages and new_cursor is None:
-        return f"Consumer complete! Processed {messages_processed} messages"
-
-    # Handle each message
-    count = 0
-    for message in messages:
-        await handle_message(message)
-        count += 1
-
-    total = messages_processed + count
-    print(f"  [Consumer] Handled {count} messages (total: {total})")
-
-    # Continue with new cursor
-    continue_as_new(cursor=new_cursor, messages_processed=total)
-
-
-# --- Recurring Task Workflow ---
-@workflow(durable=True, tags=["celery", "durable"])
-async def recurring_report(iteration: int = 1, max_iterations: int = 3) -> str:
-    """
-    Generate reports on a schedule, continuing as new for each iteration.
-
-    This demonstrates a pattern for:
-    - Daily/weekly reports
-    - Scheduled cleanup tasks
-    - Periodic sync operations
-
-    In production, you might add sleep() between iterations.
-    """
-    print(f"\n  [Report] Generating report #{iteration}")
-
-    # Simulate report generation
-    await asyncio.sleep(0.1)
-    print(f"  [Report] Report #{iteration} complete")
-
-    if iteration >= max_iterations:
-        return f"All {max_iterations} reports generated!"
-
-    # Continue with next iteration
-    continue_as_new(iteration=iteration + 1, max_iterations=max_iterations)
-
-
-async def run_examples():
-    """Run all continue-as-new examples."""
-    print("\n=== Continue-As-New Examples (Celery) ===\n")
-
-    # Example 1: Batch Processing
-    print("--- Example 1: Batch Processing ---")
-    print("Processing 50 items in batches of 10...")
-
-    run_id = await start(batch_processor, offset=0, batch_size=10)
-    print(f"Started workflow: {run_id}")
-
-    # Wait for completion (in production, use webhooks or polling)
-    print("Waiting for completion...")
-    await asyncio.sleep(5)
-
-    # Check the chain
-    from pyworkflow import get_storage
-
-    storage = get_storage()
-    chain = await get_workflow_chain(run_id, storage=storage)
-    print(f"\nWorkflow chain has {len(chain)} runs:")
-    for i, run in enumerate(chain):
-        marker = "  <- started here" if run.run_id == run_id else ""
-        print(f"  {i + 1}. {run.run_id[:20]}... [{run.status.value}]{marker}")
-
-    # Example 2: Message Consumer
-    print("\n--- Example 2: Message Consumer ---")
-    print("Consuming messages until queue is empty...")
-
-    run_id2 = await start(message_consumer)
-    print(f"Started workflow: {run_id2}")
-
-    await asyncio.sleep(3)
-
-    chain2 = await get_workflow_chain(run_id2, storage=storage)
-    print(f"\nConsumer chain has {len(chain2)} runs")
-    if chain2:
-        final = chain2[-1]
-        if final.result:
-            print(f"Final result: {final.result}")
-
-    # Example 3: Recurring Task
-    print("\n--- Example 3: Recurring Report ---")
-    print("Running 3 report iterations...")
-
-    run_id3 = await start(recurring_report)
-    print(f"Started workflow: {run_id3}")
-
-    await asyncio.sleep(2)
-
-    chain3 = await get_workflow_chain(run_id3, storage=storage)
-    print(f"\nReport chain has {len(chain3)} runs")
-
-    # Summary
-    print("\n=== Summary ===")
-    print(f"  Batch processor: {len(chain)} workflow executions")
-    print(f"  Message consumer: {len(chain2)} workflow executions")
-    print(f"  Recurring report: {len(chain3)} workflow executions")
-
-
-def main():
-    """Configure and run examples."""
-    print("Configuring PyWorkflow with Celery runtime...")
-
-    # Reset any existing config
-    reset_config()
-
-    # Configure storage
-    storage = FileStorageBackend(base_path=".workflow_data")
-
-    # Configure pyworkflow
-    configure(
-        storage=storage,
-        default_runtime="celery",
-        default_durable=True,
-    )
-
-    print("Configuration complete!")
-    print("\nMake sure Celery worker is running:")
-    print(
-        "  celery -A pyworkflow.celery.tasks worker -Q pyworkflow.workflows,pyworkflow.steps,pyworkflow.schedules -l info\n"
-    )
-
-    # Run examples
-    asyncio.run(run_examples())
-
-    print("\n=== Key Takeaways ===")
-    print("  - continue_as_new() works across distributed Celery workers")
-    print("  - Each continuation is a new Celery task execution")
-    print("  - Event history is reset, preventing unbounded growth")
-    print("  - Chains can be tracked with get_workflow_chain()")
-    print("  - Useful for long-running polling, batch processing, recurring tasks")
-
-
-if __name__ == "__main__":
-    main()

examples/celery/durable/workflows/fault_tolerance.py

@@ -1,210 +0,0 @@
-"""
-Celery Durable Workflow - Fault Tolerance Example
-
-This example demonstrates automatic recovery from worker failures for durable workflows.
-
-Key features:
-- Automatic recovery when workers crash mid-execution
-- Event replay continues from the last completed step
-- Configurable recovery attempts limit
-- WORKFLOW_INTERRUPTED events recorded for auditing
-
-When a worker crashes:
-1. The task is automatically requeued (task_reject_on_worker_lost=True)
-2. Another worker detects the RUNNING workflow and initiates recovery
-3. WORKFLOW_INTERRUPTED event is recorded
-4. Events are replayed to restore state
-5. Workflow continues from the last checkpoint
-
-Prerequisites:
-    1. Start Redis: docker run -d -p 6379:6379 redis:7-alpine
-    2. Start worker: pyworkflow --module examples.celery.durable.06_fault_tolerance worker run
-
-Run with CLI:
-    pyworkflow --module examples.celery.durable.06_fault_tolerance workflows run data_pipeline \
-        --arg data_id=data-123
-
-To test fault tolerance:
-    1. Start the workflow
-    2. While it's running, kill the worker (Ctrl+C)
-    3. Start a new worker
-    4. Watch the workflow recover and continue from the last completed step
-
-Check status:
-    pyworkflow runs list
-    pyworkflow runs status <run_id>
-    pyworkflow runs logs <run_id>  # Will show WORKFLOW_INTERRUPTED events
-"""
-
-import asyncio
-
-from pyworkflow import sleep, step, workflow
-
-
-@step()
-async def fetch_data(data_id: str) -> dict:
-    """Fetch data from external source."""
-    print(f"[Step 1] Fetching data for {data_id}...")
-    await asyncio.sleep(2)  # Simulate network delay
-    return {"data_id": data_id, "records": 1000, "fetched": True}
-
-
-@step()
-async def validate_data(data: dict) -> dict:
-    """Validate the fetched data."""
-    print(f"[Step 2] Validating {data['records']} records...")
-    await asyncio.sleep(2)  # Simulate validation time
-    return {**data, "valid_records": 950, "validated": True}
-
-
-@step()
-async def transform_data(data: dict) -> dict:
-    """Transform data for processing."""
-    print(f"[Step 3] Transforming {data['valid_records']} records...")
-    await asyncio.sleep(3)  # Simulate CPU-intensive work
-    return {**data, "transformed_records": 950, "transformed": True}
-
-
-@step()
-async def load_data(data: dict) -> dict:
-    """Load transformed data into destination."""
-    print(f"[Step 4] Loading {data['transformed_records']} records...")
-    await asyncio.sleep(2)  # Simulate database writes
-    return {**data, "loaded": True}
-
-
-@step()
-async def send_notification(data: dict) -> dict:
-    """Send completion notification."""
-    print(f"[Step 5] Sending notification for {data['data_id']}...")
-    return {**data, "notified": True}
-
-
-@workflow(
-    recover_on_worker_loss=True,  # Enable automatic recovery (default for durable)
-    max_recovery_attempts=5,  # Allow up to 5 recovery attempts
-    tags=["celery", "durable"],
-)
-async def data_pipeline(data_id: str) -> dict:
-    """
-    Data processing pipeline with fault tolerance.
-
-    This workflow demonstrates automatic recovery from worker failures:
-
-    1. Fetch data from external source
-    2. Validate the data
-    3. Transform data for processing
-    4. Load into destination
-    5. Send completion notification
-
-    If a worker crashes during any step:
-    - The workflow will be automatically recovered by another worker
-    - Already completed steps will be skipped (results from event replay)
-    - Execution continues from where it left off
-    - Up to 5 recovery attempts are allowed
-
-    Test fault tolerance:
-    - Kill the worker during step 3 (transform_data) which takes longest
-    - Start a new worker and watch it recover
-    """
-    print(f"\n{'=' * 60}")
-    print(f"Starting data pipeline for {data_id}")
-    print(f"{'=' * 60}\n")
-
-    data = await fetch_data(data_id)
-    print(f"  -> Fetch complete: {data['records']} records\n")
-
-    print("  [Sleeping 10s before validation - kill worker now to test recovery!]")
-    await sleep("10s")
-
-    data = await validate_data(data)
-    print(f"  -> Validation complete: {data['valid_records']} valid records\n")
-
-    print("  [Sleeping 10s before transform - kill worker now to test recovery!]")
-    await sleep("10s")
-
-    data = await transform_data(data)
-    print(f"  -> Transform complete: {data['transformed_records']} records\n")
-
-    print("  [Sleeping 10s before load - kill worker now to test recovery!]")
-    await sleep("10s")
-
-    data = await load_data(data)
-    print("  -> Load complete\n")
-
-    data = await send_notification(data)
-    print("  -> Notification sent\n")
-
-    print(f"{'=' * 60}")
-    print("Pipeline completed successfully!")
-    print(f"{'=' * 60}\n")
-
-    return data
-
-
-@workflow(
-    recover_on_worker_loss=False,  # Disable recovery for this workflow
-    max_recovery_attempts=0,
-    tags=["celery", "durable"],
-)
-async def critical_pipeline(data_id: str) -> dict:
-    """
-    Critical pipeline that should NOT auto-recover.
-
-    Some workflows should fail completely on worker loss rather than
-    recover, for example when:
-    - Steps have side effects that can't be safely repeated
-    - Human intervention is required after failures
-    - The workflow interacts with non-idempotent external systems
-
-    If a worker crashes during this workflow:
-    - The workflow will be marked as FAILED
-    - No automatic recovery will be attempted
-    - Manual intervention is required
-
-    Usage:
-        pyworkflow workflows run critical_pipeline --arg data_id=critical-001
-    """
-    print(f"[Critical] Processing {data_id} - NO AUTO-RECOVERY")
-
-    data = await fetch_data(data_id)
-    data = await validate_data(data)
-    data = await transform_data(data)
-
-    print(f"[Critical] Completed {data_id}")
-    return data
-
-
-async def main() -> None:
-    """Run the fault tolerance example."""
-    import argparse
-
-    import pyworkflow
-
-    parser = argparse.ArgumentParser(description="Data Pipeline with Fault Tolerance")
-    parser.add_argument("--data-id", default="data-123", help="Data ID to process")
-    parser.add_argument(
-        "--critical",
-        action="store_true",
-        help="Run the critical pipeline (no auto-recovery)",
-    )
-    args = parser.parse_args()
-
-    if args.critical:
-        print(f"Starting CRITICAL pipeline for {args.data_id}...")
-        print("NOTE: This workflow will NOT auto-recover from worker failures")
-        run_id = await pyworkflow.start(critical_pipeline, args.data_id)
-    else:
-        print(f"Starting data pipeline for {args.data_id}...")
-        print("NOTE: This workflow WILL auto-recover from worker failures")
-        print("      Kill the worker during execution to test recovery")
-        run_id = await pyworkflow.start(data_pipeline, args.data_id)
-
-    print(f"\nWorkflow started with run_id: {run_id}")
-    print("\nMonitor with:")
-    print(f"  pyworkflow runs status {run_id}")
-    print(f"  pyworkflow runs logs {run_id}")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())