pyworkflow-engine 0.1.7__py3-none-any.whl

examples/celery/durable/workflows/hooks.py

@@ -0,0 +1,211 @@
+"""
+Celery Durable Workflow - Hooks Example
+
+This example demonstrates hooks for waiting on external events with Celery workers:
+- Using hook() to suspend workflow and wait for external input
+- Using define_hook() for typed hooks with Pydantic validation
+- Using CLI commands to list and resume hooks
+- Composite tokens (run_id:hook_id) for self-describing tokens
+
+Prerequisites:
+    1. Start Redis: docker run -d -p 6379:6379 redis:7-alpine
+    2. Start worker: pyworkflow --module examples.celery.durable.07_hooks worker run
+
+Run workflow:
+    cd examples/celery/durable
+    PYTHONPATH=. pyworkflow --module 07_hooks workflows run approval_workflow --arg order_id=order-123
+
+List pending hooks:
+    pyworkflow hooks list --status pending
+
+Resume a hook (interactive):
+    pyworkflow hooks resume
+    # Step 1: Select the pending hook
+    # Step 2: Enter payload values (approved, reviewer, comments)
+
+Resume with explicit payload:
+    pyworkflow hooks resume <token> --payload '{"approved": true, "reviewer": "admin@example.com"}'
+
+Check workflow status:
+    pyworkflow runs status <run_id>
+"""
+
+from pydantic import BaseModel
+
+from pyworkflow import define_hook, hook, step, workflow
+
+
+# --- Pydantic model for typed hook payload ---
+class ApprovalPayload(BaseModel):
+    """Typed payload for approval hook.
+
+    This schema is stored with the hook and used by the CLI
+    to prompt for field values interactively.
+    """
+
+    approved: bool
+    reviewer: str
+    comments: str | None = None
+
+
+# Create typed hook - schema is stored for CLI resume
+approval_hook = define_hook("manager_approval", ApprovalPayload)
+
+
+# --- Steps ---
+@step()
+async def prepare_order(order_id: str) -> dict:
+    """Prepare the order for review."""
+    print(f"[Step] Preparing order {order_id}...")
+    return {"order_id": order_id, "status": "pending_approval"}
+
+
+@step()
+async def fulfill_order(order: dict) -> dict:
+    """Fulfill the approved order."""
+    print(f"[Step] Fulfilling order {order['order_id']}...")
+    return {**order, "status": "fulfilled"}
+
+
+@step()
+async def cancel_order(order: dict, reason: str) -> dict:
+    """Cancel the rejected order."""
+    print(f"[Step] Cancelling order {order['order_id']}: {reason}")
+    return {**order, "status": "cancelled", "reason": reason}
+
+
+# --- Workflow with simple hook ---
+@workflow(name="simple_approval_workflow", tags=["celery", "durable"])
+async def simple_approval_workflow(order_id: str) -> dict:
+    """
+    Workflow using simple hook() with untyped payload.
+
+    The workflow suspends at the hook and waits for external input.
+    Use `pyworkflow hooks resume` to send a payload and continue.
+    """
+    order = await prepare_order(order_id)
+
+    async def on_hook_created(token: str):
+        """Called when hook is created - log the token for CLI use."""
+        print(f"[Hook] Created with token: {token}")
+        print(
+            f"[Hook] Resume with: pyworkflow hooks resume {token} --payload '{{\"approved\": true}}'"
+        )
+
+    # Wait for external approval
+    # Token is auto-generated in composite format: run_id:hook_id
+    approval = await hook(
+        "simple_approval",
+        timeout="24h",  # Expire after 24 hours
+        on_created=on_hook_created,
+    )
+
+    if approval.get("approved"):
+        return await fulfill_order(order)
+    else:
+        return await cancel_order(order, approval.get("reason", "Rejected"))
+
+
+# --- Workflow with typed hook ---
+@workflow(name="approval_workflow", tags=["celery", "durable"])
+async def approval_workflow(order_id: str) -> dict:
+    """
+    Workflow using define_hook() for type-safe payloads.
+
+    The typed hook stores its Pydantic schema, which the CLI uses
+    to prompt for each field interactively during `pyworkflow hooks resume`.
+
+    Run: pyworkflow workflows run approval_workflow --arg order_id=order-123
+    Resume: pyworkflow hooks resume (interactive)
+    """
+    order = await prepare_order(order_id)
+
+    async def on_hook_created(token: str):
+        """Called when hook is created - log for CLI use."""
+        print(f"[Hook] Typed hook created with token: {token}")
+        print("[Hook] Run: pyworkflow hooks resume")
+        print(
+            f'[Hook] Or:  pyworkflow hooks resume {token} --payload \'{{"approved": true, "reviewer": "admin@example.com"}}\''
+        )
+
+    # Wait for typed approval - payload validated against ApprovalPayload
+    # CLI will prompt for: approved (bool), reviewer (str), comments (str, optional)
+    approval: ApprovalPayload = await approval_hook(
+        timeout="7d",  # Expire after 7 days
+        on_created=on_hook_created,
+    )
+
+    print(
+        f"[Workflow] Received approval: approved={approval.approved}, reviewer={approval.reviewer}"
+    )
+
+    if approval.approved:
+        return await fulfill_order(order)
+    else:
+        return await cancel_order(order, approval.comments or "No reason given")
+
+
+# --- Workflow with multiple hooks ---
+@workflow(name="multi_approval_workflow", tags=["celery", "durable"])
+async def multi_approval_workflow(order_id: str) -> dict:
+    """
+    Workflow demonstrating sequential hooks for multi-level approval.
+
+    This workflow requires two approvals:
+    1. Manager approval
+    2. Finance approval (only if amount is significant)
+    """
+    order = await prepare_order(order_id)
+
+    async def log_token(name: str):
+        async def _log(token: str):
+            print(f"[Hook] {name} hook created: {token}")
+
+        return _log
+
+    # First approval: Manager
+    print("[Workflow] Waiting for manager approval...")
+    manager_approval = await hook(
+        "manager_approval",
+        timeout="24h",
+        on_created=await log_token("Manager"),
+    )
+
+    if not manager_approval.get("approved"):
+        return await cancel_order(
+            order, f"Manager rejected: {manager_approval.get('reason', 'No reason')}"
+        )
+
+    order["manager_approved"] = True
+    order["manager"] = manager_approval.get("approver", "unknown")
+
+    # Second approval: Finance (simulating high-value order check)
+    print("[Workflow] Waiting for finance approval...")
+    finance_approval = await hook(
+        "finance_approval",
+        timeout="48h",
+        on_created=await log_token("Finance"),
+    )
+
+    if not finance_approval.get("approved"):
+        return await cancel_order(
+            order, f"Finance rejected: {finance_approval.get('reason', 'No reason')}"
+        )
+
+    order["finance_approved"] = True
+    order["finance_reviewer"] = finance_approval.get("approver", "unknown")
+
+    return await fulfill_order(order)
+
+
+async def main() -> None:
+    """Run the hooks workflow example via CLI."""
+    print(__doc__)
+    print("\nThis example should be run with Celery workers.")
+    print("See the docstring above for CLI commands.")
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())

examples/celery/durable/workflows/idempotency.py

@@ -0,0 +1,112 @@
+"""
+Celery Durable Workflow - Idempotency
+
+This example demonstrates idempotent workflow execution.
+- Use --idempotency-key to prevent duplicate executions
+- Same key returns existing run instead of starting new one
+- Critical for payment processing and other sensitive operations
+
+Prerequisites:
+    1. Start Redis: docker run -d -p 6379:6379 redis:7-alpine
+    2. Start worker: pyworkflow --module examples.celery.durable.05_idempotency worker run
+
+Run with CLI:
+    # First run - starts new workflow
+    pyworkflow --module examples.celery.durable.05_idempotency workflows run payment_workflow \
+        --arg payment_id=pay-123 --arg amount=99.99 \
+        --idempotency-key payment-pay-123
+
+    # Second run with same key - returns existing run (no duplicate charge)
+    pyworkflow --module examples.celery.durable.05_idempotency workflows run payment_workflow \
+        --arg payment_id=pay-123 --arg amount=99.99 \
+        --idempotency-key payment-pay-123
+
+Check status:
+    pyworkflow runs list
+"""
+
+from pyworkflow import step, workflow
+
+
+@step()
+async def validate_payment(payment_id: str, amount: float) -> dict:
+    """Validate the payment request."""
+    print(f"[Step] Validating payment {payment_id} for ${amount:.2f}...")
+    return {"payment_id": payment_id, "amount": amount, "valid": True}
+
+
+@step()
+async def charge_payment(payment: dict) -> dict:
+    """
+    Charge the payment.
+
+    IMPORTANT: This step should only run once per payment!
+    Idempotency keys ensure duplicate requests don't double-charge.
+    """
+    print(f"[Step] CHARGING payment {payment['payment_id']} for ${payment['amount']:.2f}...")
+    print("[Step] (In production, this would call Stripe/PayPal with idempotency key)")
+    return {**payment, "charged": True, "transaction_id": f"txn_{payment['payment_id']}"}
+
+
+@step()
+async def send_receipt(payment: dict) -> dict:
+    """Send payment receipt."""
+    print(f"[Step] Sending receipt for {payment['payment_id']}...")
+    return {**payment, "receipt_sent": True}
+
+
+@workflow(tags=["celery", "durable"])
+async def payment_workflow(payment_id: str, amount: float) -> dict:
+    """
+    Payment processing workflow with idempotency.
+
+    ALWAYS use --idempotency-key when running this workflow to prevent
+    duplicate charges. The key should be unique per payment attempt.
+
+    Example keys:
+    - payment-{payment_id}
+    - order-{order_id}-payment
+    - user-{user_id}-{timestamp}
+
+    If a workflow with the same idempotency key already exists:
+    - If RUNNING: raises WorkflowAlreadyRunningError
+    - If COMPLETED/FAILED/SUSPENDED: returns existing run_id
+    """
+    payment = await validate_payment(payment_id, amount)
+    payment = await charge_payment(payment)
+    payment = await send_receipt(payment)
+    return payment
+
+
+async def main() -> None:
+    """Run the payment workflow example with idempotency."""
+    import argparse
+
+    import pyworkflow
+
+    parser = argparse.ArgumentParser(description="Payment Workflow with Idempotency")
+    parser.add_argument("--payment-id", default="pay-123", help="Payment ID")
+    parser.add_argument("--amount", type=float, default=99.99, help="Payment amount")
+    parser.add_argument("--idempotency-key", help="Idempotency key (recommended)")
+    args = parser.parse_args()
+
+    idempotency_key = args.idempotency_key or f"payment-{args.payment_id}"
+
+    # Configuration is automatically loaded from pyworkflow.config.yaml
+    print(f"Starting payment workflow for {args.payment_id} (${args.amount:.2f})...")
+    print(f"Idempotency key: {idempotency_key}")
+    run_id = await pyworkflow.start(
+        payment_workflow,
+        args.payment_id,
+        args.amount,
+        idempotency_key=idempotency_key,
+    )
+    print(f"Workflow started with run_id: {run_id}")
+    print(f"\nCheck status: pyworkflow runs status {run_id}")
+    print("\nRun again with same --idempotency-key to see duplicate prevention!")
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())

examples/celery/durable/workflows/long_running.py

@@ -0,0 +1,99 @@
+"""
+Celery Durable Workflow - Long Running with Sleep
+
+This example demonstrates automatic sleep resumption with Celery workers.
+- Workflow suspends during sleep (releases resources)
+- Celery automatically resumes after sleep completes
+- No manual intervention required (unlike local runtime)
+
+Prerequisites:
+    1. Start Redis: docker run -d -p 6379:6379 redis:7-alpine
+    2. Start worker: pyworkflow --module examples.celery.durable.02_long_running worker run
+
+Run with CLI:
+    pyworkflow --module examples.celery.durable.02_long_running workflows run onboarding_workflow \
+        --arg user_id=user-456
+
+Watch the worker output to see automatic resumption after each sleep.
+
+Check status:
+    pyworkflow runs list --status suspended
+    pyworkflow runs status <run_id>
+"""
+
+from pyworkflow import sleep, step, workflow
+
+
+@step()
+async def send_welcome_email(user_id: str) -> dict:
+    """Send welcome email to new user."""
+    print(f"[Step] Sending welcome email to {user_id}...")
+    return {"user_id": user_id, "welcome_sent": True}
+
+
+@step()
+async def send_tips_email(user: dict) -> dict:
+    """Send helpful tips email after delay."""
+    print(f"[Step] Sending tips email to {user['user_id']}...")
+    return {**user, "tips_sent": True}
+
+
+@step()
+async def send_survey_email(user: dict) -> dict:
+    """Send feedback survey after delay."""
+    print(f"[Step] Sending survey email to {user['user_id']}...")
+    return {**user, "survey_sent": True}
+
+
+@workflow(tags=["celery", "durable"])
+async def onboarding_workflow(user_id: str) -> dict:
+    """
+    User onboarding workflow with scheduled emails.
+
+    Demonstrates automatic sleep resumption:
+    1. Send welcome email immediately
+    2. Wait 30 seconds, then send tips email
+    3. Wait another 30 seconds, then send survey
+
+    With Celery runtime, sleeps are handled automatically:
+    - Workflow suspends and worker is freed
+    - Celery schedules resumption task
+    - Worker picks up and continues execution
+    """
+    user = await send_welcome_email(user_id)
+
+    print("[Workflow] Sleeping for 30 seconds before tips email...")
+    await sleep("30s")
+
+    user = await send_tips_email(user)
+
+    print("[Workflow] Sleeping for 30 seconds before survey...")
+    await sleep("30s")
+
+    user = await send_survey_email(user)
+    return user
+
+
+async def main() -> None:
+    """Run the onboarding workflow example."""
+    import argparse
+
+    import pyworkflow
+
+    parser = argparse.ArgumentParser(description="User Onboarding Workflow with Sleeps")
+    parser.add_argument("--user-id", default="user-456", help="User ID to onboard")
+    args = parser.parse_args()
+
+    # Configuration is automatically loaded from pyworkflow.config.yaml
+    print(f"Starting onboarding workflow for {args.user_id}...")
+    print("(Workflow will sleep between emails - watch the worker output)")
+    run_id = await pyworkflow.start(onboarding_workflow, args.user_id)
+    print(f"Workflow started with run_id: {run_id}")
+    print(f"\nCheck status: pyworkflow runs status {run_id}")
+    print("List suspended: pyworkflow runs list --status suspended")
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())

examples/celery/durable/workflows/retries.py

@@ -0,0 +1,101 @@
+"""
+Celery Durable Workflow - Retry Handling
+
+This example demonstrates automatic retry handling on Celery workers.
+- Steps can specify max_retries and retry_delay
+- RetryableError triggers automatic retry with backoff
+- FatalError stops workflow immediately (no retry)
+
+Prerequisites:
+    1. Start Redis: docker run -d -p 6379:6379 redis:7-alpine
+    2. Start worker: pyworkflow --module examples.celery.durable.03_retries worker run
+
+Run with CLI:
+    pyworkflow --module examples.celery.durable.03_retries workflows run retry_demo_workflow \
+        --arg endpoint=/api/data
+
+The workflow has a 30% failure rate - run multiple times to see retry behavior.
+
+Check status:
+    pyworkflow runs list
+    pyworkflow runs logs <run_id> --filter failed
+"""
+
+import random
+
+from pyworkflow import step, workflow
+from pyworkflow.core.exceptions import FatalError, RetryableError
+
+
+@step(max_retries=3)
+async def flaky_api_call(endpoint: str) -> dict:
+    """
+    Simulate a flaky API call that may fail.
+
+    - 30% chance of RetryableError (will retry)
+    - 10% chance of FatalError (will not retry)
+    - 60% chance of success
+    """
+    print(f"[Step] Calling API: {endpoint}...")
+
+    roll = random.random()
+
+    if roll < 0.3:
+        print("[Step] API temporarily unavailable, will retry...")
+        raise RetryableError("API temporarily unavailable", retry_after="5s")
+
+    if roll < 0.4:
+        print("[Step] API returned invalid response, fatal error...")
+        raise FatalError("API returned invalid response - cannot retry")
+
+    print("[Step] API call successful!")
+    return {"endpoint": endpoint, "status": "success", "data": {"value": 42}}
+
+
+@step()
+async def process_response(response: dict) -> dict:
+    """Process the API response."""
+    print(f"[Step] Processing response from {response['endpoint']}...")
+    return {**response, "processed": True}
+
+
+@workflow(tags=["celery", "durable"])
+async def retry_demo_workflow(endpoint: str) -> dict:
+    """
+    Workflow demonstrating automatic retry handling.
+
+    The flaky_api_call step has:
+    - 30% failure rate with RetryableError (auto-retry)
+    - 10% failure rate with FatalError (no retry)
+    - 60% success rate
+
+    Retries happen automatically with exponential backoff on workers.
+    """
+    response = await flaky_api_call(endpoint)
+    result = await process_response(response)
+    return {"message": "API call and processing succeeded", **result}
+
+
+async def main() -> None:
+    """Run the retry demo workflow example."""
+    import argparse
+
+    import pyworkflow
+
+    parser = argparse.ArgumentParser(description="Retry Handling Demo Workflow")
+    parser.add_argument("--endpoint", default="/api/data", help="API endpoint to call")
+    args = parser.parse_args()
+
+    # Configuration is automatically loaded from pyworkflow.config.yaml
+    print(f"Starting retry demo workflow for endpoint {args.endpoint}...")
+    print("(30% chance of retry, 10% chance of fatal error, 60% success)")
+    run_id = await pyworkflow.start(retry_demo_workflow, args.endpoint)
+    print(f"Workflow started with run_id: {run_id}")
+    print(f"\nCheck status: pyworkflow runs status {run_id}")
+    print(f"View logs: pyworkflow runs logs {run_id}")
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())