pyworkflow-engine 0.1.7__py3-none-any.whl

examples/local/durable/11_continue_as_new.py

@@ -0,0 +1,249 @@
+"""
+Durable Workflow - Continue-As-New
+
+This example demonstrates continue_as_new() for long-running workflows:
+- Polling workflows that need fresh event history
+- Batch processing with continuation
+- Tracking workflow chains
+- Using get_workflow_chain() to view the full history
+
+Run: python examples/local/durable/11_continue_as_new.py 2>/dev/null
+"""
+
+import asyncio
+import tempfile
+
+from pyworkflow import (
+    configure,
+    continue_as_new,
+    get_workflow_chain,
+    reset_config,
+    start,
+    step,
+    workflow,
+)
+from pyworkflow.storage import FileStorageBackend
+
+
+# --- Steps ---
+@step()
+async def fetch_batch(offset: int, batch_size: int) -> list:
+    """Fetch a batch of items to process."""
+    # Simulate fetching items - returns empty when done
+    total_items = 25  # Simulate 25 total items
+    if offset >= total_items:
+        return []
+    end = min(offset + batch_size, total_items)
+    items = list(range(offset, end))
+    print(f"  [Step] Fetched items {offset} to {end - 1}")
+    return items
+
+
+@step()
+async def process_batch_item(item: int) -> dict:
+    """Process a single item."""
+    await asyncio.sleep(0.01)  # Simulate work
+    return {"item": item, "processed": True}
+
+
+@step()
+async def check_for_updates(cursor: str | None) -> tuple[str | None, list]:
+    """Check for new updates since cursor."""
+    # Simulate polling - returns new cursor and items
+    if cursor is None:
+        return "cursor_1", [{"id": 1, "data": "first"}]
+    elif cursor == "cursor_1":
+        return "cursor_2", [{"id": 2, "data": "second"}]
+    elif cursor == "cursor_2":
+        return "cursor_3", [{"id": 3, "data": "third"}]
+    else:
+        # No more updates
+        return None, []
+
+
+# --- Example 1: Batch Processing Workflow ---
+@workflow(durable=True, tags=["local", "durable"])
+async def batch_processor(offset: int = 0, batch_size: int = 10) -> str:
+    """
+    Process items in batches, continuing as new for each batch.
+
+    This pattern prevents event history from growing unbounded
+    when processing large datasets.
+    """
+    print(f"\n  [Workflow] Processing batch starting at offset {offset}")
+
+    # Fetch batch
+    items = await fetch_batch(offset, batch_size)
+
+    if not items:
+        # No more items - we're done!
+        return f"Completed! Processed {offset} total items"
+
+    # Process each item in this batch
+    for item in items:
+        await process_batch_item(item)
+
+    print("  [Workflow] Batch complete. Continuing with next batch...")
+
+    # Continue with next batch - fresh event history!
+    continue_as_new(offset=offset + batch_size, batch_size=batch_size)
+
+
+# --- Example 2: Polling Workflow ---
+@workflow(durable=True, tags=["local", "durable"])
+async def polling_workflow(cursor: str | None = None, poll_count: int = 0) -> str:
+    """
+    Poll for updates indefinitely, continuing as new to reset history.
+
+    This pattern is useful for:
+    - Webhook listeners
+    - Queue consumers
+    - Real-time sync workflows
+    """
+    print(f"\n  [Workflow] Poll #{poll_count + 1}, cursor: {cursor}")
+
+    # Check for updates
+    new_cursor, updates = await check_for_updates(cursor)
+
+    if updates:
+        print(f"  [Workflow] Found {len(updates)} update(s)")
+        for update in updates:
+            print(f"    - Processing: {update}")
+
+    if new_cursor is None:
+        # No more updates - done polling
+        return f"Polling complete after {poll_count + 1} polls"
+
+    # Continue polling with new cursor
+    print(f"  [Workflow] Continuing with new cursor: {new_cursor}")
+    continue_as_new(cursor=new_cursor, poll_count=poll_count + 1)
+
+
+# --- Example 3: Counter Workflow (Simple Demo) ---
+@workflow(durable=True, tags=["local", "durable"])
+async def countdown_workflow(count: int) -> str:
+    """
+    Simple countdown that demonstrates continue_as_new.
+    Each continuation has fresh event history.
+    """
+    print(f"\n  [Workflow] Count: {count}")
+
+    if count <= 0:
+        return "Countdown complete!"
+
+    # Continue with decremented count
+    continue_as_new(count=count - 1)
+
+
+async def example_batch_processing(storage):
+    """Example 1: Batch processing with continuation."""
+    print("\n--- Example 1: Batch Processing ---")
+    print("Processing 25 items in batches of 10...")
+
+    # Start batch processor
+    run_id = await start(batch_processor, offset=0, batch_size=10)
+
+    # Wait for completion
+    await asyncio.sleep(0.5)
+
+    # Get the workflow chain
+    chain = await get_workflow_chain(run_id, storage=storage)
+
+    print(f"\n  Workflow chain has {len(chain)} runs:")
+    for i, run in enumerate(chain):
+        status = run.status.value
+        result = run.result if run.result else "-"
+        print(f"    {i + 1}. {run.run_id[:20]}... - {status}")
+        if "Completed" in str(result):
+            print(f"       Result: {result}")
+
+    return run_id
+
+
+async def example_polling(storage):
+    """Example 2: Polling workflow."""
+    print("\n--- Example 2: Polling Workflow ---")
+    print("Polling for updates until no more available...")
+
+    # Start polling workflow
+    run_id = await start(polling_workflow)
+
+    # Wait for completion
+    await asyncio.sleep(0.5)
+
+    # Get the workflow chain
+    chain = await get_workflow_chain(run_id, storage=storage)
+
+    print(f"\n  Polling chain has {len(chain)} runs")
+
+    # Get final result
+    final_run = chain[-1]
+    if final_run.result:
+        print(f"  Final result: {final_run.result}")
+
+    return run_id
+
+
+async def example_countdown(storage):
+    """Example 3: Simple countdown."""
+    print("\n--- Example 3: Countdown ---")
+    print("Counting down from 3...")
+
+    # Start countdown
+    run_id = await start(countdown_workflow, count=3)
+
+    # Wait for completion
+    await asyncio.sleep(0.3)
+
+    # Get the workflow chain
+    chain = await get_workflow_chain(run_id, storage=storage)
+
+    print(f"\n  Chain: {' -> '.join(r.run_id[:8] + '...' for r in chain)}")
+    print(f"  Total runs: {len(chain)}")
+
+    return run_id
+
+
+async def example_view_chain_details(storage, run_id: str):
+    """Show detailed chain information."""
+    print("\n--- Chain Details ---")
+
+    chain = await get_workflow_chain(run_id, storage=storage)
+
+    for i, run in enumerate(chain):
+        position = "START" if i == 0 else ("CURRENT" if i == len(chain) - 1 else f"#{i + 1}")
+        print(f"\n  [{position}] {run.run_id}")
+        print(f"    Status: {run.status.value}")
+        print(f"    Continued from: {run.continued_from_run_id or '(none)'}")
+        print(f"    Continued to: {run.continued_to_run_id or '(none)'}")
+
+
+async def main():
+    # Use temp directory
+    with tempfile.TemporaryDirectory() as tmpdir:
+        print("=== Durable Workflow - Continue-As-New ===")
+
+        # Configure with FileStorageBackend
+        reset_config()
+        storage = FileStorageBackend(base_path=tmpdir)
+        configure(storage=storage, default_durable=True)
+
+        # Run examples
+        batch_run_id = await example_batch_processing(storage)
+        await example_polling(storage)
+        await example_countdown(storage)
+
+        # Show detailed chain for batch processing
+        await example_view_chain_details(storage, batch_run_id)
+
+        print("\n=== Key Takeaways ===")
+        print("  - continue_as_new() completes current run and starts fresh")
+        print("  - Each continuation has clean event history")
+        print("  - Use for long-running polling, batch processing, recurring tasks")
+        print("  - get_workflow_chain() retrieves all runs in the chain")
+        print("  - Runs are linked via continued_from_run_id/continued_to_run_id")
+        print("  - Requires at least one argument (explicit args required)")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/local/durable/12_schedules.py

@@ -0,0 +1,198 @@
+"""
+Durable Workflow - Schedules Example
+
+This example demonstrates scheduled workflow execution in local runtime.
+- Cron-based scheduling (every minute)
+- Interval-based scheduling (every 30 seconds)
+- Overlap policies to control concurrent executions
+- Schedule management (create, pause, resume, trigger)
+- LocalScheduler for automatic execution
+
+Run: python examples/local/durable/12_schedules.py 2>/dev/null
+
+Or use the CLI:
+    pyworkflow --module examples.local.durable.12_schedules scheduler run --duration 65
+"""
+
+import asyncio
+from datetime import datetime
+
+from pyworkflow import (
+    LocalScheduler,
+    OverlapPolicy,
+    ScheduleSpec,
+    configure,
+    create_schedule,
+    get_schedule,
+    list_schedules,
+    pause_schedule,
+    reset_config,
+    resume_schedule,
+    step,
+    trigger_schedule,
+    workflow,
+)
+from pyworkflow.storage import InMemoryStorageBackend
+
+
+# --- Steps ---
+@step()
+async def collect_metrics() -> dict:
+    """Collect system metrics."""
+    timestamp = datetime.now().isoformat()
+    print(f"  [Step] Collecting metrics at {timestamp}...")
+    return {
+        "timestamp": timestamp,
+        "cpu_usage": 45.2,
+        "memory_usage": 62.8,
+        "disk_usage": 78.1,
+    }
+
+
+@step()
+async def store_metrics(metrics: dict) -> dict:
+    """Store metrics (simulated)."""
+    print("  [Step] Storing metrics...")
+    return {**metrics, "stored": True}
+
+
+@step()
+async def check_alerts(metrics: dict) -> dict:
+    """Check if any metrics exceed thresholds."""
+    alerts = []
+    if metrics.get("cpu_usage", 0) > 80:
+        alerts.append("High CPU usage")
+    if metrics.get("memory_usage", 0) > 90:
+        alerts.append("High memory usage")
+    if metrics.get("disk_usage", 0) > 85:
+        alerts.append("High disk usage")
+
+    print(f"  [Step] Alert check: {alerts or 'None'}")
+    return {**metrics, "alerts": alerts}
+
+
+# --- Workflow ---
+@workflow(durable=True)
+async def metrics_workflow() -> dict:
+    """
+    Metrics collection workflow.
+
+    Collects system metrics, stores them, and checks for alerts.
+    """
+    metrics = await collect_metrics()
+    metrics = await store_metrics(metrics)
+    metrics = await check_alerts(metrics)
+    return metrics
+
+
+async def main():
+    # Configure with InMemoryStorageBackend
+    reset_config()
+    storage = InMemoryStorageBackend()
+    configure(storage=storage, default_durable=True)
+
+    print("=== Durable Workflow - Schedules Example ===\n")
+
+    # Create a schedule that runs every minute
+    print("Creating schedule (runs every minute)...")
+    spec = ScheduleSpec(
+        cron="* * * * *",  # Every minute
+        timezone="UTC",
+    )
+
+    schedule = await create_schedule(
+        workflow_name="metrics_workflow",
+        spec=spec,
+        overlap_policy=OverlapPolicy.SKIP,
+        schedule_id="metrics-every-minute",
+    )
+
+    print("\nSchedule created:")
+    print(f"  ID: {schedule.schedule_id}")
+    print(f"  Workflow: {schedule.workflow_name}")
+    print(f"  Cron: {schedule.spec.cron}")
+    print(f"  Next run: {schedule.next_run_time}")
+    print(f"  Overlap policy: {schedule.overlap_policy.value}")
+
+    # Also create an interval-based schedule
+    print("\nCreating interval schedule (every 30 seconds)...")
+    interval_spec = ScheduleSpec(interval="30s", timezone="UTC")
+
+    interval_schedule = await create_schedule(
+        workflow_name="metrics_workflow",
+        spec=interval_spec,
+        overlap_policy=OverlapPolicy.SKIP,
+        schedule_id="metrics-30s-interval",
+    )
+    print(f"  ID: {interval_schedule.schedule_id}")
+    print(f"  Interval: {interval_schedule.spec.interval}")
+
+    # Show all schedules
+    print("\n=== All Schedules ===")
+    schedules = await list_schedules()
+    for sched in schedules:
+        print(f"  {sched.schedule_id}: {sched.status.value}")
+
+    # Demonstrate pause/resume
+    print("\n=== Pause/Resume Demo ===")
+    print(f"Pausing {interval_schedule.schedule_id}...")
+    await pause_schedule(interval_schedule.schedule_id)
+
+    schedules = await list_schedules()
+    for sched in schedules:
+        print(f"  {sched.schedule_id}: {sched.status.value}")
+
+    print(f"\nResuming {interval_schedule.schedule_id}...")
+    await resume_schedule(interval_schedule.schedule_id)
+
+    # Demonstrate manual trigger
+    print("\n=== Manual Trigger Demo ===")
+    print("Triggering schedule immediately (bypasses cron)...")
+    run_id = await trigger_schedule(schedule.schedule_id)
+    print(f"Triggered run: {run_id}")
+
+    # Check stats after trigger
+    schedule = await get_schedule(schedule.schedule_id)
+    print("\nSchedule stats after trigger:")
+    print(f"  Total runs: {schedule.total_runs}")
+    print(f"  Successful: {schedule.successful_runs}")
+    print(f"  Failed: {schedule.failed_runs}")
+
+    # Run the LocalScheduler
+    print("\n" + "=" * 50)
+    print("Starting LocalScheduler to demonstrate automatic execution.")
+    print("The scheduler will poll for due schedules every 5 seconds.")
+    print("Watch for workflow executions when schedules become due!")
+    print("=" * 50)
+
+    # Create and run the local scheduler
+    local_scheduler = LocalScheduler(
+        storage=storage,
+        poll_interval=5.0,
+    )
+
+    print("\nScheduler running for 65 seconds...")
+    await local_scheduler.run(duration=65.0)
+
+    # Final stats
+    print("\n=== Final Schedule Stats ===")
+    for sched_id in ["metrics-every-minute", "metrics-30s-interval"]:
+        sched = await get_schedule(sched_id)
+        if sched:
+            print(f"\n{sched.schedule_id}:")
+            print(f"  Total runs: {sched.total_runs}")
+            print(f"  Successful: {sched.successful_runs}")
+            print(f"  Last run: {sched.last_run_at}")
+
+    print("\n=== Key Takeaways ===")
+    print("- Schedules created with cron and interval specs")
+    print("- LocalScheduler polls storage and triggers due schedules")
+    print("- Overlap policies prevent concurrent runs")
+    print("- Pause/resume controls schedule execution")
+    print("- Manual trigger bypasses schedule timing")
+    print("- For production, use: pyworkflow scheduler run")
+    print("- For Celery, use: pyworkflow worker run --beat")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/local/durable/__init__.py

@@ -0,0 +1 @@
+# PyWorkflow Local Durable Examples Package

examples/local/transient/01_quick_tasks.py

@@ -0,0 +1,87 @@
+"""
+Transient Workflow - Quick Tasks
+
+This example demonstrates simple transient mode execution.
+- Simple 3-step order workflow
+- No storage backend required
+- Fast, direct execution
+- No event recording
+
+Run: python examples/local/transient/01_quick_tasks.py 2>/dev/null
+"""
+
+import asyncio
+
+from pyworkflow import (
+    configure,
+    reset_config,
+    start,
+    step,
+    workflow,
+)
+
+
+# --- Steps ---
+@step()
+async def process_order(order_id: str) -> dict:
+    """Process the order and validate it."""
+    print(f"  Processing order {order_id}...")
+    return {"order_id": order_id, "status": "processed"}
+
+
+@step()
+async def charge_payment(order: dict, amount: float) -> dict:
+    """Charge the payment for the order."""
+    print(f"  Charging payment: ${amount:.2f}...")
+    return {**order, "charged": amount}
+
+
+@step()
+async def send_notification(order: dict) -> dict:
+    """Send order confirmation notification."""
+    print(f"  Sending notification for order {order['order_id']}...")
+    return {**order, "notified": True}
+
+
+# --- Workflow ---
+@workflow(durable=False, tags=["local", "transient"])
+async def order_workflow(order_id: str, amount: float) -> dict:
+    """Complete order processing workflow (transient mode)."""
+    order = await process_order(order_id)
+    order = await charge_payment(order, amount)
+    order = await send_notification(order)
+    return order
+
+
+async def main():
+    # Configure for transient mode (no storage backend needed)
+    reset_config()
+    configure(default_durable=False)
+
+    print("=== Transient Workflow - Quick Tasks ===\n")
+    print("Running order workflow in transient mode...\n")
+
+    # Start workflow
+    run_id = await start(order_workflow, "order-123", 99.99)
+
+    print(f"\nWorkflow completed: {run_id}")
+
+    print("\n=== Key Characteristics ===")
+    print("✓ No storage backend required")
+    print("✓ Fast execution (no event recording overhead)")
+    print("✓ Perfect for scripts and CLI tools")
+    print("✓ State lost on process exit (no crash recovery)")
+
+    print("\n=== When to Use Transient Mode ===")
+    print("✓ Short-lived workflows (seconds to minutes)")
+    print("✓ CLI tools and data processing scripts")
+    print("✓ Development and testing")
+    print("✓ Tasks where simplicity > durability")
+
+    print("\n=== Comparison with Durable Mode ===")
+    print("For crash recovery and persistence, see:")
+    print("  examples/local/durable/01_basic_workflow.py")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/local/transient/02_retries.py

@@ -0,0 +1,130 @@
+"""
+Transient Workflow - Retry Mechanics
+
+This example demonstrates inline retry behavior in transient mode.
+- Shows @step(max_retries=...) in action
+- Simulates flaky API (fails 2x, succeeds on 3rd try)
+- Retry logic works without event sourcing
+- Global counter tracks retry attempts
+
+Run: python examples/local/transient/02_retries.py 2>/dev/null
+"""
+
+import asyncio
+
+from pyworkflow import (
+    FatalError,
+    configure,
+    reset_config,
+    start,
+    step,
+    workflow,
+)
+
+# Global counter to track API call attempts
+attempt_count = 0
+
+
+# --- Steps ---
+@step()
+async def validate_request(request_id: str) -> dict:
+    """Validate the request."""
+    print(f"  Validating request {request_id}...")
+    return {"request_id": request_id, "valid": True}
+
+
+@step(max_retries=3, retry_delay=1)
+async def call_flaky_api(request: dict) -> dict:
+    """Simulate unreliable external API - fails twice then succeeds."""
+    global attempt_count
+    attempt_count += 1
+
+    print(f"  Calling external API (attempt {attempt_count})...")
+
+    if attempt_count < 3:
+        # Simulate temporary failure
+        print("    ✗ API call failed (timeout)")
+        raise Exception(f"API timeout - connection refused (attempt {attempt_count})")
+
+    # Third attempt succeeds
+    print("    ✓ API call successful!")
+    return {**request, "api_response": "success", "attempts": attempt_count}
+
+
+@step()
+async def process_response(request: dict) -> dict:
+    """Process the successful API response."""
+    print(f"  Processing API response for {request['request_id']}...")
+    return {**request, "processed": True}
+
+
+@step()
+async def validate_input(value: int) -> int:
+    """Validate input - demonstrates FatalError (no retry)."""
+    if value < 0:
+        raise FatalError("Negative values not allowed")
+    return value
+
+
+# --- Workflows ---
+@workflow(durable=False, tags=["local", "transient"])
+async def api_workflow(request_id: str) -> dict:
+    """Workflow with automatic retry logic."""
+    request = await validate_request(request_id)
+    request = await call_flaky_api(request)  # Will retry on failure
+    request = await process_response(request)
+    return request
+
+
+@workflow(durable=False, tags=["local", "transient"])
+async def validation_workflow(value: int) -> int:
+    """Workflow with fatal error (no retry)."""
+    return await validate_input(value)
+
+
+async def main():
+    global attempt_count
+
+    # Configure for transient mode
+    reset_config()
+    configure(default_durable=False)
+
+    print("=== Transient Workflow - Retry Mechanics ===\n")
+
+    # Example 1: Successful retry
+    print("Example 1: API call with retries\n")
+    attempt_count = 0  # Reset counter
+
+    run_id = await start(api_workflow, "request-123")
+    print(f"\nWorkflow completed: {run_id}")
+    print(f"Total attempts: {attempt_count}")
+
+    # Example 2: FatalError (no retry)
+    print("\n" + "=" * 60)
+    print("\nExample 2: FatalError (no retry)\n")
+
+    try:
+        await start(validation_workflow, -5)
+    except FatalError as e:
+        print(f"✗ Workflow failed with FatalError: {e}")
+        print("  (No retries attempted)")
+
+    print("\n=== Retry Behavior ===")
+    print("✓ max_retries=3 means: 1 initial + 3 retries = 4 total attempts")
+    print("✓ retry_delay=1 adds 1 second delay between retries")
+    print("✓ Retries happen inline (no event log needed)")
+    print("✓ FatalError skips retries and fails immediately")
+
+    print("\n=== Error Types ===")
+    print("Exception           - Will retry if max_retries > 0")
+    print("FatalError          - Never retries, fails immediately")
+    print("RetryableError      - Will retry (same as Exception)")
+
+    print("\n=== Difference from Durable Mode ===")
+    print("Transient: Retries happen inline, not recorded")
+    print("Durable:   Retries recorded in event log for audit")
+    print("\nSee examples/local/durable/03_retries.py for comparison")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())