pyworkflow-engine 0.1.7__py3-none-any.whl → 0.1.10__py3-none-any.whl

examples/local/durable/01_basic_workflow.py

@@ -1,94 +0,0 @@
-"""
-Durable Workflow - Basic Example
-
-This example demonstrates a simple event-sourced workflow using InMemoryStorageBackend.
-- 3-step order processing workflow
-- Events recorded for each step
-- Event log inspection after completion
-- Basic @workflow and @step decorators
-
-Run: python examples/local/durable/01_basic_workflow.py 2>/dev/null
-"""
-
-import asyncio
-
-from pyworkflow import (
-    configure,
-    get_workflow_events,
-    get_workflow_run,
-    reset_config,
-    start,
-    step,
-    workflow,
-)
-from pyworkflow.storage import InMemoryStorageBackend
-
-
-# --- Steps ---
-@step()
-async def process_order(order_id: str) -> dict:
-    """Process the order and validate it."""
-    print(f"  Processing order {order_id}...")
-    return {"order_id": order_id, "status": "processed"}
-
-
-@step()
-async def charge_payment(order: dict, amount: float) -> dict:
-    """Charge the payment for the order."""
-    print(f"  Charging payment: ${amount:.2f}...")
-    return {**order, "charged": amount}
-
-
-@step()
-async def send_notification(order: dict) -> dict:
-    """Send order confirmation notification."""
-    print(f"  Sending notification for order {order['order_id']}...")
-    return {**order, "notified": True}
-
-
-# --- Workflow ---
-@workflow(durable=True, tags=["local", "durable"])
-async def order_workflow(order_id: str, amount: float) -> dict:
-    """Complete order processing workflow."""
-    order = await process_order(order_id)
-    order = await charge_payment(order, amount)
-    order = await send_notification(order)
-    return order
-
-
-async def main():
-    # Configure with InMemoryStorageBackend
-    reset_config()
-    storage = InMemoryStorageBackend()
-    configure(storage=storage, default_durable=True)
-
-    print("=== Durable Workflow - Basic Example ===\n")
-    print("Running order workflow...")
-
-    # Start workflow
-    run_id = await start(order_workflow, "order-123", 99.99)
-    print(f"\nWorkflow completed: {run_id}\n")
-
-    # Check workflow status
-    run = await get_workflow_run(run_id)
-    print(f"Status: {run.status.value}")
-    print(f"Result: {run.result}")
-
-    # Inspect event log
-    events = await get_workflow_events(run_id)
-    print(f"\n=== Event Log ({len(events)} events) ===")
-    for event in events:
-        print(f"  {event.sequence}: {event.type.value}")
-        if event.type.value == "step_completed":
-            step_name = event.data.get("step_name", "unknown")
-            print(f"     Step: {step_name}")
-
-    print("\n=== Key Takeaways ===")
-    print("✓ Workflow executed with event sourcing")
-    print("✓ Each step recorded as an event")
-    print("✓ InMemoryStorageBackend used (data lost on exit)")
-    print("✓ Try 02_file_storage.py for persistence!")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

examples/local/durable/02_file_storage.py

@@ -1,132 +0,0 @@
-"""
-Durable Workflow - File Storage
-
-This example demonstrates persistent workflow storage using FileStorageBackend.
-- Same 3-step workflow as 01_basic_workflow.py
-- Data persists to filesystem in workflow_data/ directory
-- Human-readable JSON files
-- Inspect stored file structure and JSONL event log
-
-Run: python examples/local/durable/02_file_storage.py 2>/dev/null
-"""
-
-import asyncio
-import json
-import os
-from pathlib import Path
-
-from pyworkflow import (
-    configure,
-    get_workflow_events,
-    get_workflow_run,
-    reset_config,
-    start,
-    step,
-    workflow,
-)
-from pyworkflow.storage import FileStorageBackend
-
-
-# --- Steps ---
-@step()
-async def process_order(order_id: str) -> dict:
-    """Process the order and validate it."""
-    print(f"  Processing order {order_id}...")
-    return {"order_id": order_id, "status": "processed"}
-
-
-@step()
-async def charge_payment(order: dict, amount: float) -> dict:
-    """Charge the payment for the order."""
-    print(f"  Charging payment: ${amount:.2f}...")
-    return {**order, "charged": amount}
-
-
-@step()
-async def send_notification(order: dict) -> dict:
-    """Send order confirmation notification."""
-    print(f"  Sending notification for order {order['order_id']}...")
-    return {**order, "notified": True}
-
-
-# --- Workflow ---
-@workflow(durable=True, tags=["local", "durable"])
-async def order_workflow(order_id: str, amount: float) -> dict:
-    """Complete order processing workflow."""
-    order = await process_order(order_id)
-    order = await charge_payment(order, amount)
-    order = await send_notification(order)
-    return order
-
-
-async def main():
-    # Use local directory for persistence (added to .gitignore)
-    data_dir = Path(__file__).parent / "workflow_data"
-    data_dir.mkdir(exist_ok=True)
-
-    print("=== Durable Workflow - File Storage ===\n")
-    print(f"Storage directory: {data_dir}\n")
-
-    # Configure with FileStorageBackend
-    reset_config()
-    storage = FileStorageBackend(base_path=str(data_dir))
-    configure(storage=storage, default_durable=True)
-
-    print("Running order workflow...")
-
-    # Start workflow
-    run_id = await start(order_workflow, "order-456", 149.99)
-    print(f"\nWorkflow completed: {run_id}\n")
-
-    # Check workflow status
-    run = await get_workflow_run(run_id)
-    print(f"Status: {run.status.value}")
-    print(f"Result: {run.result}")
-
-    # Show events
-    events = await get_workflow_events(run_id)
-    print(f"\n=== Event Log ({len(events)} events) ===")
-    for event in events:
-        print(f"  {event.sequence}: {event.type.value}")
-
-    # Show stored files
-    print("\n=== Stored Files ===")
-    for root, dirs, files in os.walk(data_dir):
-        # Skip hidden directories (.locks)
-        dirs[:] = [d for d in dirs if not d.startswith(".")]
-        for f in files:
-            path = os.path.join(root, f)
-            rel_path = os.path.relpath(path, data_dir)
-            size = os.path.getsize(path)
-            print(f"  {rel_path} ({size} bytes)")
-
-    # Show JSONL event log contents
-    event_log_path = data_dir / "events" / f"{run_id}.jsonl"
-    if event_log_path.exists():
-        print(f"\n=== Event Log File Contents ({event_log_path.name}) ===")
-        with open(event_log_path) as f:
-            for i, line in enumerate(f, 1):
-                event_data = json.loads(line.strip())
-                event_type = event_data.get("type", "unknown")
-                print(f"  Line {i}: {event_type}")
-                # Show full data for first event
-                if i == 1:
-                    print(f"    Full data: {json.dumps(event_data, indent=6)}")
-
-    print("\n=== Directory Structure ===")
-    print("  runs/       - Workflow run metadata (JSON)")
-    print("  events/     - Event log (JSONL, append-only)")
-    print("  steps/      - Step execution records (JSON)")
-    print("  .locks/     - Internal file locks")
-
-    print("\n=== Key Takeaways ===")
-    print("✓ Data persists to filesystem in workflow_data/")
-    print("✓ Human-readable JSON format")
-    print("✓ JSONL (JSON Lines) for event log (one event per line)")
-    print("✓ Survives process restarts")
-    print("✓ Good for development and single-machine deployments")
-    print(f"\nℹ  Storage persisted at: {data_dir.absolute()}")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

examples/local/durable/03_retries.py

@@ -1,169 +0,0 @@
-"""
-Durable Workflow - Automatic Retries with Suspension
-
-This example demonstrates automatic retry behavior with workflow suspension/resumption.
-- Simulates flaky external API (fails 2x, succeeds on 3rd try)
-- Workflow suspends between retry attempts
-- Retry events recorded in event log for audit trail
-- Demonstrates manual resume after each retry delay
-
-IMPORTANT: Manual resumption (await resume(run_id)) is ONLY for local development/CI.
-In production, use Celery runtime for automatic scheduled resumption.
-See examples/celery/ for production-ready distributed execution.
-
-Run: python examples/local/durable/03_retries.py 2>/dev/null
-"""
-
-import asyncio
-
-from pyworkflow import (
-    configure,
-    get_workflow_events,
-    get_workflow_run,
-    reset_config,
-    resume,
-    start,
-    step,
-    workflow,
-)
-from pyworkflow.storage import InMemoryStorageBackend
-
-# Simulate API call counter
-attempt_count = 0
-
-
-# --- Steps ---
-@step()
-async def validate_order(order_id: str) -> dict:
-    """Validate the order."""
-    print(f"  Validating order {order_id}...")
-    return {"order_id": order_id, "valid": True}
-
-
-@step(max_retries=3, retry_delay=1)
-async def call_flaky_api(order: dict) -> dict:
-    """Simulate unreliable external API - fails twice then succeeds."""
-    global attempt_count
-    attempt_count += 1
-
-    print(f"  Calling external API (attempt {attempt_count})...")
-
-    if attempt_count < 3:
-        # Simulate temporary failure
-        raise Exception(f"API timeout - connection refused (attempt {attempt_count})")
-
-    # Third attempt succeeds
-    print(f"  ✓ API call successful on attempt {attempt_count}!")
-    return {**order, "api_response": "payment_approved", "attempts": attempt_count}
-
-
-@step()
-async def finalize_order(order: dict) -> dict:
-    """Finalize the order after successful API call."""
-    print(f"  Finalizing order {order['order_id']}...")
-    return {**order, "finalized": True}
-
-
-# --- Workflow ---
-@workflow(durable=True, tags=["local", "durable"])
-async def order_workflow(order_id: str) -> dict:
-    """Complete order processing with retry logic."""
-    order = await validate_order(order_id)
-    order = await call_flaky_api(order)  # Will retry on failure
-    order = await finalize_order(order)
-    return order
-
-
-async def main():
-    global attempt_count
-
-    # Configure with InMemoryStorageBackend
-    reset_config()
-    storage = InMemoryStorageBackend()
-    configure(storage=storage, default_durable=True)
-
-    print("=== Durable Workflow - Automatic Retries with Suspension ===\n")
-    print("Simulating flaky API (fails 2x, succeeds on 3rd try)...\n")
-
-    # Reset counter
-    attempt_count = 0
-
-    # Start workflow
-    print("Starting workflow...")
-    run_id = await start(order_workflow, "order-789")
-
-    # Check status after first attempt
-    run = await get_workflow_run(run_id)
-    print(f"\nStatus after attempt 1: {run.status.value}")
-
-    if run.status.value == "suspended":
-        print("→ Workflow suspended for retry (waiting 1 second...)")
-
-        # Show events so far
-        events = await get_workflow_events(run_id)
-        print(f"\n=== Event Log (After Attempt 1) - {len(events)} events ===")
-        for event in events:
-            event_type = event.type.value
-            attempt = event.data.get("attempt", "?")
-            print(f"  {event.sequence}: {event_type} (attempt={attempt})")
-            if event_type == "step_retrying":
-                next_attempt = event.data.get("attempt")
-                print(f"     → Will retry as attempt {next_attempt}")
-
-        # Wait for retry delay and resume
-        await asyncio.sleep(1.5)
-        print("\nResuming workflow for attempt 2...")
-        await resume(run_id)
-
-        # Check status again
-        run = await get_workflow_run(run_id)
-        print(f"Status after attempt 2: {run.status.value}")
-
-        if run.status.value == "suspended":
-            print("→ Workflow suspended for retry again (waiting 1 second...)")
-
-            # Wait and resume for attempt 3
-            await asyncio.sleep(1.5)
-            print("\nResuming workflow for attempt 3...")
-            result = await resume(run_id)
-
-            print("\n✓ Workflow completed successfully!")
-            print(f"Result: {result}")
-
-    # Final status
-    run = await get_workflow_run(run_id)
-    print(f"\nFinal status: {run.status.value}")
-
-    # Show complete event log
-    events = await get_workflow_events(run_id)
-    print(f"\n=== Complete Event Log ({len(events)} events) ===")
-
-    for event in events:
-        event_type = event.type.value
-        attempt = event.data.get("attempt", "")
-        step_name = event.data.get("step_name", "")
-
-        if attempt:
-            print(f"  {event.sequence}: {event_type} (attempt={attempt}, step={step_name})")
-        else:
-            print(f"  {event.sequence}: {event_type}")
-
-        if event_type == "step_failed":
-            error = event.data.get("error", "")[:50]
-            print(f"     Error: {error}...")
-        elif event_type == "step_retrying":
-            retry_after = event.data.get("retry_after")
-            resume_at = event.data.get("resume_at", "")[:19]
-            print(f"     Retry after: {retry_after}s, resume at: {resume_at}")
-
-    print("\n=== Key Takeaways ===")
-    print("✓ Workflow suspends between retry attempts (releases resources)")
-    print("✓ Each retry requires manual resume() or automatic Celery scheduling")
-    print("✓ Event log shows STEP_FAILED + STEP_RETRYING for each retry")
-    print("✓ Resume restores state via event replay and continues from retry")
-    print("✓ max_retries=3, retry_delay=1 (1 initial + 3 retries = 4 total attempts)")
-    print(f"✓ Total attempts in this run: {attempt_count}")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

examples/local/durable/04_long_running.py

@@ -1,119 +0,0 @@
-"""
-Durable Workflow - Long Running with Sleep
-
-This example demonstrates workflow suspension and resumption with sleep().
-- Workflow suspends during sleep (releases resources)
-- Can be resumed after sleep completes
-- Uses FileStorageBackend for persistence across process restarts
-- Demonstrates manual resumption pattern
-
-IMPORTANT: Manual resumption (await resume(run_id)) is ONLY for local development/CI.
-In production, use Celery runtime for automatic scheduled resumption.
-See examples/celery/ for production-ready distributed execution.
-
-Run: python examples/local/durable/04_long_running.py 2>/dev/null
-"""
-
-import asyncio
-import tempfile
-
-from pyworkflow import (
-    configure,
-    get_workflow_run,
-    reset_config,
-    resume,
-    sleep,
-    start,
-    step,
-    workflow,
-)
-from pyworkflow.storage import FileStorageBackend
-
-
-# --- Steps ---
-@step()
-async def prepare_batch(batch_id: str) -> dict:
-    """Prepare the batch for processing."""
-    print(f"  Preparing batch {batch_id}...")
-    return {"batch_id": batch_id, "status": "prepared"}
-
-
-@step()
-async def process_batch(batch: dict) -> dict:
-    """Process the batch after sleep completes."""
-    print(f"  Processing batch {batch['batch_id']}...")
-    return {**batch, "status": "processed", "items": 1000}
-
-
-@step()
-async def finalize_batch(batch: dict) -> dict:
-    """Finalize the batch."""
-    print(f"  Finalizing batch {batch['batch_id']}...")
-    return {**batch, "status": "completed"}
-
-
-# --- Workflow ---
-@workflow(durable=True, tags=["local", "durable"])
-async def batch_workflow(batch_id: str) -> dict:
-    """Long-running batch processing workflow with sleep."""
-    batch = await prepare_batch(batch_id)
-
-    print("  Sleeping for 5 seconds (workflow will suspend)...")
-    await sleep("5s")  # Suspends workflow here
-
-    print("  Resuming after sleep...")
-    batch = await process_batch(batch)
-    batch = await finalize_batch(batch)
-    return batch
-
-
-async def main():
-    # Use temp directory (use real path like "./workflow_data" for production)
-    with tempfile.TemporaryDirectory() as tmpdir:
-        print("=== Durable Workflow - Long Running ===\n")
-
-        # Configure with FileStorageBackend (for persistence)
-        reset_config()
-        storage = FileStorageBackend(base_path=tmpdir)
-        configure(storage=storage, default_durable=True)
-
-        print("Starting batch workflow...\n")
-
-        # Start workflow
-        run_id = await start(batch_workflow, "batch-001")
-
-        # Check status after start
-        run = await get_workflow_run(run_id)
-        print(f"\nWorkflow status after sleep: {run.status.value}")
-
-        if run.status.value == "suspended":
-            print("Workflow is suspended (waiting for sleep to complete)")
-
-            # Wait for sleep duration, then resume
-            print("\nWaiting 5 seconds for sleep to complete...")
-            await asyncio.sleep(5)
-
-            print(f"Resuming workflow {run_id}...\n")
-            result = await resume(run_id)
-
-            print("\nWorkflow completed!")
-            print(f"Result: {result}")
-        else:
-            # Workflow already completed (sleep was short enough)
-            print("Workflow completed without suspension")
-            print(f"Result: {run.result}")
-
-        # Final status check
-        run = await get_workflow_run(run_id)
-        print(f"\nFinal status: {run.status.value}")
-
-        print("\n=== Key Takeaways ===")
-        print("✓ Workflow suspends during sleep() (releases resources)")
-        print("✓ FileStorageBackend persists state during suspension")
-        print("✓ Can resume after sleep completes (even after process restart)")
-        print("✓ Perfect for rate limiting, scheduled tasks, waiting for events")
-        print("\nℹ  For production: use real storage path, implement auto-resume logic")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

examples/local/durable/05_event_log.py

@@ -1,145 +0,0 @@
-"""
-Durable Workflow - Event Log Deep Dive
-
-This example demonstrates detailed event sourcing inspection.
-- Multiple workflows to show different event sequences
-- Deep dive into event structure (sequence, type, timestamp, data)
-- Understanding event types and their meaning
-- Event replay concepts
-
-Run: python examples/local/durable/05_event_log.py 2>/dev/null
-"""
-
-import asyncio
-
-from pyworkflow import (
-    configure,
-    get_workflow_events,
-    get_workflow_run,
-    reset_config,
-    start,
-    step,
-    workflow,
-)
-from pyworkflow.storage import InMemoryStorageBackend
-
-
-# --- Steps ---
-@step()
-async def step_a(value: int) -> int:
-    """Simple step that doubles the value."""
-    return value * 2
-
-
-@step()
-async def step_b(value: int) -> int:
-    """Simple step that adds 10."""
-    return value + 10
-
-
-@step()
-async def step_c(value: int) -> int:
-    """Simple step that subtracts 5."""
-    return value - 5
-
-
-# --- Workflows ---
-@workflow(durable=True, tags=["local", "durable"])
-async def simple_workflow(value: int) -> int:
-    """Simple 2-step workflow."""
-    result = await step_a(value)
-    result = await step_b(result)
-    return result
-
-
-@workflow(durable=True, tags=["local", "durable"])
-async def complex_workflow(value: int) -> int:
-    """More complex 3-step workflow."""
-    result = await step_a(value)
-    result = await step_b(result)
-    result = await step_c(result)
-    return result
-
-
-def print_event_details(event, index: int):
-    """Pretty print event details."""
-    print(f"\nEvent #{index + 1}:")
-    print(f"  Sequence: {event.sequence}")
-    print(f"  Type: {event.type.value}")
-    print(f"  Timestamp: {event.timestamp.strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]}")
-
-    if event.data:
-        print("  Data:")
-        for key, value in event.data.items():
-            # Format value nicely
-            if isinstance(value, str) and len(value) > 50:
-                value = value[:50] + "..."
-            print(f"    {key}: {value}")
-
-
-async def main():
-    # Configure with InMemoryStorageBackend
-    reset_config()
-    storage = InMemoryStorageBackend()
-    configure(storage=storage, default_durable=True)
-
-    print("=== Durable Workflow - Event Log Deep Dive ===\n")
-
-    # Run first workflow
-    print("Running simple_workflow(5)...\n")
-    run_id_1 = await start(simple_workflow, 5)
-
-    run = await get_workflow_run(run_id_1)
-    print(f"Result: {run.result}")
-    print(f"Status: {run.status.value}")
-
-    # Inspect events
-    events = await get_workflow_events(run_id_1)
-    print(f"\n=== Event Log for simple_workflow ({len(events)} events) ===")
-
-    for i, event in enumerate(events):
-        print_event_details(event, i)
-
-    # Run second workflow
-    print("\n" + "=" * 60)
-    print("\nRunning complex_workflow(10)...\n")
-    run_id_2 = await start(complex_workflow, 10)
-
-    run = await get_workflow_run(run_id_2)
-    print(f"Result: {run.result}")
-    print(f"Status: {run.status.value}")
-
-    # Inspect events
-    events = await get_workflow_events(run_id_2)
-    print(f"\n=== Event Log for complex_workflow ({len(events)} events) ===")
-
-    for i, event in enumerate(events):
-        print_event_details(event, i)
-
-    # Event type summary
-    print("\n" + "=" * 60)
-    print("\n=== Event Types Explained ===")
-    print("workflow_started   - Workflow execution begins")
-    print("step_completed     - Step successfully executed (result cached)")
-    print("step_failed        - Step failed (will retry if configured)")
-    print("sleep_started      - Workflow suspended (sleep begins)")
-    print("sleep_completed    - Workflow resumed (sleep ends)")
-    print("workflow_completed - Workflow finished successfully")
-    print("workflow_failed    - Workflow failed permanently")
-
-    print("\n=== Event Replay Concepts ===")
-    print("✓ Events are immutable - never modified, only appended")
-    print("✓ On crash/restart, events replayed to restore state")
-    print("✓ step_completed events: result cached, step not re-executed")
-    print("✓ Sequence numbers ensure deterministic ordering")
-    print("✓ Timestamps enable time-travel debugging")
-
-    print("\n=== Key Takeaways ===")
-    print("✓ Every state change recorded as an event")
-    print("✓ Events contain sequence, type, timestamp, and data")
-    print("✓ Event log enables crash recovery via replay")
-    print("✓ Complete audit trail for compliance and debugging")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())