pyworkflow-engine 0.1.7__py3-none-any.whl → 0.1.10__py3-none-any.whl

examples/celery/durable/workflows/schedules.py

@@ -1,209 +0,0 @@
-"""
-Celery Durable Workflow - Schedules Example
-
-This example demonstrates scheduled workflow execution with Celery Beat.
-- Cron-based scheduling (every minute)
-- Interval-based scheduling (every 30 seconds)
-- Overlap policies to control concurrent executions
-- Schedule management (pause, resume, delete)
-
-Prerequisites:
-    1. Start Redis: docker run -d -p 6379:6379 redis:7-alpine
-    2. Start worker: pyworkflow --module examples.celery.durable.12_schedules worker run
-    3. Start beat: pyworkflow --module examples.celery.durable.12_schedules beat run
-
-CLI Commands:
-    # Create a schedule via CLI
-    pyworkflow schedules create metrics_workflow --cron "* * * * *" --overlap skip
-
-    # List all schedules
-    pyworkflow schedules list
-
-    # Pause/resume a schedule
-    pyworkflow schedules pause <schedule_id>
-    pyworkflow schedules resume <schedule_id>
-
-    # Trigger immediately (bypass schedule)
-    pyworkflow schedules trigger <schedule_id>
-
-    # View schedule details
-    pyworkflow schedules show <schedule_id>
-"""
-
-from datetime import datetime
-
-from pyworkflow import (
-    OverlapPolicy,
-    scheduled_workflow,
-    step,
-    workflow,
-)
-
-
-# --- Steps ---
-@step()
-async def collect_metrics() -> dict:
-    """Collect system metrics."""
-    timestamp = datetime.now().isoformat()
-    print(f"[Step] Collecting metrics at {timestamp}...")
-    return {
-        "timestamp": timestamp,
-        "cpu_usage": 45.2,
-        "memory_usage": 62.8,
-        "disk_usage": 78.1,
-    }
-
-
-@step()
-async def store_metrics(metrics: dict) -> dict:
-    """Store metrics in database (simulated)."""
-    print(f"[Step] Storing metrics: {metrics}")
-    return {**metrics, "stored": True}
-
-
-@step()
-async def check_alerts(metrics: dict) -> dict:
-    """Check if any metrics exceed thresholds."""
-    alerts = []
-    if metrics.get("cpu_usage", 0) > 80:
-        alerts.append("High CPU usage")
-    if metrics.get("memory_usage", 0) > 90:
-        alerts.append("High memory usage")
-    if metrics.get("disk_usage", 0) > 85:
-        alerts.append("High disk usage")
-
-    print(f"[Step] Alert check complete. Alerts: {alerts or 'None'}")
-    return {**metrics, "alerts": alerts}
-
-
-# --- Scheduled Workflow (using decorator) ---
-@scheduled_workflow(
-    cron="* * * * *",  # Every minute
-    overlap_policy=OverlapPolicy.SKIP,  # Skip if previous run still active
-    timezone="UTC",
-)
-async def metrics_workflow() -> dict:
-    """
-    Scheduled metrics collection workflow.
-
-    Runs every minute via Celery Beat.
-    Uses SKIP overlap policy - if a previous run is still active,
-    new runs are skipped to prevent resource exhaustion.
-
-    Steps:
-    1. Collect current system metrics
-    2. Store metrics in database
-    3. Check for threshold alerts
-    """
-    metrics = await collect_metrics()
-    metrics = await store_metrics(metrics)
-    metrics = await check_alerts(metrics)
-    return metrics
-
-
-# --- Regular Workflow (for programmatic scheduling) ---
-@workflow()
-async def cleanup_workflow(days_old: int = 30) -> dict:
-    """
-    Cleanup old data workflow.
-
-    This workflow is scheduled programmatically in main().
-    """
-    print(f"[Workflow] Cleaning up data older than {days_old} days...")
-    return {"cleaned": True, "days_old": days_old}
-
-
-async def main() -> None:
-    """
-    Create schedules programmatically.
-
-    The @scheduled_workflow decorator automatically creates a schedule
-    when activate_scheduled_workflows() is called (done by Beat).
-
-    For regular @workflow functions, use create_schedule() to create
-    schedules programmatically.
-    """
-    import argparse
-
-    from pyworkflow import (
-        OverlapPolicy,
-        ScheduleSpec,
-        create_schedule,
-        delete_schedule,
-        list_schedules,
-        pause_schedule,
-        resume_schedule,
-    )
-
-    parser = argparse.ArgumentParser(description="Schedule Management Example")
-    parser.add_argument(
-        "--action",
-        choices=["create", "list", "pause", "resume", "delete"],
-        default="create",
-        help="Action to perform",
-    )
-    parser.add_argument("--schedule-id", help="Schedule ID for pause/resume/delete")
-    args = parser.parse_args()
-
-    print("=== Celery Schedules Example ===\n")
-
-    if args.action == "create":
-        # Create a schedule for the cleanup workflow
-        print("Creating cleanup schedule (runs every 2 minutes)...")
-        spec = ScheduleSpec(cron="*/2 * * * *", timezone="UTC")
-
-        schedule = await create_schedule(
-            workflow_name="cleanup_workflow",
-            spec=spec,
-            overlap_policy=OverlapPolicy.SKIP,
-            schedule_id="cleanup-hourly",
-            days_old=7,  # kwargs passed to workflow
-        )
-        print(f"Schedule created: {schedule.schedule_id}")
-        print(f"  Workflow: {schedule.workflow_name}")
-        print(f"  Cron: {schedule.spec.cron}")
-        print(f"  Next run: {schedule.next_run_time}")
-
-        # Also show the decorated workflow schedule
-        print("\nThe @scheduled_workflow decorator creates:")
-        print("  - metrics_workflow: runs every minute")
-        print("  - Activated automatically when Beat starts")
-
-    elif args.action == "list":
-        schedules = await list_schedules()
-        print(f"Found {len(schedules)} schedule(s):\n")
-        for sched in schedules:
-            print(f"  {sched.schedule_id}")
-            print(f"    Workflow: {sched.workflow_name}")
-            print(f"    Status: {sched.status.value}")
-            print(f"    Spec: cron={sched.spec.cron}, interval={sched.spec.interval}")
-            print(f"    Total runs: {sched.total_runs}")
-            print()
-
-    elif args.action == "pause" and args.schedule_id:
-        schedule = await pause_schedule(args.schedule_id)
-        print(f"Paused schedule: {schedule.schedule_id}")
-        print(f"Status: {schedule.status.value}")
-
-    elif args.action == "resume" and args.schedule_id:
-        schedule = await resume_schedule(args.schedule_id)
-        print(f"Resumed schedule: {schedule.schedule_id}")
-        print(f"Status: {schedule.status.value}")
-
-    elif args.action == "delete" and args.schedule_id:
-        await delete_schedule(args.schedule_id)
-        print(f"Deleted schedule: {args.schedule_id}")
-
-    else:
-        print("Invalid action or missing schedule-id")
-
-    print("\n=== How to Run ===")
-    print("1. Start worker: pyworkflow --module examples.celery.durable.12_schedules worker run")
-    print("2. Start beat:   pyworkflow --module examples.celery.durable.12_schedules beat run")
-    print("3. Watch logs to see scheduled executions!")
-
-
-if __name__ == "__main__":
-    import asyncio
-
-    asyncio.run(main())

examples/celery/transient/01_basic_workflow.py

@@ -1,91 +0,0 @@
-"""
-Celery Transient Workflow - Basic Example
-
-This example demonstrates a simple transient workflow running on Celery workers.
-
-Transient workflows:
-- Do NOT record events
-- Do NOT persist state
-- Are simpler and faster
-- Best for short-lived, stateless tasks
-
-Prerequisites:
-    1. Start Redis: docker run -d -p 6379:6379 redis:7-alpine
-    2. Start worker: pyworkflow --module examples.celery.transient.01_basic_workflow worker run
-
-Run with CLI:
-    pyworkflow --module examples.celery.transient.01_basic_workflow workflows run quick_task \
-        --arg item_id=item-123
-
-Note: Since this is transient, runs list and runs status won't show this workflow.
-"""
-
-import asyncio
-
-from pyworkflow import step, workflow
-
-
-@step(name="transient_process_item")
-async def process_item(item_id: str) -> dict:
-    """Process a single item."""
-    print(f"[Step] Processing item {item_id}...")
-    await asyncio.sleep(0.5)  # Simulate quick processing
-    return {"item_id": item_id, "processed": True}
-
-
-@step(name="transient_enrich_item")
-async def enrich_item(item: dict) -> dict:
-    """Enrich item with additional data."""
-    print(f"[Step] Enriching item {item['item_id']}...")
-    await asyncio.sleep(0.3)
-    return {**item, "enriched": True, "score": 0.95}
-
-
-@step(name="transient_store_result")
-async def store_result(item: dict) -> dict:
-    """Store the processed result."""
-    print(f"[Step] Storing result for {item['item_id']}...")
-    await asyncio.sleep(0.2)
-    return {**item, "stored": True}
-
-
-@workflow(durable=False, tags=["celery", "transient"])  # Transient workflow - no event recording
-async def quick_task(item_id: str) -> dict:
-    """
-    Quick processing task (transient).
-
-    This workflow runs without event recording for maximum performance.
-    Ideal for:
-    - High-throughput processing
-    - Stateless transformations
-    - Quick API calls
-    - Tasks that can be safely retried from scratch
-    """
-    print(f"\n[Workflow] Quick task for {item_id}")
-
-    item = await process_item(item_id)
-    item = await enrich_item(item)
-    item = await store_result(item)
-
-    print(f"[Workflow] Completed: {item}\n")
-    return item
-
-
-async def main() -> None:
-    """Run the transient workflow example."""
-    import argparse
-
-    import pyworkflow
-
-    parser = argparse.ArgumentParser(description="Quick Processing Task (Transient)")
-    parser.add_argument("--item-id", default="item-123", help="Item ID to process")
-    args = parser.parse_args()
-
-    print(f"Starting quick task for {args.item_id}...")
-    print("NOTE: This is a transient workflow - no events are recorded")
-    run_id = await pyworkflow.start(quick_task, args.item_id)
-    print(f"Task dispatched with run_id: {run_id}")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

examples/celery/transient/02_fault_tolerance.py

@@ -1,257 +0,0 @@
-"""
-Celery Transient Workflow - Fault Tolerance Example
-
-This example demonstrates fault tolerance options for transient workflows.
-
-Key difference from durable workflows:
-- Transient workflows do NOT record events
-- On worker failure, there's no state to recover from
-- By default, failed transient workflows stay FAILED
-- Optionally, they can be rescheduled to run from scratch
-
-Configuration options:
-1. recover_on_worker_loss=False (DEFAULT for transient)
-   - On worker crash: workflow is marked as FAILED
-   - No automatic retry
-   - Use when: tasks have side effects or can't be safely repeated
-
-2. recover_on_worker_loss=True
-   - On worker crash: workflow is rescheduled from scratch
-   - All steps run again (no event replay - there are no events!)
-   - Use when: tasks are idempotent and can be safely restarted
-
-Prerequisites:
-    1. Start Redis: docker run -d -p 6379:6379 redis:7-alpine
-    2. Start worker: pyworkflow --module examples.celery.transient.02_fault_tolerance worker run
-
-Run with CLI:
-    # Default behavior (no recovery)
-    pyworkflow --module examples.celery.transient.02_fault_tolerance workflows run image_processor \
-        --arg image_id=img-123
-
-    # With recovery enabled
-    pyworkflow --module examples.celery.transient.02_fault_tolerance workflows run batch_processor \
-        --arg batch_id=batch-456
-
-To test fault tolerance:
-    1. Start the workflow
-    2. Kill the worker during execution
-    3. Start a new worker
-    4. Observe the difference between recover_on_worker_loss=True/False
-"""
-
-import asyncio
-
-from pyworkflow import step, workflow
-
-
-@step(name="transient_download_image")
-async def download_image(image_id: str) -> dict:
-    """Download image from storage."""
-    print(f"[Step] Downloading image {image_id}...")
-    await asyncio.sleep(2)
-    return {"image_id": image_id, "size_mb": 5.2, "downloaded": True}
-
-
-@step(name="transient_resize_image")
-async def resize_image(image: dict) -> dict:
-    """Resize image to standard dimensions."""
-    print(f"[Step] Resizing image {image['image_id']}...")
-    print("       (taking 8 seconds - kill worker now to test!)")
-    await asyncio.sleep(8)  # Long operation - good time to kill worker
-    return {**image, "resized": True, "new_size_mb": 1.2}
-
-
-@step(name="transient_apply_filters")
-async def apply_filters(image: dict) -> dict:
-    """Apply visual filters to image."""
-    print(f"[Step] Applying filters to {image['image_id']}...")
-    print("       (taking 6 seconds - kill worker now to test!)")
-    await asyncio.sleep(6)  # Another good time to kill worker
-    return {**image, "filtered": True}
-
-
-@step(name="transient_upload_result")
-async def upload_result(image: dict) -> dict:
-    """Upload processed image."""
-    print(f"[Step] Uploading processed {image['image_id']}...")
-    await asyncio.sleep(2)
-    return {**image, "uploaded": True, "url": f"https://cdn.example.com/{image['image_id']}"}
-
-
-# ============================================================================
-# Workflow 1: No Recovery (Default for Transient)
-# ============================================================================
-
-
-@workflow(
-    durable=False,
-    recover_on_worker_loss=False,  # DEFAULT for transient - no auto-recovery
-    tags=["celery", "transient"],
-)
-async def image_processor(image_id: str) -> dict:
-    """
-    Image processing workflow - NO AUTO-RECOVERY.
-
-    This is the default behavior for transient workflows.
-
-    If a worker crashes during execution:
-    - The workflow is marked as FAILED
-    - No automatic retry occurs
-    - A new workflow must be manually started
-
-    Why use this:
-    - The upload step has side effects (can't safely repeat)
-    - Need manual review of failures
-    - Each image should only be processed once
-    """
-    print(f"\n{'=' * 60}")
-    print(f"Image Processor (NO RECOVERY): {image_id}")
-    print("If worker crashes, workflow will FAIL permanently")
-    print(f"{'=' * 60}\n")
-
-    image = await download_image(image_id)
-    image = await resize_image(image)
-    image = await apply_filters(image)
-    image = await upload_result(image)
-
-    print(f"\n[Complete] Image available at: {image['url']}\n")
-    return image
-
-
-# ============================================================================
-# Workflow 2: With Recovery (Restart from Scratch)
-# ============================================================================
-
-
-@step(name="transient_fetch_batch_items")
-async def fetch_batch_items(batch_id: str) -> dict:
-    """Fetch items in a batch."""
-    print(f"[Step] Fetching batch {batch_id}...")
-    await asyncio.sleep(2)
-    return {"batch_id": batch_id, "items": ["a", "b", "c", "d", "e"], "fetched": True}
-
-
-@step(name="transient_process_batch_items")
-async def process_batch_items(batch: dict) -> dict:
-    """Process all items in batch (idempotent)."""
-    print(f"[Step] Processing {len(batch['items'])} items (kill worker during this step!)...")
-    for i, item in enumerate(batch["items"]):
-        print(f"  Processing item {item} ({i + 1}/{len(batch['items'])})...")
-        await asyncio.sleep(3)  # 3 seconds per item - plenty of time to kill worker
-    return {**batch, "processed": True, "processed_count": len(batch["items"])}
-
-
-@step(name="transient_generate_report")
-async def generate_report(batch: dict) -> dict:
-    """Generate processing report (idempotent)."""
-    print(f"[Step] Generating report for batch {batch['batch_id']}...")
-    await asyncio.sleep(0.5)
-    return {
-        **batch,
-        "report": f"Processed {batch['processed_count']} items successfully",
-        "reported": True,
-    }
-
-
-@workflow(
-    durable=False,
-    recover_on_worker_loss=True,  # Enable recovery - restarts from scratch
-    max_recovery_attempts=3,  # Allow up to 3 restarts
-    tags=["celery", "transient"],
-)
-async def batch_processor(batch_id: str) -> dict:
-    """
-    Batch processing workflow - WITH AUTO-RECOVERY.
-
-    This transient workflow will restart from scratch on worker failure.
-
-    If a worker crashes during execution:
-    - A WORKFLOW_INTERRUPTED event is recorded (even for transient!)
-    - The workflow restarts from the beginning
-    - All steps run again (no event replay for transient)
-    - Up to 3 recovery attempts allowed
-
-    Why use this:
-    - All steps are idempotent (safe to repeat)
-    - Processing can be safely restarted
-    - Better reliability for batch jobs
-    - Items are processed atomically (all or nothing)
-
-    Note: For transient workflows, recovery means RESTART, not RESUME.
-    Unlike durable workflows, there are no events to replay.
-    """
-    print(f"\n{'=' * 60}")
-    print(f"Batch Processor (WITH RECOVERY): {batch_id}")
-    print("If worker crashes, workflow will RESTART from scratch")
-    print(f"{'=' * 60}\n")
-
-    batch = await fetch_batch_items(batch_id)
-    batch = await process_batch_items(batch)
-    batch = await generate_report(batch)
-
-    print(f"\n[Complete] {batch['report']}\n")
-    return batch
-
-
-# ============================================================================
-# Comparison Helper
-# ============================================================================
-
-
-async def main() -> None:
-    """Run the transient fault tolerance examples."""
-    import argparse
-
-    import pyworkflow
-
-    parser = argparse.ArgumentParser(
-        description="Transient Workflow Fault Tolerance Examples",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog="""
-Examples:
-  # Run image processor (no recovery on failure)
-  python 02_fault_tolerance.py --workflow image --id img-123
-
-  # Run batch processor (restarts on failure)
-  python 02_fault_tolerance.py --workflow batch --id batch-456
-
-To test:
-  1. Start the workflow
-  2. Kill the worker (Ctrl+C) during processing
-  3. Start a new worker
-  4. Observe: image_processor stays FAILED, batch_processor restarts
-        """,
-    )
-    parser.add_argument(
-        "--workflow",
-        choices=["image", "batch"],
-        default="batch",
-        help="Which workflow to run",
-    )
-    parser.add_argument("--id", default="test-001", help="ID for the workflow")
-    args = parser.parse_args()
-
-    print("\n" + "=" * 60)
-    print("TRANSIENT WORKFLOW FAULT TOLERANCE DEMO")
-    print("=" * 60)
-
-    if args.workflow == "image":
-        print("\nRunning: image_processor (recover_on_worker_loss=False)")
-        print("Behavior: On worker crash -> FAILED (no recovery)")
-        run_id = await pyworkflow.start(image_processor, args.id)
-    else:
-        print("\nRunning: batch_processor (recover_on_worker_loss=True)")
-        print("Behavior: On worker crash -> RESTART from scratch")
-        run_id = await pyworkflow.start(batch_processor, args.id)
-
-    print(f"\nWorkflow dispatched with run_id: {run_id}")
-    print("\nTo test fault tolerance:")
-    print("  1. Watch the worker output")
-    print("  2. Kill the worker during processing (Ctrl+C)")
-    print("  3. Start a new worker")
-    print("  4. Observe the recovery behavior")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

examples/celery/transient/__init__.py

@@ -1,20 +0,0 @@
-"""
-Celery Transient Workflow Examples
-
-These examples demonstrate transient (non-durable) workflows running on Celery workers.
-
-Transient workflows:
-- Do NOT persist state to storage
-- Do NOT record events
-- Cannot be resumed after suspension
-- Are simpler and faster for short-lived tasks
-
-Key differences from durable workflows:
-| Feature               | Durable           | Transient         |
-|-----------------------|-------------------|-------------------|
-| Event recording       | Yes               | No                |
-| State persistence     | Yes               | No                |
-| Resumable after crash | Yes (from events) | No (starts fresh) |
-| Sleep behavior        | Suspends workflow | Blocks inline     |
-| Best for              | Long-running      | Quick tasks       |
-"""

examples/celery/transient/pyworkflow.config.yaml

@@ -1,25 +0,0 @@
-# PyWorkflow Configuration for Celery Transient Examples
-#
-# This file configures the pyworkflow CLI when running from this directory.
-# Simply run: pyworkflow worker run
-#
-# Priority order:
-#   1. --module CLI argument
-#   2. PYWORKFLOW_DISCOVER environment variable
-#   3. This config file (pyworkflow.config.yaml)
-
-# Module containing workflow definitions
-module: examples.celery.transient
-
-# Runtime configuration
-runtime: celery
-
-# Storage is minimal for transient workflows
-# (only used for tracking run IDs, not for event sourcing)
-storage:
-  type: memory
-
-# Celery broker and result backend
-celery:
-  broker: redis://localhost:6379/0
-  result_backend: redis://localhost:6379/1

examples/local/__init__.py

@@ -1 +0,0 @@
-# PyWorkflow Local Examples Package