synapse-sdk 1.0.0a11__py3-none-any.whl → 2026.1.1b2__py3-none-any.whl

synapse_sdk/plugins/pipelines/display.py

@@ -0,0 +1,311 @@
+"""Rich console progress display for pipeline execution.
+
+Provides a real-time progress display using the Rich library for
+monitoring pipeline execution in the terminal. Supports both sync and async.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator, Iterator
+
+    from rich.panel import Panel
+
+    from synapse_sdk.plugins.models.logger import PipelineProgress
+
+
+def _create_display_panel(
+    progress: 'PipelineProgress',
+    show_actions: bool = True,
+) -> 'Panel':
+    """Create the Rich display panel for progress."""
+    from rich.console import Group
+    from rich.panel import Panel
+    from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn
+    from rich.table import Table
+
+    from synapse_sdk.plugins.models.pipeline import ActionStatus, RunStatus
+
+    # Main progress bar
+    main_progress = Progress(
+        SpinnerColumn(),
+        TextColumn('[bold blue]{task.description}'),
+        BarColumn(),
+        TextColumn('[progress.percentage]{task.percentage:>3.0f}%'),
+    )
+
+    # Calculate overall progress including current action progress
+    total_actions = len(progress.actions) if progress.actions else 1
+    completed_actions = sum(1 for a in (progress.actions or []) if a.status == ActionStatus.COMPLETED)
+
+    # Find current running action and its progress
+    current_action_progress = 0.0
+    if progress.actions:
+        for a in progress.actions:
+            if a.status == ActionStatus.RUNNING and a.progress:
+                current_action_progress = a.progress
+                break
+
+    if progress.status == RunStatus.COMPLETED:
+        percentage = 100.0
+    elif progress.status == RunStatus.FAILED:
+        percentage = (completed_actions / total_actions) * 100
+    else:
+        # Include current action's partial progress
+        base_progress = (completed_actions / total_actions) * 100
+        action_contribution = (current_action_progress / total_actions) * 100
+        percentage = base_progress + action_contribution
+
+    main_progress.add_task(
+        description=f'Pipeline: {progress.run_id[:8]}...',
+        total=100,
+        completed=percentage,
+    )
+
+    # Status text
+    status_colors = {
+        RunStatus.PENDING: 'yellow',
+        RunStatus.RUNNING: 'blue',
+        RunStatus.COMPLETED: 'green',
+        RunStatus.FAILED: 'red',
+        RunStatus.CANCELLED: 'orange3',
+    }
+    status_color = status_colors.get(progress.status, 'white')
+    status_text = f'[{status_color}]Status: {progress.status.value.upper()}[/]'
+
+    if progress.current_action:
+        status_text += f'  |  Current: [cyan]{progress.current_action}[/]'
+
+    # Find current action's detailed progress message
+    current_action_msg = None
+    if progress.actions:
+        for a in progress.actions:
+            if a.status == ActionStatus.RUNNING:
+                if a.message:
+                    current_action_msg = a.message
+                elif a.progress_category and a.progress:
+                    current_action_msg = f'{a.progress_category}: {a.progress * 100:.0f}%'
+                break
+
+    # Build content
+    content_parts: list = [main_progress, '', status_text]
+
+    # Add detailed progress message if available
+    if current_action_msg:
+        content_parts.append(f'[dim]  {current_action_msg}[/]')
+
+    # Action table (if enabled and actions exist)
+    if show_actions and progress.actions:
+        action_table = Table(show_header=True, header_style='bold magenta')
+        action_table.add_column('#', style='dim', width=3)
+        action_table.add_column('Action', min_width=20)
+        action_table.add_column('Status', width=12)
+        action_table.add_column('Progress', width=15)
+
+        action_status_icons = {
+            ActionStatus.PENDING: '[dim]-[/]',
+            ActionStatus.RUNNING: '[blue]...[/]',
+            ActionStatus.COMPLETED: '[green]OK[/]',
+            ActionStatus.FAILED: '[red]FAIL[/]',
+            ActionStatus.SKIPPED: '[yellow]SKIP[/]',
+        }
+
+        for i, action in enumerate(progress.actions):
+            icon = action_status_icons.get(action.status, '?')
+            if action.message:
+                prog_text = action.message
+            elif action.progress:
+                prog_text = f'{action.progress * 100:.0f}%'
+            else:
+                prog_text = '-'
+            action_table.add_row(
+                str(i + 1),
+                action.name,
+                icon,
+                prog_text,
+            )
+
+        content_parts.extend(['', action_table])
+
+    # Error message (if any)
+    if progress.error:
+        content_parts.extend(['', f'[red]Error: {progress.error}[/]'])
+
+    return Panel(
+        Group(*content_parts),
+        title='[bold]Pipeline Progress[/]',
+        border_style='blue' if progress.status == RunStatus.RUNNING else 'green',
+    )
+
+
+def display_progress(
+    progress_stream: 'Iterator[PipelineProgress]',
+    *,
+    show_actions: bool = True,
+    refresh_rate: float = 4.0,
+) -> 'PipelineProgress':
+    """Display pipeline progress in the console using Rich (sync version).
+
+    Creates a live-updating display showing:
+    - Overall pipeline status and progress bar
+    - Current action being executed
+    - Individual action statuses in a table
+
+    Args:
+        progress_stream: Iterator yielding PipelineProgress updates.
+        show_actions: If True, show individual action statuses.
+        refresh_rate: Refreshes per second for the live display.
+
+    Returns:
+        Final PipelineProgress after completion.
+
+    Example:
+        >>> from synapse_sdk.plugins.pipelines.display import display_progress
+        >>> run_id = executor.submit(pipeline, params)
+        >>> final = display_progress(executor.stream_progress(run_id))
+        >>> print(f"Completed with status: {final.status}")
+    """
+    try:
+        from rich.console import Console
+        from rich.live import Live
+    except ImportError as e:
+        raise ImportError('Rich library is required for progress display. Install it with: pip install rich') from e
+
+    console = Console()
+    last_progress: PipelineProgress | None = None
+
+    with Live(console=console, refresh_per_second=refresh_rate) as live:
+        for progress in progress_stream:
+            last_progress = progress
+            live.update(_create_display_panel(progress, show_actions))
+
+    if last_progress is None:
+        raise RuntimeError('No progress updates received')
+
+    return last_progress
+
+
+async def display_progress_async(
+    progress_stream: 'AsyncIterator[PipelineProgress]',
+    *,
+    show_actions: bool = True,
+    refresh_rate: float = 4.0,
+) -> 'PipelineProgress':
+    """Display pipeline progress in the console using Rich (async version).
+
+    Creates a live-updating display showing:
+    - Overall pipeline status and progress bar
+    - Current action being executed
+    - Individual action statuses in a table
+
+    Args:
+        progress_stream: AsyncIterator yielding PipelineProgress updates.
+        show_actions: If True, show individual action statuses.
+        refresh_rate: Refreshes per second for the live display.
+
+    Returns:
+        Final PipelineProgress after completion.
+
+    Example:
+        >>> from synapse_sdk.plugins.pipelines.display import display_progress_async
+        >>> run_id = await executor.submit_async(pipeline, params)
+        >>> async for progress in executor.stream_progress_async(run_id):
+        ...     # Process updates
+        >>> final = await display_progress_async(executor.stream_progress_async(run_id))
+    """
+    try:
+        from rich.console import Console
+        from rich.live import Live
+    except ImportError as e:
+        raise ImportError('Rich library is required for progress display. Install it with: pip install rich') from e
+
+    console = Console()
+    last_progress: PipelineProgress | None = None
+
+    with Live(console=console, refresh_per_second=refresh_rate) as live:
+        async for progress in progress_stream:
+            last_progress = progress
+            live.update(_create_display_panel(progress, show_actions))
+
+    if last_progress is None:
+        raise RuntimeError('No progress updates received')
+
+    return last_progress
+
+
+def print_progress_summary(progress: 'PipelineProgress') -> None:
+    """Print a summary of pipeline execution.
+
+    Args:
+        progress: Final pipeline progress to summarize.
+    """
+    try:
+        from rich.console import Console
+        from rich.table import Table
+    except ImportError:
+        # Fallback to plain print
+        print(f'Pipeline {progress.run_id}: {progress.status.value}')
+        return
+
+    console = Console()
+    from synapse_sdk.plugins.models.pipeline import ActionStatus, RunStatus
+
+    # Status emoji
+    status_emoji = {
+        RunStatus.COMPLETED: '[green]OK[/]',
+        RunStatus.FAILED: '[red]FAILED[/]',
+        RunStatus.CANCELLED: '[yellow]CANCELLED[/]',
+        RunStatus.RUNNING: '[blue]RUNNING[/]',
+        RunStatus.PENDING: '[dim]PENDING[/]',
+    }
+
+    console.print()
+    console.print(f'[bold]Pipeline:[/] {progress.run_id}')
+    console.print(f'[bold]Status:[/] {status_emoji.get(progress.status, progress.status.value)}')
+
+    if progress.started_at:
+        console.print(f'[bold]Started:[/] {progress.started_at.isoformat()}')
+    if progress.completed_at:
+        console.print(f'[bold]Completed:[/] {progress.completed_at.isoformat()}')
+        if progress.started_at:
+            duration = progress.completed_at - progress.started_at
+            console.print(f'[bold]Duration:[/] {duration}')
+
+    if progress.actions:
+        console.print()
+        table = Table(title='Actions', show_header=True)
+        table.add_column('Action', style='cyan')
+        table.add_column('Status')
+        table.add_column('Duration')
+
+        action_status_style = {
+            ActionStatus.COMPLETED: '[green]COMPLETED[/]',
+            ActionStatus.FAILED: '[red]FAILED[/]',
+            ActionStatus.RUNNING: '[blue]RUNNING[/]',
+            ActionStatus.PENDING: '[dim]PENDING[/]',
+            ActionStatus.SKIPPED: '[yellow]SKIPPED[/]',
+        }
+
+        for action in progress.actions:
+            duration = '-'
+            if action.started_at and action.completed_at:
+                duration = str(action.completed_at - action.started_at)
+            elif action.started_at:
+                duration = 'running...'
+
+            table.add_row(
+                action.name,
+                action_status_style.get(action.status, str(action.status)),
+                duration,
+            )
+
+        console.print(table)
+
+    if progress.error:
+        console.print()
+        console.print(f'[red bold]Error:[/] {progress.error}')
+
+
+__all__ = ['display_progress', 'display_progress_async', 'print_progress_summary']

synapse_sdk/plugins/runner.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+import importlib
+from collections.abc import Callable
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Literal
+
+from synapse_sdk.plugins.discovery import PluginDiscovery
+from synapse_sdk.plugins.executors.local import LocalExecutor
+from synapse_sdk.plugins.executors.ray.job import RayJobExecutor
+from synapse_sdk.plugins.executors.ray.task import RayActorExecutor
+
+if TYPE_CHECKING:
+    from synapse_sdk.plugins.action import BaseAction
+
+
+def _discover_action(plugin_code: str, action: str) -> type[BaseAction] | Callable:
+    """Discover action class from plugin code.
+
+    Args:
+        plugin_code: Either a module path ('my_plugins.yolov8') or
+                     a filesystem path to config.yaml ('/path/to/plugin')
+        action: Action name to load
+
+    Returns:
+        Action class or decorated function
+    """
+    path = Path(plugin_code)
+
+    # Check if it's a filesystem path (config.yaml or directory)
+    if path.exists() or (path.parent.exists() and path.suffix == '.yaml'):
+        discovery = PluginDiscovery.from_path(path)
+    else:
+        # Treat as module path - import and introspect
+        module = importlib.import_module(plugin_code)
+        discovery = PluginDiscovery.from_module(module)
+
+    return discovery.get_action_class(action)
+
+
+def run_plugin(
+    plugin_code: str,
+    action: str,
+    params: dict[str, Any] | None = None,
+    *,
+    mode: Literal['local', 'task', 'job'] = 'local',
+    **executor_kwargs: Any,
+) -> Any:
+    """Run a plugin action.
+
+    This is the main entry point for executing plugin actions. It handles
+    plugin discovery, parameter validation, and execution delegation to
+    the appropriate executor based on the mode.
+
+    Args:
+        plugin_code: Plugin identifier. Can be:
+            - Module path: 'my_plugins.yolov8' (discovers via @action decorators or BaseAction classes)
+            - Filesystem path: '/path/to/plugin' or '/path/to/config.yaml'
+        action: Action name to execute (e.g., 'train', 'infer', 'export').
+        params: Action parameters as a dictionary. Will be validated against
+            the action's params schema if defined.
+        mode: Execution mode:
+            - 'local': Run in the current process (default, good for dev).
+            - 'task': Run via Ray Actor pool (fast startup, <1s).
+            - 'job': Run via Ray Job API (for heavy/long-running workloads).
+        **executor_kwargs: Additional executor options:
+            - action_cls: Optional explicit action class (skips discovery).
+            - env: PluginEnvironment or dict for environment config.
+            - job_id: Optional job identifier for tracking.
+
+    Returns:
+        For 'local' and 'task' modes: Action result (type depends on the action).
+        For 'job' mode: Job ID string for tracking (async submission).
+
+    Raises:
+        ActionNotFoundError: If the action doesn't exist in the plugin.
+        ValidationError: If params fail schema validation.
+        ExecutionError: If action execution fails.
+
+    Example:
+        >>> from synapse_sdk.plugins.runner import run_plugin
+        >>>
+        >>> # Auto-discover from module path
+        >>> result = run_plugin('my_plugins.yolov8', 'train', {'epochs': 10})
+        >>>
+        >>> # Auto-discover from config.yaml path
+        >>> result = run_plugin('/path/to/plugin', 'train', {'epochs': 10})
+        >>>
+        >>> # Explicit action class (skips discovery)
+        >>> result = run_plugin('yolov8', 'train', {'epochs': 10}, action_cls=TrainAction)
+    """
+    params = params or {}
+
+    if mode == 'local':
+        action_cls = executor_kwargs.pop('action_cls', None)
+        if action_cls is None:
+            action_cls = _discover_action(plugin_code, action)
+        executor = LocalExecutor(**executor_kwargs)
+        return executor.execute(action_cls, params)
+
+    elif mode == 'task':
+        action_cls = executor_kwargs.pop('action_cls', None)
+        if action_cls is None:
+            action_cls = _discover_action(plugin_code, action)
+        executor = RayActorExecutor(**executor_kwargs)
+        return executor.execute(action_cls, params)
+
+    else:  # mode == 'job'
+        executor = RayJobExecutor(**executor_kwargs)
+        # Job mode is async - submit and return job_id
+        return executor.submit(action, params)
+
+
+__all__ = ['run_plugin']

synapse_sdk/plugins/schemas/__init__.py

@@ -0,0 +1,19 @@
+"""Common schema types for plugin actions."""
+
+from synapse_sdk.plugins.schemas.results import (
+    ExportResult,
+    InferenceResult,
+    MetricsResult,
+    TrainResult,
+    UploadResult,
+    WeightsResult,
+)
+
+__all__ = [
+    'ExportResult',
+    'InferenceResult',
+    'MetricsResult',
+    'TrainResult',
+    'UploadResult',
+    'WeightsResult',
+]

synapse_sdk/plugins/schemas/results.py

@@ -0,0 +1,152 @@
+"""Common result types for plugin actions.
+
+These provide standardized schemas for common action outputs.
+Plugin developers can use these directly or extend them for
+custom result types.
+
+Example usage:
+    >>> from synapse_sdk.plugins import BaseTrainAction
+    >>> from synapse_sdk.plugins.schemas import TrainResult
+    >>>
+    >>> class MyTrainAction(BaseTrainAction[MyParams, TrainResult]):
+    ...     def execute(self) -> TrainResult:
+    ...         return TrainResult(
+    ...             weights_path='/models/best.pt',
+    ...             final_epoch=100,
+    ...             train_metrics={'loss': 0.05},
+    ...             val_metrics={'mAP50': 0.85},
+    ...         )
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class WeightsResult(BaseModel):
+    """Result type for actions that produce model weights.
+
+    Use this for actions that output trained model weights.
+
+    Example:
+        >>> class TrainAction(BaseAction[TrainParams, WeightsResult]):
+        ...     def execute(self) -> WeightsResult:
+        ...         return WeightsResult(
+        ...             weights_path='/models/best.pt',
+        ...             checkpoint_paths=['/models/epoch_10.pt'],
+        ...         )
+    """
+
+    weights_path: str = Field(description='Path to the best/final model weights')
+    checkpoint_paths: list[str] = Field(default_factory=list, description='Paths to intermediate checkpoints')
+    format: str = Field(default='pt', description='Weights format (pt, onnx, safetensors, etc.)')
+
+
+class MetricsResult(BaseModel):
+    """Result type for actions that produce metrics only.
+
+    Use this for evaluation/testing actions that output metrics.
+
+    Example:
+        >>> class EvalAction(BaseAction[EvalParams, MetricsResult]):
+        ...     def execute(self) -> MetricsResult:
+        ...         return MetricsResult(
+        ...             metrics={'mAP50': 0.85, 'mAP50-95': 0.72},
+        ...             category='validation',
+        ...         )
+    """
+
+    metrics: dict[str, float] = Field(description='Metric name to value mapping')
+    category: str = Field(default='default', description='Metrics category (train, validation, test)')
+
+
+class TrainResult(BaseModel):
+    """Combined result type for training actions.
+
+    Includes both weights and training metrics. Use this for
+    standard training workflows.
+
+    Example:
+        >>> class TrainAction(BaseTrainAction[TrainParams, TrainResult]):
+        ...     def execute(self) -> TrainResult:
+        ...         return TrainResult(
+        ...             weights_path='/models/best.pt',
+        ...             final_epoch=100,
+        ...             best_epoch=85,
+        ...             train_metrics={'loss': 0.05},
+        ...             val_metrics={'mAP50': 0.85, 'mAP50-95': 0.72},
+        ...         )
+    """
+
+    weights_path: str = Field(description='Path to trained model weights')
+    final_epoch: int = Field(description='Last completed epoch')
+    best_epoch: int | None = Field(default=None, description='Best epoch by validation metric')
+    train_metrics: dict[str, float] = Field(default_factory=dict, description='Final training metrics')
+    val_metrics: dict[str, float] = Field(default_factory=dict, description='Final validation metrics')
+
+
+class InferenceResult(BaseModel):
+    """Result type for inference actions.
+
+    Generic container for inference outputs.
+
+    Example:
+        >>> class InferAction(BaseInferenceAction[InferParams, InferenceResult]):
+        ...     def execute(self) -> InferenceResult:
+        ...         return InferenceResult(
+        ...             predictions=[{'class': 'dog', 'confidence': 0.95}],
+        ...             processed_count=100,
+        ...         )
+    """
+
+    predictions: list[dict[str, Any]] = Field(default_factory=list, description='List of prediction results')
+    processed_count: int = Field(default=0, description='Number of items processed')
+    output_path: str | None = Field(default=None, description='Path to output file if results were saved')
+
+
+class ExportResult(BaseModel):
+    """Result type for export actions.
+
+    Example:
+        >>> class ExportAction(BaseExportAction[ExportParams, ExportResult]):
+        ...     def execute(self) -> ExportResult:
+        ...         return ExportResult(
+        ...             output_path='/exports/dataset.zip',
+        ...             exported_count=1000,
+        ...             format='coco',
+        ...         )
+    """
+
+    output_path: str = Field(description='Path to exported file/directory')
+    exported_count: int = Field(description='Number of items exported')
+    format: str = Field(description='Export format (coco, yolo, voc, csv, etc.)')
+    file_size_bytes: int | None = Field(default=None, description='Size of exported file in bytes')
+
+
+class UploadResult(BaseModel):
+    """Result type for upload actions.
+
+    Example:
+        >>> class UploadAction(BaseUploadAction[UploadParams, UploadResult]):
+        ...     def execute(self) -> UploadResult:
+        ...         return UploadResult(
+        ...             uploaded_count=500,
+        ...             remote_path='s3://bucket/dataset/',
+        ...         )
+    """
+
+    uploaded_count: int = Field(description='Number of items uploaded')
+    remote_path: str | None = Field(default=None, description='Remote path or URL where data was uploaded')
+    status: str = Field(default='completed', description='Upload status')
+
+
+__all__ = [
+    'ExportResult',
+    'InferenceResult',
+    'MetricsResult',
+    'TrainResult',
+    'UploadResult',
+    'WeightsResult',
+]

synapse_sdk/plugins/steps/__init__.py

@@ -0,0 +1,63 @@
+"""Step workflow module for Synapse SDK.
+
+This module provides the core abstractions for building step-based workflows.
+Classes here are not pipeline-specific and can be used standalone for any
+action-based workflow.
+
+Classes:
+    BaseStep: Abstract base class for implementing individual steps.
+    StepResult: Data class representing the result of a step execution.
+    BaseStepContext: Context shared between steps during execution.
+    StepRegistry: Registry for managing step registration and ordering.
+    Orchestrator: Executes steps in order and handles rollback on failure.
+
+Utility Steps:
+    LoggingStep: Wrapper that adds logging to step execution.
+    TimingStep: Wrapper that adds timing measurement to step execution.
+    ValidationStep: Wrapper that adds pre-execution validation to steps.
+
+Example:
+    >>> from synapse_sdk.plugins.steps import (
+    ...     BaseStep,
+    ...     BaseStepContext,
+    ...     Orchestrator,
+    ...     StepRegistry,
+    ...     StepResult,
+    ... )
+    >>>
+    >>> @dataclass
+    ... class MyContext(BaseStepContext):
+    ...     data: list[str] = field(default_factory=list)
+    >>>
+    >>> class LoadStep(BaseStep[MyContext]):
+    ...     @property
+    ...     def name(self) -> str:
+    ...         return 'load'
+    ...
+    ...     @property
+    ...     def progress_weight(self) -> float:
+    ...         return 0.3
+    ...
+    ...     def execute(self, context: MyContext) -> StepResult:
+    ...         context.data.append('loaded')
+    ...         return StepResult(success=True)
+"""
+
+from synapse_sdk.plugins.steps.base import BaseStep, StepResult
+from synapse_sdk.plugins.steps.context import BaseStepContext
+from synapse_sdk.plugins.steps.orchestrator import Orchestrator
+from synapse_sdk.plugins.steps.registry import StepRegistry
+from synapse_sdk.plugins.steps.utils import LoggingStep, TimingStep, ValidationStep
+
+__all__ = [
+    # Core
+    'BaseStep',
+    'BaseStepContext',
+    'Orchestrator',
+    'StepRegistry',
+    'StepResult',
+    # Utilities
+    'LoggingStep',
+    'TimingStep',
+    'ValidationStep',
+]