synapse-sdk 1.0.0a11__py3-none-any.whl → 2026.1.1b2__py3-none-any.whl

synapse_sdk/plugins/executors/ray/task.py

@@ -0,0 +1,257 @@
+"""Ray Actor executor for plugin actions."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Literal
+
+from synapse_sdk.plugins.context import PluginEnvironment
+from synapse_sdk.plugins.enums import PackageManager
+from synapse_sdk.plugins.errors import ExecutionError
+from synapse_sdk.plugins.executors.ray.base import BaseRayExecutor, read_requirements
+
+if TYPE_CHECKING:
+    from synapse_sdk.plugins.action import BaseAction
+
+
+class RayActorExecutor(BaseRayExecutor):
+    """Ray Actor based synchronous task execution.
+
+    Executes actions using a persistent Ray Actor. Best for fast startup
+    with pre-warmed workers. The actor maintains state across executions
+    and methods are executed serially within each actor.
+
+    Example:
+        >>> executor = RayActorExecutor(
+        ...     ray_address='auto',
+        ...     working_dir='/path/to/plugin',  # Auto-reads requirements.txt
+        ... )
+        >>> result = executor.execute(TrainAction, {'epochs': 10})
+        >>> # Reuse the same actor for subsequent executions
+        >>> result2 = executor.execute(InferAction, {'batch_size': 32})
+    """
+
+    def __init__(
+        self,
+        env: PluginEnvironment | dict[str, Any] | None = None,
+        job_id: str | None = None,
+        *,
+        ray_address: str = 'auto',
+        runtime_env: dict[str, Any] | None = None,
+        working_dir: str | Path | None = None,
+        requirements_file: str | Path | None = None,
+        package_manager: PackageManager | Literal['pip', 'uv'] = PackageManager.PIP,
+        package_manager_options: list[str] | None = None,
+        wheels_dir: str = 'wheels',
+        num_cpus: int | None = None,
+        num_gpus: int | None = None,
+        include_sdk: bool = False,
+    ) -> None:
+        """Initialize Ray actor executor.
+
+        Args:
+            env: Environment config for the action. If None, loads from os.environ.
+            job_id: Optional job identifier for tracking.
+            ray_address: Ray cluster address. Defaults to 'auto'.
+            runtime_env: Ray runtime environment config (pip packages, working_dir, env_vars).
+            working_dir: Plugin working directory. Sets runtime_env['working_dir'] and
+                auto-reads requirements.txt if requirements_file is not specified.
+            requirements_file: Path to requirements.txt. If None and working_dir is set,
+                looks for requirements.txt in working_dir.
+            package_manager: Package manager to use ('pip' or 'uv'). Defaults to 'pip'.
+            package_manager_options: Additional options for the package manager.
+                For uv: defaults to ['--no-cache']. For pip: defaults to ['--upgrade'].
+            wheels_dir: Directory containing .whl files relative to working_dir (default: 'wheels').
+            num_cpus: Number of CPUs to request for the actor.
+            num_gpus: Number of GPUs to request for the actor.
+            include_sdk: If True, bundle local SDK with upload (for development).
+        """
+        super().__init__(
+            env=env,
+            runtime_env=runtime_env,
+            working_dir=working_dir,
+            requirements_file=requirements_file,
+            package_manager=package_manager,
+            package_manager_options=package_manager_options,
+            wheels_dir=wheels_dir,
+            ray_address=ray_address,
+            include_sdk=include_sdk,
+        )
+        self._job_id = job_id
+        self._num_cpus = num_cpus
+        self._num_gpus = num_gpus
+        self._actor: Any | None = None
+
+    def _get_or_create_actor(self) -> Any:
+        """Get existing actor or create a new one."""
+        import ray
+
+        if self._actor is not None:
+            return self._actor
+
+        self._ray_init()
+
+        # Build remote options
+        remote_options: dict[str, Any] = {'runtime_env': self._build_runtime_env()}
+        if self._num_cpus is not None:
+            remote_options['num_cpus'] = self._num_cpus
+        if self._num_gpus is not None:
+            remote_options['num_gpus'] = self._num_gpus
+
+        # Create the actor class dynamically with ray.remote
+        ActionExecutorActor = ray.remote(**remote_options)(_ActionExecutorActor)
+        self._actor = ActionExecutorActor.remote(self._job_id)
+        return self._actor
+
+    def execute(
+        self,
+        action_cls: type[BaseAction] | str,
+        params: dict[str, Any],
+        **kwargs: Any,
+    ) -> Any:
+        """Execute action using the Ray actor.
+
+        Args:
+            action_cls: BaseAction subclass or entrypoint string (e.g., 'plugin.test.TestAction').
+            params: Parameters dict to validate and pass.
+            **kwargs: Ignored (for protocol compatibility).
+
+        Returns:
+            Action result from execute().
+
+        Raises:
+            ValidationError: If params fail validation.
+            ExecutionError: If action execution fails.
+        """
+        import ray
+        from ray.exceptions import RayActorError, RayTaskError
+
+        # Convert class to entrypoint string for remote import
+        if isinstance(action_cls, str):
+            entrypoint = action_cls
+        else:
+            entrypoint = f'{action_cls.__module__}.{action_cls.__name__}'
+
+        actor = self._get_or_create_actor()
+
+        try:
+            return ray.get(actor.run_action.remote(entrypoint, params))
+        except (RayTaskError, RayActorError) as e:
+            # Actor may have died, reset it for next call
+            self._actor = None
+            cause = getattr(e, 'cause', e)
+            raise ExecutionError(f'Ray actor execution failed: {cause}') from e
+
+    def shutdown(self) -> None:
+        """Shutdown the actor."""
+        import ray
+
+        if self._actor is not None:
+            ray.kill(self._actor)
+            self._actor = None
+
+
+class _ActionExecutorActor:
+    """Ray Actor that executes plugin actions.
+
+    This actor maintains state across executions and provides
+    serial execution guarantees for methods called on it.
+    """
+
+    def __init__(self, job_id: str | None = None) -> None:
+        """Initialize the actor.
+
+        Args:
+            job_id: Optional job identifier for tracking.
+        """
+        self._job_id = job_id
+
+    def run_action(
+        self,
+        entrypoint: str,
+        params: dict[str, Any],
+    ) -> Any:
+        """Execute an action within this actor.
+
+        Args:
+            entrypoint: Action entrypoint string (e.g., 'plugin.test.TestAction').
+            params: Parameters dict to validate and pass.
+
+        Returns:
+            Action result from execute().
+
+        Raises:
+            ValidationError: If params fail validation.
+            ExecutionError: If action execution fails.
+        """
+        import importlib
+        import logging
+        import os
+        import sys
+
+        # Configure logging to ensure ConsoleLogger output is visible
+        # This is needed because Ray workers don't have logging configured by default
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            force=True,  # Override any existing config
+        )
+
+        from synapse_sdk.loggers import ConsoleLogger
+        from synapse_sdk.plugins.action import NoResult, validate_result
+        from synapse_sdk.plugins.context import RuntimeContext
+        from synapse_sdk.plugins.errors import ExecutionError, ValidationError
+        from synapse_sdk.utils.auth import create_backend_client
+
+        # Dynamically import action class from entrypoint
+        try:
+            # Ensure working directory is in sys.path (Ray should do this, but be explicit)
+            cwd = os.getcwd()
+            if cwd not in sys.path:
+                sys.path.insert(0, cwd)
+
+            module_path, class_name = entrypoint.rsplit('.', 1)
+            module = importlib.import_module(module_path)
+            action_cls = getattr(module, class_name)
+        except Exception as e:
+            raise ExecutionError(f'Failed to import action {entrypoint}: {e}') from e
+
+        # Validate params
+        try:
+            validated_params = action_cls.params_model.model_validate(params)
+        except Exception as e:
+            raise ValidationError(f'Parameter validation failed: {e}') from e
+
+        # Try to create BackendClient from environment/credentials
+        client = create_backend_client()
+
+        # Build context (inside actor)
+        logger = ConsoleLogger()
+        ctx = RuntimeContext(
+            logger=logger,
+            env=dict(os.environ),
+            job_id=self._job_id,
+            client=client,
+        )
+
+        # Execute
+        action = action_cls(validated_params, ctx)
+        try:
+            result = action.execute()
+        except Exception as e:
+            logger.finish()
+            raise ExecutionError(f'Action execution failed: {e}') from e
+
+        # Validate result with warning-only mode
+        result_model = getattr(action_cls, 'result_model', NoResult)
+        if result_model is not NoResult:
+            result = validate_result(result, result_model, logger)
+
+        logger.finish()
+        return result
+
+
+# Keep old name as alias for backwards compatibility
+RayTaskExecutor = RayActorExecutor
+
+__all__ = ['RayActorExecutor', 'RayTaskExecutor', 'read_requirements']

synapse_sdk/plugins/models/__init__.py

@@ -0,0 +1,26 @@
+"""Plugin models module."""
+
+from synapse_sdk.plugins.models.logger import (
+    ActionProgress,
+    Checkpoint,
+    LogEntry,
+    LoggerBackend,
+    LogLevel,
+    PipelineProgress,
+    ProgressData,
+)
+from synapse_sdk.plugins.models.pipeline import ActionStatus, RunStatus
+
+__all__ = [
+    # pipeline.py
+    'RunStatus',
+    'ActionStatus',
+    # logger.py
+    'LogLevel',
+    'ProgressData',
+    'LogEntry',
+    'LoggerBackend',
+    'ActionProgress',
+    'PipelineProgress',
+    'Checkpoint',
+]

synapse_sdk/plugins/models/logger.py

@@ -0,0 +1,173 @@
+"""Logger models and enums.
+
+This module re-exports core logger types from synapse_sdk.loggers
+and defines additional logger-related models.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import TYPE_CHECKING, Any
+
+# Re-export core logger types from loggers.py to avoid circular imports
+from synapse_sdk.loggers import LogEntry, LoggerBackend, ProgressData
+
+if TYPE_CHECKING:
+    from synapse_sdk.plugins.models.pipeline import ActionStatus
+
+__all__ = [
+    'LogLevel',
+    'ProgressData',
+    'LogEntry',
+    'LoggerBackend',
+    'ActionProgress',
+    'PipelineProgress',
+    'Checkpoint',
+]
+
+
+class LogLevel(str, Enum):
+    """Log level for logger."""
+
+    DEBUG = 'debug'
+    INFO = 'info'
+    WARNING = 'warning'
+    ERROR = 'error'
+    CRITICAL = 'critical'
+
+
+@dataclass
+class ActionProgress:
+    """Progress state for a single action."""
+
+    name: str
+    status: ActionStatus = None  # type: ignore[assignment]
+    progress: float = 0.0
+    progress_category: str | None = None
+    message: str | None = None
+    metrics: dict[str, Any] = field(default_factory=dict)
+    started_at: datetime | None = None
+    completed_at: datetime | None = None
+    error: str | None = None
+
+    def __post_init__(self) -> None:
+        # Lazy import to avoid circular imports
+        if self.status is None:
+            from synapse_sdk.plugins.models.pipeline import ActionStatus
+
+            self.status = ActionStatus.PENDING
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for API serialization."""
+        return {
+            'name': self.name,
+            'status': self.status.value,
+            'progress': self.progress,
+            'progress_category': self.progress_category,
+            'message': self.message,
+            'metrics': self.metrics,
+            'started_at': self.started_at.isoformat() if self.started_at else None,
+            'completed_at': self.completed_at.isoformat() if self.completed_at else None,
+            'error': self.error,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> ActionProgress:
+        """Create from dictionary."""
+        from synapse_sdk.plugins.models.pipeline import ActionStatus
+
+        return cls(
+            name=data['name'],
+            status=ActionStatus(data['status']),
+            progress=data.get('progress', 0.0),
+            progress_category=data.get('progress_category'),
+            message=data.get('message'),
+            metrics=data.get('metrics', {}),
+            started_at=datetime.fromisoformat(data['started_at']) if data.get('started_at') else None,
+            completed_at=datetime.fromisoformat(data['completed_at']) if data.get('completed_at') else None,
+            error=data.get('error'),
+        )
+
+
+@dataclass
+class PipelineProgress:
+    """Overall pipeline progress state."""
+
+    run_id: str
+    status: Any = None  # RunStatus, but lazy loaded
+    actions: dict[str, ActionProgress] = field(default_factory=dict)
+    current_action: str | None = None
+    started_at: datetime | None = None
+    completed_at: datetime | None = None
+    error: str | None = None
+
+    def __post_init__(self) -> None:
+        if self.status is None:
+            from synapse_sdk.plugins.models.pipeline import RunStatus
+
+            self.status = RunStatus.PENDING
+
+    @property
+    def overall_progress(self) -> float:
+        """Calculate overall progress across all actions."""
+        if not self.actions:
+            return 0.0
+        return sum(a.progress for a in self.actions.values()) / len(self.actions)
+
+    @property
+    def completed_actions(self) -> int:
+        """Count completed actions."""
+        from synapse_sdk.plugins.models.pipeline import ActionStatus
+
+        return sum(1 for a in self.actions.values() if a.status == ActionStatus.COMPLETED)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for API serialization."""
+        return {
+            'run_id': self.run_id,
+            'status': self.status.value,
+            'actions': {k: v.to_dict() for k, v in self.actions.items()},
+            'current_action': self.current_action,
+            'overall_progress': self.overall_progress,
+            'completed_actions': self.completed_actions,
+            'total_actions': len(self.actions),
+            'started_at': self.started_at.isoformat() if self.started_at else None,
+            'completed_at': self.completed_at.isoformat() if self.completed_at else None,
+            'error': self.error,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> PipelineProgress:
+        """Create from dictionary."""
+        from synapse_sdk.plugins.models.pipeline import RunStatus
+
+        return cls(
+            run_id=data['run_id'],
+            status=RunStatus(data['status']),
+            actions={k: ActionProgress.from_dict(v) for k, v in data.get('actions', {}).items()},
+            current_action=data.get('current_action'),
+            started_at=datetime.fromisoformat(data['started_at']) if data.get('started_at') else None,
+            completed_at=datetime.fromisoformat(data['completed_at']) if data.get('completed_at') else None,
+            error=data.get('error'),
+        )
+
+
+@dataclass
+class Checkpoint:
+    """Checkpoint for pipeline state persistence."""
+
+    run_id: str
+    action_name: str
+    result: Any
+    created_at: datetime = field(default_factory=datetime.now)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for serialization."""
+        return {
+            'run_id': self.run_id,
+            'action_name': self.action_name,
+            'result': self.result,
+            'created_at': self.created_at.isoformat(),
+        }

synapse_sdk/plugins/models/pipeline.py

@@ -0,0 +1,25 @@
+"""Pipeline status enums."""
+
+from enum import Enum
+
+__all__ = ['RunStatus', 'ActionStatus']
+
+
+class RunStatus(str, Enum):
+    """Status of a pipeline run."""
+
+    PENDING = 'pending'
+    RUNNING = 'running'
+    COMPLETED = 'completed'
+    FAILED = 'failed'
+    CANCELLED = 'cancelled'
+
+
+class ActionStatus(str, Enum):
+    """Status of an individual action."""
+
+    PENDING = 'pending'
+    RUNNING = 'running'
+    COMPLETED = 'completed'
+    FAILED = 'failed'
+    SKIPPED = 'skipped'

synapse_sdk/plugins/pipelines/__init__.py

@@ -0,0 +1,81 @@
+"""Pipeline patterns for Synapse SDK.
+
+This module provides various pipeline execution patterns:
+- ActionPipeline: Chain actions with input/output schema validation
+- PipelineContext: Shared working directory for pipeline actions
+- Progress models: Track pipeline and action progress
+- steps: Sequential step-based workflows with rollback support (for internal use)
+
+Example:
+    >>> from synapse_sdk.plugins.pipelines import ActionPipeline, PipelineContext
+    >>>
+    >>> # Unix pipe style: Download | Convert | Train
+    >>> pipeline = ActionPipeline([
+    ...     DownloadDatasetAction,
+    ...     ConvertDatasetAction,
+    ...     TrainAction,
+    ... ])
+    >>>
+    >>> result = pipeline.execute(params, ctx)
+"""
+
+from synapse_sdk.plugins.models.logger import (
+    ActionProgress,
+    Checkpoint,
+    LogEntry,
+    LogLevel,
+    PipelineProgress,
+)
+from synapse_sdk.plugins.models.pipeline import ActionStatus, RunStatus
+from synapse_sdk.plugins.pipelines.action_pipeline import (
+    ActionPipeline,
+    SchemaIncompatibleError,
+)
+from synapse_sdk.plugins.pipelines.context import PipelineContext
+from synapse_sdk.plugins.pipelines.display import (
+    display_progress,
+    display_progress_async,
+    print_progress_summary,
+)
+
+# Import from new canonical location (not deprecated pipelines.steps)
+from synapse_sdk.plugins.steps import (
+    BaseStep,
+    BaseStepContext,
+    LoggingStep,
+    Orchestrator,
+    StepRegistry,
+    StepResult,
+    TimingStep,
+    ValidationStep,
+)
+
+__all__ = [
+    # Action Pipeline (recommended)
+    'ActionPipeline',
+    'SchemaIncompatibleError',
+    # Pipeline Context
+    'PipelineContext',
+    # Display Utilities
+    'display_progress',
+    'display_progress_async',
+    'print_progress_summary',
+    # Progress Models
+    'ActionProgress',
+    'ActionStatus',
+    'Checkpoint',
+    'LogEntry',
+    'LogLevel',
+    'PipelineProgress',
+    'RunStatus',
+    # Steps - Core (internal)
+    'BaseStep',
+    'BaseStepContext',
+    'Orchestrator',
+    'StepRegistry',
+    'StepResult',
+    # Steps - Utilities
+    'LoggingStep',
+    'TimingStep',
+    'ValidationStep',
+]