synapse-sdk 1.0.0a11__py3-none-any.whl → 2026.1.1b2__py3-none-any.whl

synapse_sdk/plugins/enums.py

@@ -1,21 +1,81 @@
-from enum import Enum
+from __future__ import annotations
 
+from enum import StrEnum
 
-class RunMethod(Enum):
-    JOB = 'job'
-    TASK = 'task'
-    RESTAPI = 'restapi'
 
+class PluginCategory(StrEnum):
+    """Categories for organizing plugins by functionality."""
 
-class PluginCategory(Enum):
     NEURAL_NET = 'neural_net'
     EXPORT = 'export'
-    IMPORT = 'import'
+    UPLOAD = 'upload'
     SMART_TOOL = 'smart_tool'
     POST_ANNOTATION = 'post_annotation'
     PRE_ANNOTATION = 'pre_annotation'
     DATA_VALIDATION = 'data_validation'
+    CUSTOM = 'custom'
+
+
+class RunMethod(StrEnum):
+    """Execution methods for plugin actions."""
+
+    JOB = 'job'
+    TASK = 'task'
+    SERVE = 'serve'
+
+
+class PackageManager(StrEnum):
+    """Package managers for plugin dependencies."""
+
+    PIP = 'pip'
+    UV = 'uv'
+
+
+class DataType(StrEnum):
+    """Data types handled by plugins."""
+
+    IMAGE = 'image'
+    TEXT = 'text'
+    VIDEO = 'video'
+    PCD = 'pcd'
+    AUDIO = 'audio'
+
+
+class AnnotationCategory(StrEnum):
+    """Annotation categories for smart tools."""
+
+    OBJECT_DETECTION = 'object_detection'
+    CLASSIFICATION = 'classification'
+    SEGMENTATION = 'segmentation'
+    KEYPOINT = 'keypoint'
+    TEXT = 'text'
+
+
+class AnnotationType(StrEnum):
+    """Annotation types for smart tools."""
+
+    BBOX = 'bbox'
+    POLYGON = 'polygon'
+    POINT = 'point'
+    LINE = 'line'
+    MASK = 'mask'
+    LABEL = 'label'
+
+
+class SmartToolType(StrEnum):
+    """Smart tool implementation types."""
+
+    INTERACTIVE = 'interactive'
+    AUTOMATIC = 'automatic'
+    SEMI_AUTOMATIC = 'semi_automatic'
+
 
-    @classmethod
-    def choices(cls):
-        return [(member.value, member.name.replace('_', ' ').title()) for member in cls]
+__all__ = [
+    'PluginCategory',
+    'RunMethod',
+    'PackageManager',
+    'DataType',
+    'AnnotationCategory',
+    'AnnotationType',
+    'SmartToolType',
+]

synapse_sdk/plugins/errors.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from typing import Any
+
+
+class PluginError(Exception):
+    """Base exception for plugin-related errors."""
+
+    def __init__(self, message: str, details: Any = None):
+        self.message = message
+        self.details = details
+        super().__init__(message)
+
+    def __repr__(self) -> str:
+        return f'{self.__class__.__name__}(message={self.message!r}, details={self.details!r})'
+
+
+class ValidationError(PluginError):
+    """Raised when plugin parameters fail validation."""
+
+    pass
+
+
+class ActionNotFoundError(PluginError):
+    """Raised when the requested action doesn't exist in the plugin."""
+
+    pass
+
+
+class ExecutionError(PluginError):
+    """Raised when action execution fails."""
+
+    pass
+
+
+class PluginUploadError(PluginError):
+    """Raised when plugin upload fails.
+
+    Covers storage upload failures, network errors during upload,
+    and other upload-related issues.
+    """
+
+    pass
+
+
+class ArchiveError(PluginError):
+    """Raised when archive creation fails.
+
+    Covers ZIP creation failures, git ls-files failures,
+    and file permission errors during archiving.
+    """
+
+    pass
+
+
+class BuildError(PluginError):
+    """Raised when wheel build fails.
+
+    Covers wheel build failures, missing pyproject.toml,
+    and package manager not found errors.
+    """
+
+    pass
+
+
+class ChecksumMismatchError(PluginError):
+    """Raised when checksum verification fails.
+
+    Indicates file integrity issues - the actual checksum
+    does not match the expected value.
+    """
+
+    pass
+
+
+class PluginRunError(PluginError):
+    """Raised when plugin run fails."""
+
+    pass
+
+
+__all__ = [
+    'ActionNotFoundError',
+    'ArchiveError',
+    'BuildError',
+    'ChecksumMismatchError',
+    'ExecutionError',
+    'PluginError',
+    'PluginRunError',
+    'PluginUploadError',
+    'ValidationError',
+]

synapse_sdk/plugins/executors/__init__.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
+
+if TYPE_CHECKING:
+    from synapse_sdk.plugins.action import BaseAction
+
+
+@runtime_checkable
+class ExecutorProtocol(Protocol):
+    """Protocol for plugin action executors.
+
+    Executors handle the lifecycle of action execution:
+    1. Build RuntimeContext (logger, env, job_id)
+    2. Instantiate the action with validated params
+    3. Call execute() and handle errors
+    4. Clean up and return result
+    """
+
+    def execute(
+        self,
+        action_cls: type[BaseAction],
+        params: dict[str, Any],
+        **kwargs: Any,
+    ) -> Any:
+        """Execute an action class with parameters.
+
+        Args:
+            action_cls: The BaseAction subclass to execute.
+            params: Raw parameters dict (will be validated).
+            **kwargs: Executor-specific options.
+
+        Returns:
+            Action result.
+
+        Raises:
+            ValidationError: If params fail validation.
+            ExecutionError: If action execution fails.
+        """
+        ...
+
+
+__all__ = ['ExecutorProtocol']

synapse_sdk/plugins/executors/local.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from synapse_sdk.loggers import ConsoleLogger
+from synapse_sdk.plugins.action import NoResult, validate_result
+from synapse_sdk.plugins.context import PluginEnvironment, RuntimeContext
+from synapse_sdk.plugins.errors import ExecutionError, ValidationError
+from synapse_sdk.utils.auth import create_backend_client
+
+if TYPE_CHECKING:
+    from synapse_sdk.plugins.action import BaseAction
+
+
+class LocalExecutor:
+    """Execute actions in the current process.
+
+    Best for development and testing. Uses ConsoleLogger by default.
+
+    Example:
+        >>> executor = LocalExecutor()
+        >>> result = executor.execute(TrainAction, {'epochs': 10})
+    """
+
+    def __init__(
+        self,
+        env: PluginEnvironment | dict[str, Any] | None = None,
+        job_id: str | None = None,
+    ) -> None:
+        """Initialize executor.
+
+        Args:
+            env: Environment config. If None, loads from os.environ.
+            job_id: Optional job identifier.
+        """
+        if env is None:
+            self._env = PluginEnvironment.from_environ()
+        elif isinstance(env, dict):
+            self._env = PluginEnvironment(env)
+        else:
+            self._env = env
+        self._job_id = job_id
+        self._client = create_backend_client()
+
+    def execute(
+        self,
+        action_cls: type[BaseAction],
+        params: dict[str, Any],
+        **kwargs: Any,
+    ) -> Any:
+        """Execute action synchronously in current process.
+
+        Args:
+            action_cls: BaseAction subclass to execute.
+            params: Parameters dict to validate and pass.
+            **kwargs: Ignored (for protocol compatibility).
+
+        Returns:
+            Action result from execute().
+
+        Raises:
+            ValidationError: If params fail validation.
+            ExecutionError: If action raises an exception.
+        """
+        del kwargs  # Unused, for protocol compatibility
+
+        # Validate params
+        try:
+            validated_params = action_cls.params_model.model_validate(params)
+        except Exception as e:
+            raise ValidationError(f'Parameter validation failed: {e}') from e
+
+        # Build context
+        logger = ConsoleLogger()
+        ctx = RuntimeContext(
+            logger=logger,
+            env=self._env.to_dict(),
+            job_id=self._job_id,
+            client=self._client,
+        )
+
+        # Instantiate and execute
+        action = action_cls(validated_params, ctx)
+        try:
+            result = action.execute()
+        except Exception as e:
+            logger.finish()
+            raise ExecutionError(f'Action execution failed: {e}') from e
+
+        # Validate result with warning-only mode
+        result_model = getattr(action_cls, 'result_model', NoResult)
+        if result_model is not NoResult:
+            result = validate_result(result, result_model, logger)
+
+        logger.finish()
+        return result
+
+
+__all__ = ['LocalExecutor']

synapse_sdk/plugins/executors/ray/__init__.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from synapse_sdk.plugins.executors.ray.base import BaseRayExecutor, read_requirements
+from synapse_sdk.plugins.executors.ray.job import RayJobExecutor
+from synapse_sdk.plugins.executors.ray.jobs_api import RayJobsApiExecutor
+from synapse_sdk.plugins.executors.ray.pipeline import PipelineDefinition, RayPipelineExecutor
+from synapse_sdk.plugins.executors.ray.task import RayActorExecutor, RayTaskExecutor
+
+__all__ = [
+    'BaseRayExecutor',
+    'PipelineDefinition',
+    'RayActorExecutor',
+    'RayJobExecutor',
+    'RayJobsApiExecutor',
+    'RayPipelineExecutor',
+    'RayTaskExecutor',
+    'read_requirements',
+]

synapse_sdk/plugins/executors/ray/base.py

@@ -0,0 +1,282 @@
+"""Base class for Ray executors with shared runtime env logic."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Literal
+
+from synapse_sdk.plugins.context import PluginEnvironment
+from synapse_sdk.plugins.enums import PackageManager
+
+
+def read_requirements(file_path: str | Path) -> list[str] | None:
+    """Read and parse a requirements.txt file.
+
+    Args:
+        file_path: Path to the requirements.txt file.
+
+    Returns:
+        List of requirement strings, or None if file doesn't exist.
+    """
+    path = Path(file_path)
+    if not path.exists():
+        return None
+
+    requirements = []
+    for line in path.read_text().splitlines():
+        stripped = line.strip()
+        if stripped and not stripped.startswith('#'):
+            requirements.append(stripped)
+    return requirements
+
+
+class BaseRayExecutor:
+    """Base class for Ray executors with shared runtime env building logic."""
+
+    def __init__(
+        self,
+        env: PluginEnvironment | dict[str, Any] | None = None,
+        *,
+        runtime_env: dict[str, Any] | None = None,
+        working_dir: str | Path | None = None,
+        requirements_file: str | Path | None = None,
+        package_manager: PackageManager | Literal['pip', 'uv'] = PackageManager.PIP,
+        package_manager_options: list[str] | None = None,
+        wheels_dir: str = 'wheels',
+        ray_address: str = 'auto',
+        include_sdk: bool = False,
+    ) -> None:
+        """Initialize base Ray executor.
+
+        Args:
+            env: Environment config for the action. If None, loads from os.environ.
+            runtime_env: Ray runtime environment config.
+            working_dir: Plugin working directory.
+            requirements_file: Path to requirements.txt.
+            package_manager: Package manager to use ('pip' or 'uv').
+            package_manager_options: Additional options for the package manager.
+            wheels_dir: Directory containing .whl files relative to working_dir.
+            ray_address: Ray cluster address (for detecting remote mode).
+            include_sdk: If True, bundle local SDK with upload (for development).
+        """
+        if env is None:
+            self._env = PluginEnvironment.from_environ()
+        elif isinstance(env, dict):
+            self._env = PluginEnvironment(env)
+        else:
+            self._env = env
+
+        self._runtime_env = runtime_env or {}
+        self._working_dir = Path(working_dir) if working_dir else None
+        self._requirements_file = Path(requirements_file) if requirements_file else None
+        self._package_manager = PackageManager(package_manager)
+        self._package_manager_options = package_manager_options
+        self._wheels_dir = wheels_dir
+        self._ray_address = ray_address
+        self._include_sdk = include_sdk
+        self._gcs_uri: str | None = None  # Cached GCS URI
+
+    def _ray_init(self) -> None:
+        """Initialize Ray connection with SDK bundling if requested."""
+        import ray
+
+        if ray.is_initialized():
+            return
+
+        # Build init kwargs
+        init_kwargs: dict[str, Any] = {
+            'address': self._ray_address,
+            'ignore_reinit_error': True,
+        }
+
+        # Build runtime_env for init level
+        runtime_env: dict[str, Any] = {}
+
+        # Include SDK at init level (directories only work here, not at actor level)
+        if self._include_sdk:
+            import synapse_sdk
+
+            sdk_path = str(Path(synapse_sdk.__file__).parent)
+            runtime_env['py_modules'] = [sdk_path]
+
+        # Include working_dir at init level for local mode
+        # (local paths are only supported at ray.init() level, not at actor level)
+        if self._working_dir and not self._is_remote_cluster():
+            runtime_env['working_dir'] = str(self._working_dir)
+
+        if runtime_env:
+            init_kwargs['runtime_env'] = runtime_env
+
+        ray.init(**init_kwargs)
+
+    def _is_remote_cluster(self) -> bool:
+        """Check if connecting to a remote Ray cluster."""
+        # Remote if address starts with ray:// protocol
+        return self._ray_address.startswith('ray://')
+
+    def _get_working_dir_uri(self) -> str | None:
+        """Get working directory URI, uploading to GCS for remote clusters."""
+        if not self._working_dir:
+            return None
+
+        # For remote clusters, upload to GCS
+        if self._is_remote_cluster():
+            if self._gcs_uri is None:
+                from synapse_sdk.plugins.executors.ray.packaging import upload_working_dir_to_gcs
+
+                self._gcs_uri = upload_working_dir_to_gcs(self._working_dir)
+            return self._gcs_uri
+
+        # Local mode - use path directly
+        return str(self._working_dir)
+
+    def _build_runtime_env(self) -> dict[str, Any]:
+        """Build runtime environment with working_dir, requirements, and env vars."""
+        runtime_env = {**self._runtime_env}
+
+        # Set working_dir if provided (uploads to GCS for remote clusters)
+        # Note: Local paths are only supported at ray.init() level, not at actor level
+        if self._working_dir and 'working_dir' not in runtime_env:
+            if self._is_remote_cluster():
+                working_dir_uri = self._get_working_dir_uri()
+                if working_dir_uri:
+                    runtime_env['working_dir'] = working_dir_uri
+            # For local mode, working_dir is passed at ray.init() level
+
+        # Build package manager config with requirements and wheels
+        pm_key = str(self._package_manager)  # 'pip' or 'uv'
+        requirements = self._get_requirements() or []
+        wheel_files = self._get_wheel_files()
+
+        # Check if requirements.txt contains pip args (lines starting with -)
+        # If so, use file path instead of package list (Ray only supports pip args via file)
+        has_pip_args = any(req.strip().startswith('-') for req in requirements)
+        req_file_path = self._get_requirements_file_path()
+
+        if has_pip_args and req_file_path:
+            # Use requirements.txt file path to preserve pip args like --extra-index-url
+            # Reference the file relative to working_dir
+            runtime_env[pm_key] = 'requirements.txt'
+            # Add wheel files separately if any
+            if wheel_files:
+                # Wheels need to be in packages list, so we need a hybrid approach
+                # For now, log a warning - wheels with pip args is not fully supported
+                pass
+        else:
+            # Combine requirements and wheel files as package list
+            all_packages = requirements + wheel_files
+
+            if all_packages:
+                # Initialize package manager config
+                if pm_key not in runtime_env:
+                    runtime_env[pm_key] = {'packages': []}
+                elif isinstance(runtime_env[pm_key], list):
+                    runtime_env[pm_key] = {'packages': runtime_env[pm_key]}
+
+                runtime_env[pm_key].setdefault('packages', [])
+                runtime_env[pm_key]['packages'].extend(all_packages)
+
+            # Apply package manager options (only when using packages list)
+            pm_options = self._get_package_manager_options()
+            if pm_options and pm_key in runtime_env and isinstance(runtime_env[pm_key], dict):
+                for key, value in pm_options.items():
+                    runtime_env[pm_key][key] = value
+
+        # Add env vars
+        runtime_env.setdefault('env_vars', {})
+        runtime_env['env_vars'].update(self._env.to_dict())
+
+        # Include Synapse credentials for backend client on workers
+        from synapse_sdk.utils.auth import ENV_SYNAPSE_ACCESS_TOKEN, ENV_SYNAPSE_HOST, load_credentials
+
+        host, token = load_credentials()
+        if host and ENV_SYNAPSE_HOST not in runtime_env['env_vars']:
+            runtime_env['env_vars'][ENV_SYNAPSE_HOST] = host
+        if token and ENV_SYNAPSE_ACCESS_TOKEN not in runtime_env['env_vars']:
+            runtime_env['env_vars'][ENV_SYNAPSE_ACCESS_TOKEN] = token
+
+        return runtime_env
+
+    def _get_package_manager_options(self) -> dict[str, Any]:
+        """Get package manager options with defaults.
+
+        Returns:
+            Dict of package manager options.
+        """
+        user_options = self._package_manager_options or []
+
+        if self._package_manager == PackageManager.UV:
+            defaults = ['--no-cache']
+            options_list = defaults.copy()
+            for opt in user_options:
+                if opt not in options_list:
+                    options_list.append(opt)
+            return {'uv_pip_install_options': options_list}
+        else:
+            # pip - use pip_install_options with --upgrade flag
+            defaults = ['--upgrade']
+            options_list = defaults.copy()
+            for opt in user_options:
+                if opt not in options_list:
+                    options_list.append(opt)
+            return {'pip_install_options': options_list}
+
+    def _get_requirements(self) -> list[str] | None:
+        """Get requirements from file.
+
+        Returns:
+            List of requirements, or None if no requirements file found.
+        """
+        # Explicit requirements file takes priority
+        if self._requirements_file:
+            return read_requirements(self._requirements_file)
+
+        # Auto-discover from working_dir
+        if self._working_dir:
+            req_path = self._working_dir / 'requirements.txt'
+            return read_requirements(req_path)
+
+        return None
+
+    def _get_requirements_file_path(self) -> Path | None:
+        """Get path to requirements.txt file if it exists.
+
+        Returns:
+            Path to requirements.txt, or None if not found.
+        """
+        if self._requirements_file and Path(self._requirements_file).exists():
+            return Path(self._requirements_file)
+
+        if self._working_dir:
+            req_path = self._working_dir / 'requirements.txt'
+            if req_path.exists():
+                return req_path
+
+        return None
+
+    def _get_wheel_files(self) -> list[str]:
+        """Get wheel file paths for Ray runtime env.
+
+        Scans the wheels_dir for .whl files and returns them as Ray-compatible
+        paths using ${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}.
+
+        Returns:
+            List of wheel file paths for Ray.
+        """
+        if not self._working_dir:
+            return []
+
+        wheels_path = self._working_dir / self._wheels_dir
+        if not wheels_path.exists():
+            return []
+
+        wheel_files = []
+        for whl in wheels_path.glob('*.whl'):
+            # Use Ray's working dir variable for the path
+            ray_path = f'${{RAY_RUNTIME_ENV_CREATE_WORKING_DIR}}/{self._wheels_dir}/{whl.name}'
+            wheel_files.append(ray_path)
+
+        return wheel_files
+
+
+__all__ = ['BaseRayExecutor', 'read_requirements']