synapse-sdk 1.0.0a23__py3-none-any.whl → 2025.12.3__py3-none-any.whl

synapse_sdk/clients/agent/container.py

@@ -0,0 +1,143 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Dict, Iterable, Optional, Union
+
+from synapse_sdk.clients.base import BaseClient
+
+
+class ContainerClientMixin(BaseClient):
+    """Client mixin exposing the agent container management API."""
+
+    def health_check(self):
+        """Perform a health check on Docker sock."""
+        path = 'health/'
+        return self._get(path)
+
+    def list_containers(self, params: Optional[Dict[str, Any]] = None, *, list_all: bool = False):
+        """List containers managed by the agent.
+
+        Args:
+            params: Optional query parameters (e.g. {'status': 'running'}).
+            list_all: When True, returns ``(generator, count)`` covering every page.
+
+        Returns:
+            dict | tuple: Standard paginated response or a tuple for ``list_all``.
+        """
+        path = 'containers/'
+        return self._list(path, params=params, list_all=list_all)
+
+    def get_container(self, container_id: Union[int, str]):
+        """Retrieve details for a specific container."""
+        path = f'containers/{container_id}/'
+        return self._get(path)
+
+    def delete_container(self, container_id: Union[int, str]):
+        """Stop and remove a container."""
+        path = f'containers/{container_id}/'
+        return self._delete(path)
+
+    def create_container(
+        self,
+        plugin_release: Optional[Union[str, Any]] = None,
+        *,
+        model: Optional[int] = None,
+        params: Optional[Dict[str, Any]] = None,
+        envs: Optional[Dict[str, str]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        labels: Optional[Iterable[str]] = None,
+        plugin_file: Optional[Union[str, Path]] = None,
+    ):
+        """Create a Docker container running a plugin Gradio interface.
+
+        If a container with the same ``plugin_release`` and ``model`` already exists,
+        it will be restarted instead of creating a new one.
+
+        Args:
+            plugin_release: Plugin identifier. Accepts either ``synapse_sdk.plugins.models.PluginRelease``
+                instances or the ``"<plugin_code>@<version>"`` shorthand string.
+            model: Optional model ID to associate with the container. Used together with
+                ``plugin_release`` to uniquely identify a container for restart behavior.
+            params: Arbitrary parameters forwarded to ``plugin/gradio_interface.py``.
+            envs: Extra environment variables injected into the container.
+            metadata: Additional metadata stored with the container record.
+            labels: Optional container labels/tags for display or filtering.
+            plugin_file: Optional path to a packaged plugin release to upload directly.
+                The archive must contain ``plugin/gradio_interface.py``.
+
+        Returns:
+            dict: Container creation response that includes the exposed Gradio endpoint.
+                If an existing container was restarted, the response includes ``restarted: True``.
+
+        Raises:
+            FileNotFoundError: If ``plugin_file`` is provided but does not exist.
+            ValueError: If neither ``plugin_release`` nor ``plugin_file`` are provided.
+        """
+        if not plugin_release and not plugin_file:
+            raise ValueError('Either "plugin_release" or "plugin_file" must be provided to create a container.')
+
+        data: Dict[str, Any] = {}
+
+        if plugin_release:
+            data.update(self._serialize_plugin_release(plugin_release))
+
+        if model is not None:
+            data['model'] = model
+
+        optional_payload = {
+            'params': params if params is not None else None,
+            'envs': envs or None,
+            'metadata': metadata or None,
+            'labels': list(labels) if labels else None,
+        }
+        data.update({key: value for key, value in optional_payload.items() if value is not None})
+
+        files = None
+        if plugin_file:
+            file_path = Path(plugin_file)
+            if not file_path.exists():
+                raise FileNotFoundError(f'Plugin release file not found: {file_path}')
+            files = {'file': file_path}
+        post_kwargs = {'data': data}
+        if files:
+            post_kwargs['files'] = files
+
+        return self._post('containers/', **post_kwargs)
+
+    @staticmethod
+    def _serialize_plugin_release(plugin_release: Union[str, Any]) -> Dict[str, Any]:
+        """Normalize plugin release data for API payloads."""
+        if hasattr(plugin_release, 'code') and hasattr(plugin_release, 'version'):
+            payload = {
+                'plugin_release': plugin_release.code,
+                'plugin': getattr(plugin_release, 'plugin', None),
+                'version': plugin_release.version,
+            }
+
+            # Extract action and entrypoint from the first action in the config
+            if hasattr(plugin_release, 'config') and 'actions' in plugin_release.config:
+                actions = plugin_release.config['actions']
+                if actions:
+                    # Get the first action (typically 'gradio')
+                    action_name = next(iter(actions.keys()))
+                    action_config = actions[action_name]
+                    payload['action'] = action_name
+
+                    # Convert entrypoint from dotted path to file path
+                    if 'entrypoint' in action_config:
+                        entrypoint = action_config['entrypoint']
+                        # Convert 'plugin.gradio_interface.app' to 'plugin/gradio_interface.py'
+                        file_path = entrypoint.rsplit('.', 1)[0].replace('.', '/') + '.py'
+                        payload['entrypoint'] = file_path
+
+            return payload
+
+        if isinstance(plugin_release, str):
+            payload = {'plugin_release': plugin_release}
+            if '@' in plugin_release:
+                plugin, version = plugin_release.rsplit('@', 1)
+                payload.setdefault('plugin', plugin)
+                payload.setdefault('version', version)
+            return payload
+
+        raise TypeError('plugin_release must be a PluginRelease instance or a formatted string "code@version"')

synapse_sdk/clients/agent/core.py

@@ -5,3 +5,22 @@ class CoreClientMixin(BaseClient):
     def health_check(self):
         path = 'health/'
         return self._get(path)
+
+    def get_metrics(self, panel):
+        path = f'metrics/{panel}/'
+        return self._get(path)
+
+    def get_code_server_info(self, workspace_path=None):
+        """Get code-server connection information from the agent.
+
+        Args:
+            workspace_path: Optional path to set as the workspace directory
+
+        Returns:
+            dict: Code-server connection information
+        """
+        path = 'code-server/info/'
+        params = {}
+        if workspace_path:
+            params['workspace'] = workspace_path
+        return self._get(path, params=params)

synapse_sdk/clients/agent/ray.py

@@ -1,8 +1,86 @@
+import weakref
+from concurrent.futures import ThreadPoolExecutor
+
 from synapse_sdk.clients.base import BaseClient
 from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.utils.network import (
+    HTTPStreamManager,
+    StreamLimits,
+    WebSocketStreamManager,
+    http_to_websocket_url,
+    sanitize_error_message,
+    validate_resource_id,
+    validate_timeout,
+)
 
 
 class RayClientMixin(BaseClient):
+    """Mixin class providing Ray cluster management and monitoring functionality.
+
+    This mixin extends BaseClient with Ray-specific operations for interacting with
+    Apache Ray distributed computing clusters. It provides comprehensive job management,
+    node monitoring, task tracking, and Ray Serve application control capabilities.
+
+    Key Features:
+        - Job lifecycle management (list, get, monitor)
+        - Real-time log streaming via WebSocket and HTTP protocols
+        - Node and task monitoring
+        - Ray Serve application deployment and management
+        - Robust error handling with input validation
+        - Resource management with automatic cleanup
+
+    Streaming Capabilities:
+        - WebSocket streaming for real-time log tailing
+        - HTTP streaming as fallback protocol
+        - Configurable timeouts and stream limits
+        - Automatic protocol validation and error recovery
+
+    Resource Management:
+        - Thread pool for concurrent operations (5 workers)
+        - WeakSet for tracking active connections
+        - Automatic cleanup on object destruction
+        - Stream limits to prevent resource exhaustion
+
+    Usage Examples:
+        Basic job operations:
+            >>> client = RayClient(base_url="http://ray-head:8265")
+            >>> jobs = client.list_jobs()
+            >>> job = client.get_job('job-12345')
+
+        Real-time log streaming:
+            >>> # WebSocket streaming (preferred)
+            >>> for log_line in client.tail_job_logs('job-12345', protocol='websocket'):
+            ...     print(log_line)
+
+            >>> # HTTP streaming (fallback)
+            >>> for log_line in client.tail_job_logs('job-12345', protocol='stream'):
+            ...     print(log_line)
+
+        Node and task monitoring:
+            >>> nodes = client.list_nodes()
+            >>> tasks = client.list_tasks()
+            >>> node_details = client.get_node('node-id')
+
+        Ray Serve management:
+            >>> apps = client.list_serve_applications()
+            >>> client.delete_serve_application('app-id')
+
+    Note:
+        This class is designed as a mixin and should be combined with other
+        client classes that provide authentication and base functionality.
+        It requires the BaseClient foundation for HTTP operations.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._thread_pool = ThreadPoolExecutor(max_workers=5, thread_name_prefix='ray_client_')
+        self._active_connections = weakref.WeakSet()
+
+        # Initialize stream managers
+        stream_limits = StreamLimits()
+        self._websocket_manager = WebSocketStreamManager(self._thread_pool, stream_limits)
+        self._http_manager = HTTPStreamManager(self.requests_session, stream_limits)
+
     def get_job(self, pk):
         path = f'jobs/{pk}/'
         return self._get(path)
@@ -15,19 +93,180 @@ class RayClientMixin(BaseClient):
         path = f'jobs/{pk}/logs/'
         return self._get(path)
 
-    def tail_job_logs(self, pk):
-        if self.long_poll_handler:
-            raise ClientError(400, '"tail_job_logs" does not support long polling')
+    def websocket_tail_job_logs(self, pk, stream_timeout=10):
+        """Stream job logs in real-time using WebSocket protocol.
+
+        Establishes a WebSocket connection to stream job logs as they are generated.
+        This method provides the lowest latency for real-time log monitoring and is
+        the preferred protocol when available.
+
+        Args:
+            pk (str): Job primary key or identifier. Must be alphanumeric with
+                     optional hyphens/underscores, max 100 characters.
+            stream_timeout (float, optional): Maximum time in seconds to wait for
+                                            log data. Defaults to 10. Must be positive
+                                            and cannot exceed 300 seconds.
+
+        Returns:
+            Generator[str, None, None]: A generator yielding log lines as strings.
+                                      Each line includes a newline character.
+
+        Raises:
+            ClientError:
+                - 400: If long polling is enabled (incompatible)
+                - 400: If pk is empty, contains invalid characters, or too long
+                - 400: If stream_timeout is not positive or exceeds maximum
+                - 500: If WebSocket library is unavailable
+                - 503: If connection to Ray cluster fails
+                - 408: If connection timeout occurs
+                - 429: If stream limits are exceeded (lines, size, messages)
+
+        Usage:
+            >>> # Basic log streaming
+            >>> for log_line in client.websocket_tail_job_logs('job-12345'):
+            ...     print(log_line.strip())
+
+            >>> # With custom timeout
+            >>> for log_line in client.websocket_tail_job_logs('job-12345', stream_timeout=30):
+            ...     if 'ERROR' in log_line:
+            ...         break
+
+        Technical Notes:
+            - Uses WebSocketStreamManager for connection management
+            - Automatic input validation and sanitization
+            - Resource cleanup handled by WeakSet tracking
+            - Stream limits prevent memory exhaustion
+            - Thread pool manages WebSocket operations
+
+        See Also:
+            stream_tail_job_logs: HTTP-based alternative
+            tail_job_logs: Protocol-agnostic wrapper method
+        """
+        if hasattr(self, 'long_poll_handler') and self.long_poll_handler:
+            raise ClientError(400, '"websocket_tail_job_logs" does not support long polling')
+
+        # Validate inputs using network utilities
+        validated_pk = validate_resource_id(pk, 'job')
+        validated_timeout = validate_timeout(stream_timeout)
+
+        # Build WebSocket URL
+        path = f'ray/jobs/{validated_pk}/logs/ws/'
+        url = self._get_url(path, trailing_slash=True)
+        ws_url = http_to_websocket_url(url)
+
+        # Get headers and use WebSocket manager
+        headers = self._get_headers()
+        headers['Agent-Token'] = f'Token {self.agent_token}'
+        context = f'job {validated_pk}'
+
+        return self._websocket_manager.stream_logs(ws_url, headers, validated_timeout, context)
+
+    def stream_tail_job_logs(self, pk, stream_timeout=10):
+        """Stream job logs in real-time using HTTP chunked transfer encoding.
+
+        Establishes an HTTP connection with chunked transfer encoding to stream
+        job logs as they are generated. This method serves as a reliable fallback
+        when WebSocket connections are not available or suitable.
+
+        Args:
+            pk (str): Job primary key or identifier. Must be alphanumeric with
+                     optional hyphens/underscores, max 100 characters.
+            stream_timeout (float, optional): Maximum time in seconds to wait for
+                                            log data. Defaults to 10. Must be positive
+                                            and cannot exceed 300 seconds.
+
+        Returns:
+            Generator[str, None, None]: A generator yielding log lines as strings.
+                                      Each line includes a newline character.
+
+        Raises:
+            ClientError:
+                - 400: If long polling is enabled (incompatible)
+                - 400: If pk is empty, contains invalid characters, or too long
+                - 400: If stream_timeout is not positive or exceeds maximum
+                - 503: If connection to Ray cluster fails
+                - 408: If connection or read timeout occurs
+                - 404: If job is not found
+                - 429: If stream limits are exceeded (lines, size, messages)
+                - 500: If unexpected streaming error occurs
+
+        Usage:
+            >>> # Basic HTTP log streaming
+            >>> for log_line in client.stream_tail_job_logs('job-12345'):
+            ...     print(log_line.strip())
 
-        path = f'jobs/{pk}/tail_logs/'
+            >>> # With error handling and custom timeout
+            >>> try:
+            ...     for log_line in client.stream_tail_job_logs('job-12345', stream_timeout=60):
+            ...         if 'COMPLETED' in log_line:
+            ...             break
+            ... except ClientError as e:
+            ...     print(f"Streaming failed: {e}")
 
-        url = self._get_url(path)
+        Technical Notes:
+            - Uses HTTPStreamManager for connection management
+            - Automatic input validation and sanitization
+            - Proper HTTP response cleanup on completion/error
+            - Stream limits prevent memory exhaustion
+            - Filters out oversized lines (>10KB) automatically
+            - Connection reuse through requests session
+
+        See Also:
+            websocket_tail_job_logs: WebSocket-based alternative (preferred)
+            tail_job_logs: Protocol-agnostic wrapper method
+        """
+        if hasattr(self, 'long_poll_handler') and self.long_poll_handler:
+            raise ClientError(400, '"stream_tail_job_logs" does not support long polling')
+
+        # Validate inputs using network utilities
+        validated_pk = validate_resource_id(pk, 'job')
+        validated_timeout = validate_timeout(stream_timeout)
+
+        # Build HTTP URL and prepare request
+        path = f'ray/jobs/{validated_pk}/logs/stream/'
+        url = self._get_url(path, trailing_slash=True)
         headers = self._get_headers()
+        headers['Agent-Token'] = f'Token {self.agent_token}'
+        timeout = (self.timeout['connect'], validated_timeout)
+        context = f'job {validated_pk}'
+
+        return self._http_manager.stream_logs(url, headers, timeout, context)
+
+    def tail_job_logs(self, pk, stream_timeout=10, protocol='stream'):
+        """Tail job logs using either WebSocket or HTTP streaming.
+
+        Args:
+            pk: Job primary key
+            stream_timeout: Timeout for streaming operations
+            protocol: 'websocket' or 'stream' (default: 'stream')
+        """
+        # Validate protocol first
+        if protocol not in ('websocket', 'stream'):
+            raise ClientError(400, f'Unsupported protocol: {protocol}. Use "websocket" or "stream"')
 
-        response = self.requests_session.get(url, headers=headers, stream=True)
-        for line in response.iter_lines(decode_unicode=True):
-            if line:
-                yield f'{line}\n'
+        # Pre-validate common inputs using network utilities
+        validate_resource_id(pk, 'job')
+        validate_timeout(stream_timeout)
+
+        try:
+            if protocol == 'websocket':
+                return self.websocket_tail_job_logs(pk, stream_timeout)
+            else:  # protocol == 'stream'
+                return self.stream_tail_job_logs(pk, stream_timeout)
+        except ClientError:
+            raise
+        except Exception as e:
+            # Fallback error handling using network utility
+            sanitized_error = sanitize_error_message(str(e), f'job {pk}')
+            raise ClientError(500, f'Protocol {protocol} failed: {sanitized_error}')
+
+    def __del__(self):
+        """Cleanup resources when object is destroyed."""
+        try:
+            if hasattr(self, '_thread_pool'):
+                self._thread_pool.shutdown(wait=False)
+        except Exception:
+            pass  # Ignore cleanup errors during destruction
 
     def get_node(self, pk):
         path = f'nodes/{pk}/'
@@ -56,3 +295,53 @@ class RayClientMixin(BaseClient):
     def delete_serve_application(self, pk):
         path = f'serve_applications/{pk}/'
         return self._delete(path)
+
+    def stop_job(self, pk):
+        """Stop a running job gracefully.
+
+        Uses Ray's stop_job() API to request graceful termination of the job.
+        This preserves job state and allows for potential resubmission later.
+
+        Args:
+            pk (str): Job primary key or identifier. Must be alphanumeric with
+                     optional hyphens/underscores, max 100 characters.
+
+        Returns:
+            dict: Response containing job status and stop details.
+
+        Raises:
+            ClientError:
+                - 400: If pk is empty, contains invalid characters, or too long
+                - 400: If job is already in terminal state (STOPPED, FAILED, etc.)
+                - 404: If job is not found
+                - 503: If connection to Ray cluster fails
+                - 500: If unexpected error occurs during stop
+
+        Usage:
+            >>> # Stop a running job
+            >>> result = client.stop_job('job-12345')
+            >>> print(result['status'])  # Should show 'STOPPING' or similar
+
+            >>> # Handle stop errors
+            >>> try:
+            ...     client.stop_job('job-12345')
+            ... except ClientError as e:
+            ...     print(f"Stop failed: {e}")
+
+        Technical Notes:
+            - Uses Ray's stop_job() API for graceful termination
+            - Validates job state before attempting stop
+            - Maintains consistency with existing SDK patterns
+            - Provides detailed error messages for debugging
+
+        See Also:
+            resume_job: Method for restarting stopped jobs
+        """
+        # Validate inputs using network utilities
+        validated_pk = validate_resource_id(pk, 'job')
+
+        # Build API path for job stop
+        path = f'jobs/{validated_pk}/stop/'
+
+        # Use _post method with empty data to match Ray's API pattern
+        return self._post(path)

synapse_sdk/clients/backend/__init__.py

@@ -1,6 +1,7 @@
 from synapse_sdk.clients.backend.annotation import AnnotationClientMixin
 from synapse_sdk.clients.backend.core import CoreClientMixin
-from synapse_sdk.clients.backend.dataset import DatasetClientMixin
+from synapse_sdk.clients.backend.data_collection import DataCollectionClientMixin
+from synapse_sdk.clients.backend.hitl import HITLClientMixin
 from synapse_sdk.clients.backend.integration import IntegrationClientMixin
 from synapse_sdk.clients.backend.ml import MLClientMixin
 
@@ -8,27 +9,44 @@ from synapse_sdk.clients.backend.ml import MLClientMixin
 class BackendClient(
     AnnotationClientMixin,
     CoreClientMixin,
-    DatasetClientMixin,
+    DataCollectionClientMixin,
     IntegrationClientMixin,
     MLClientMixin,
+    HITLClientMixin,
 ):
+    """BackendClient is a client for the synapse backend API.
+
+    * Access token overrides authorization token and tenant token.
+
+    Attrs:
+        access_token (str): The synapse access token for the synapse backend API.
+        authorization_token (str): The authorization token for the synapse backend API.
+        tenant_token (str): The tenant token for the synapse backend API.
+        agent_token (str): The agent token for the backend API.
+        timeout (Dict): Set reasonable default timeouts for better UX. It can receive keys called 'connect' and 'read'.
+    """
+
     name = 'Backend'
-    token = None
-    tenant = None
+    access_token = None
+    authorization_token = None
+    tenant_token = None
     agent_token = None
 
-    def __init__(self, base_url, token=None, tenant=None, agent_token=None):
-        super().__init__(base_url)
-        self.token = token
-        self.tenant = tenant
+    def __init__(self, base_url, access_token=None, token=None, tenant=None, agent_token=None, timeout=None, **kwargs):
+        super().__init__(base_url, timeout=timeout)
+        self.access_token = access_token
+        self.authorization_token = token
+        self.tenant_token = tenant
         self.agent_token = agent_token
 
     def _get_headers(self):
         headers = {}
-        if self.token:
-            headers['Authorization'] = f'Token {self.token}'
-        if self.tenant:
-            headers['SYNAPSE-Tenant'] = f'Token {self.tenant}'
+        if self.access_token:
+            headers['Synapse-Access-Token'] = f'Token {self.access_token}'
+        if self.authorization_token:
+            headers['Authorization'] = f'Token {self.authorization_token}'
+        if self.tenant_token:
+            headers['Synapse-Tenant'] = f'Token {self.tenant_token}'
         if self.agent_token:
             headers['SYNAPSE-Agent'] = f'Token {self.agent_token}'
         return headers

synapse_sdk/clients/backend/annotation.py

@@ -7,18 +7,26 @@ class AnnotationClientMixin(BaseClient):
         path = f'projects/{pk}/'
         return self._get(path)
 
+    def get_task(self, pk, params):
+        path = f'tasks/{pk}/'
+        return self._get(path, params=params)
+
+    def annotate_task_data(self, pk, data):
+        path = f'tasks/{pk}/annotate_task_data/'
+        return self._put(path, data=data)
+
     def get_task_tag(self, pk):
         path = f'task_tags/{pk}/'
         return self._get(path)
 
-    def list_task_tags(self, data):
+    def list_task_tags(self, params):
         path = 'task_tags/'
-        return self._list(path, data=data)
+        return self._list(path, params=params)
 
-    def list_tasks(self, data, url_conversion=None, list_all=False):
-        path = 'tasks/'
+    def list_tasks(self, params=None, url_conversion=None, list_all=False):
+        path = 'sdk/tasks/'
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
-        return self._list(path, data=data, url_conversion=url_conversion, list_all=list_all)
+        return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
 
     def create_tasks(self, data):
         path = 'tasks/'

synapse_sdk/clients/backend/core.py

@@ -3,15 +3,42 @@ import os
 from pathlib import Path
 
 from synapse_sdk.clients.base import BaseClient
+from synapse_sdk.utils.file import read_file_in_chunks
 
 
 class CoreClientMixin(BaseClient):
     def create_chunked_upload(self, file_path):
-        def read_file_in_chunks(file_path, chunk_size=1024 * 1024 * 50):
-            with open(file_path, 'rb') as file:
-                while chunk := file.read(chunk_size):
-                    yield chunk
+        """
+        Upload a file using chunked upload for efficient handling of large files.
 
+        This method breaks the file into chunks and uploads them sequentially to the server.
+        It calculates an MD5 hash of the entire file to ensure data integrity during upload.
+
+        Args:
+            file_path (str | Path): Path to the file to upload
+
+        Returns:
+            dict: Response from the server after successful upload completion,
+                  typically containing upload confirmation and file metadata
+
+        Raises:
+            FileNotFoundError: If the specified file doesn't exist
+            PermissionError: If the file can't be read due to permissions
+            ClientError: If there's an error during the upload process
+            OSError: If there's an OS-level error accessing the file
+
+        Example:
+            ```python
+            client = CoreClientMixin(base_url='https://api.example.com')
+            result = client.create_chunked_upload('/path/to/large_file.zip')
+            print(f"Upload completed: {result}")
+            ```
+
+        Note:
+            - Uses 50MB chunks by default for optimal upload performance
+            - Automatically resumes from the last successfully uploaded chunk
+            - Verifies upload integrity using MD5 checksum
+        """
         file_path = Path(file_path)
         size = os.path.getsize(file_path)
         hash_md5 = hashlib.md5()