synapse-sdk 1.0.0b13__py3-none-any.whl → 1.0.0b14__py3-none-any.whl

synapse_sdk/clients/agent/ray.py

@@ -15,8 +15,7 @@ from synapse_sdk.utils.network import (
 
 
 class RayClientMixin(BaseClient):
-    """
-    Mixin class providing Ray cluster management and monitoring functionality.
+    """Mixin class providing Ray cluster management and monitoring functionality.
 
     This mixin extends BaseClient with Ray-specific operations for interacting with
     Apache Ray distributed computing clusters. It provides comprehensive job management,
@@ -95,8 +94,7 @@ class RayClientMixin(BaseClient):
         return self._get(path)
 
     def websocket_tail_job_logs(self, pk, stream_timeout=10):
-        """
-        Stream job logs in real-time using WebSocket protocol.
+        """Stream job logs in real-time using WebSocket protocol.
 
         Establishes a WebSocket connection to stream job logs as they are generated.
         This method provides the lowest latency for real-time log monitoring and is
@@ -153,18 +151,18 @@ class RayClientMixin(BaseClient):
 
         # Build WebSocket URL
         path = f'ray/jobs/{validated_pk}/logs/ws/'
-        url = self._get_url(path)
+        url = self._get_url(path, trailing_slash=True)
         ws_url = http_to_websocket_url(url)
 
         # Get headers and use WebSocket manager
         headers = self._get_headers()
+        headers['Agent-Token'] = f'Token {self.agent_token}'
         context = f'job {validated_pk}'
 
         return self._websocket_manager.stream_logs(ws_url, headers, validated_timeout, context)
 
     def stream_tail_job_logs(self, pk, stream_timeout=10):
-        """
-        Stream job logs in real-time using HTTP chunked transfer encoding.
+        """Stream job logs in real-time using HTTP chunked transfer encoding.
 
         Establishes an HTTP connection with chunked transfer encoding to stream
         job logs as they are generated. This method serves as a reliable fallback
@@ -225,17 +223,17 @@ class RayClientMixin(BaseClient):
         validated_timeout = validate_timeout(stream_timeout)
 
         # Build HTTP URL and prepare request
-        path = f'ray/jobs/{validated_pk}/logs/stream'
-        url = self._get_url(path)
+        path = f'ray/jobs/{validated_pk}/logs/stream/'
+        url = self._get_url(path, trailing_slash=True)
         headers = self._get_headers()
+        headers['Agent-Token'] = f'Token {self.agent_token}'
         timeout = (self.timeout['connect'], validated_timeout)
         context = f'job {validated_pk}'
 
         return self._http_manager.stream_logs(url, headers, timeout, context)
 
     def tail_job_logs(self, pk, stream_timeout=10, protocol='stream'):
-        """
-        Tail job logs using either WebSocket or HTTP streaming.
+        """Tail job logs using either WebSocket or HTTP streaming.
 
         Args:
             pk: Job primary key

synapse_sdk/clients/backend/annotation.py

@@ -24,7 +24,7 @@ class AnnotationClientMixin(BaseClient):
         return self._list(path, params=params)
 
     def list_tasks(self, params=None, url_conversion=None, list_all=False):
-        path = 'tasks/'
+        path = 'sdk/tasks/'
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
         return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
 

synapse_sdk/clients/backend/core.py

@@ -3,15 +3,42 @@ import os
 from pathlib import Path
 
 from synapse_sdk.clients.base import BaseClient
+from synapse_sdk.utils.file import read_file_in_chunks
 
 
 class CoreClientMixin(BaseClient):
     def create_chunked_upload(self, file_path):
-        def read_file_in_chunks(file_path, chunk_size=1024 * 1024 * 50):
-            with open(file_path, 'rb') as file:
-                while chunk := file.read(chunk_size):
-                    yield chunk
+        """
+        Upload a file using chunked upload for efficient handling of large files.
 
+        This method breaks the file into chunks and uploads them sequentially to the server.
+        It calculates an MD5 hash of the entire file to ensure data integrity during upload.
+
+        Args:
+            file_path (str | Path): Path to the file to upload
+
+        Returns:
+            dict: Response from the server after successful upload completion,
+                  typically containing upload confirmation and file metadata
+
+        Raises:
+            FileNotFoundError: If the specified file doesn't exist
+            PermissionError: If the file can't be read due to permissions
+            ClientError: If there's an error during the upload process
+            OSError: If there's an OS-level error accessing the file
+
+        Example:
+            ```python
+            client = CoreClientMixin(base_url='https://api.example.com')
+            result = client.create_chunked_upload('/path/to/large_file.zip')
+            print(f"Upload completed: {result}")
+            ```
+
+        Note:
+            - Uses 50MB chunks by default for optimal upload performance
+            - Automatically resumes from the last successfully uploaded chunk
+            - Verifies upload integrity using MD5 checksum
+        """
         file_path = Path(file_path)
         size = os.path.getsize(file_path)
         hash_md5 = hashlib.md5()

synapse_sdk/clients/backend/data_collection.py

@@ -1,6 +1,6 @@
 from multiprocessing import Pool
 from pathlib import Path
-from typing import Dict, Optional
+from typing import Dict, Optional, Union
 
 from tqdm import tqdm
 
@@ -9,6 +9,17 @@ from synapse_sdk.clients.utils import get_batched_list, get_default_url_conversi
 
 
 class DataCollectionClientMixin(BaseClient):
+    """Mixin class for data collection operations.
+
+    Provides methods for managing data collections, files, and units
+    in the Synapse backend. Supports both regular file uploads and
+    chunked uploads for large files.
+
+    This mixin extends BaseClient with data collection-specific functionality
+    including file upload capabilities, data unit management, and batch processing
+    operations for efficient data collection workflows.
+    """
+
     def list_data_collection(self):
         path = 'data_collections/'
         return self._list(path)
@@ -22,14 +33,66 @@ class DataCollectionClientMixin(BaseClient):
         path = f'data_collections/{data_collection_id}/?expand=file_specifications'
         return self._get(path)
 
-    def create_data_file(self, file_path: Path):
-        """Create data file to synapse-backend.
+    def create_data_file(
+        self, file_path: Path, use_chunked_upload: bool = False
+    ) -> Union[Dict[str, Union[str, int]], str]:
+        """Create and upload a data file to the Synapse backend.
+
+        This method supports two upload strategies:
+        1. Direct file upload for smaller files (default)
+        2. Chunked upload for large files (automatic when flag is enabled)
 
         Args:
-            file_path: The file pathlib object to upload.
+            file_path: Path object pointing to the file to upload.
+                Must be a valid file path that exists on the filesystem.
+            use_chunked_upload: Boolean flag to enable chunked upload for the file.
+                When True, automatically creates a chunked upload for the file
+                instead of uploading it directly. Defaults to False.
+
+        Returns:
+            Dictionary containing the created data file information including:
+                - id: The unique identifier of the created data file
+                - checksum: The MD5 checksum of the uploaded file
+                - size: The file size in bytes
+                - created_at: Timestamp of creation
+                - Additional metadata fields from the backend
+            Or a string response in case of non-JSON response.
+
+        Raises:
+            FileNotFoundError: If the specified file doesn't exist (for direct upload)
+            PermissionError: If the file can't be read due to permissions
+            ClientError: If the backend returns an error response
+            ValueError: If file_path is not a valid Path object
+
+        Examples:
+            Direct file upload for small files:
+            ```python
+            client = DataCollectionClientMixin(base_url='https://api.example.com')
+            file_path = Path('/path/to/small_file.csv')
+            result = client.create_data_file(file_path)
+            print(f"File uploaded with ID: {result['id']}")
+            ```
+
+            Using chunked upload for large files:
+            ```python
+            # Automatically create chunked upload for large file
+            file_path = Path('/path/to/large_file.zip')
+            result = client.create_data_file(file_path, use_chunked_upload=True)
+            print(f"Large file uploaded with ID: {result['id']}")
+            ```
+
+        Note:
+            - For files larger than 100MB, consider using chunked upload
+            - The chunked upload will be automatically created when the flag is enabled
+            - Chunked uploads provide better reliability for large files
         """
         path = 'data_files/'
-        return self._post(path, files={'file': file_path})
+        if use_chunked_upload:
+            chunked_upload = self.create_chunked_upload(file_path)
+            data = {'chunked_upload': chunked_upload['id']}
+            return self._post(path, data=data)
+        else:
+            return self._post(path, files={'file': file_path})
 
     def get_data_unit(self, data_unit_id: int, params=None):
         path = f'data_units/{data_unit_id}/'
@@ -49,6 +112,16 @@ class DataCollectionClientMixin(BaseClient):
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
         return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
 
+    def data_file_verify_checksums(self, checksums: list[str]):
+        """Check checksums from files are exists.
+
+        Args:
+            checksums: A list of MD5 checksums to verify.
+        """
+        path = 'data_file/verify_checksums/'
+        data = {'checksums': checksums}
+        return self._post(path, data=data)
+
     def upload_data_collection(
         self,
         data_collection_id: int,

synapse_sdk/clients/backend/hitl.py

@@ -8,7 +8,7 @@ class HITLClientMixin(BaseClient):
         return self._get(path)
 
     def list_assignments(self, params=None, url_conversion=None, list_all=False):
-        path = 'assignments/'
+        path = 'sdk/assignments/'
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
         return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
 

synapse_sdk/clients/backend/ml.py

@@ -19,7 +19,7 @@ class MLClientMixin(BaseClient):
         return self._post(path, data=data)
 
     def list_ground_truth_events(self, params=None, url_conversion=None, list_all=False):
-        path = 'ground_truth_events/'
+        path = 'sdk/ground_truth_events/'
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
         return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
 

synapse_sdk/clients/base.py

@@ -39,10 +39,24 @@ class BaseClient:
 
         self.requests_session = requests_session
 
-    def _get_url(self, path):
-        if not path.startswith('http'):
-            return f'{self.base_url}/{path.lstrip("/")}'
-        return path
+    def _get_url(self, path, trailing_slash=False):
+        """Construct a full URL from a path.
+
+        Args:
+            path (str): URL path or full URL
+            trailing_slash (bool): Whether to ensure URL ends with trailing slash
+
+        Returns:
+            str: Complete URL
+        """
+        # Use the path as-is if it's already a full URL, otherwise construct from base_url and path
+        url = path if path.startswith(('http://', 'https://')) else f'{self.base_url}/{path.lstrip("/")}'
+
+        # Add trailing slash if requested and not present
+        if trailing_slash and not url.endswith('/'):
+            url += '/'
+
+        return url
 
     def _get_headers(self):
         return {}
@@ -125,8 +139,7 @@ class BaseClient:
             return response.text
 
     def _get(self, path, url_conversion=None, response_model=None, **kwargs):
-        """
-        Perform a GET request and optionally convert response to a pydantic model.
+        """Perform a GET request and optionally convert response to a pydantic model.
 
         Args:
             path (str): URL path to request.
@@ -152,8 +165,7 @@ class BaseClient:
         return response
 
     def _post(self, path, request_model=None, response_model=None, **kwargs):
-        """
-        Perform a POST request and optionally convert response to a pydantic model.
+        """Perform a POST request and optionally convert response to a pydantic model.
 
         Args:
             path (str): URL path to request.
@@ -173,8 +185,7 @@ class BaseClient:
             return response
 
     def _put(self, path, request_model=None, response_model=None, **kwargs):
-        """
-        Perform a PUT request to the specified path.
+        """Perform a PUT request to the specified path.
 
         Args:
             path (str): The URL path for the request.
@@ -196,8 +207,7 @@ class BaseClient:
             return response
 
     def _patch(self, path, request_model=None, response_model=None, **kwargs):
-        """
-        Perform a PATCH HTTP request to the specified path.
+        """Perform a PATCH HTTP request to the specified path.
 
         Args:
             path (str): The API endpoint path to make the request to.
@@ -210,7 +220,6 @@ class BaseClient:
             Union[dict, BaseModel]: If response_model is provided, returns an instance of that model.
                 Otherwise, returns the raw response data.
         """
-
         if kwargs.get('data') and request_model:
             kwargs['data'] = self._validate_request_body_with_pydantic_model(kwargs['data'], request_model)
         response = self._request('patch', path, **kwargs)
@@ -220,8 +229,7 @@ class BaseClient:
             return response
 
     def _delete(self, path, request_model=None, response_model=None, **kwargs):
-        """
-        Performs a DELETE request to the specified path.
+        """Performs a DELETE request to the specified path.
 
         Args:
             path (str): The API endpoint path to send the DELETE request to.
@@ -234,7 +242,6 @@ class BaseClient:
             Union[dict, BaseModel]: If response_model is provided, returns an instance of that model.
                                    Otherwise, returns the raw response data as a dictionary.
         """
-
         if kwargs.get('data') and request_model:
             kwargs['data'] = self._validate_request_body_with_pydantic_model(kwargs['data'], request_model)
         response = self._request('delete', path, **kwargs)

synapse_sdk/devtools/docs/docs/api/clients/agent.md

@@ -0,0 +1,43 @@
+---
+id: agent
+title: AgentClient
+sidebar_position: 2
+---
+
+# AgentClient
+
+Client for agent-specific operations and distributed execution.
+
+## Overview
+
+The `AgentClient` provides access to agent operations, including plugin execution, job management, and distributed computing integration.
+
+## Constructor
+
+```python
+AgentClient(
+    base_url: str,
+    agent_token: str = None,
+    timeout: dict = None
+)
+```
+
+## Usage
+
+```python
+from synapse_sdk.clients.agent import AgentClient
+
+client = AgentClient(
+    base_url="https://api.synapse.sh",
+    agent_token="your-agent-token"
+)
+```
+
+## Methods
+
+Coming soon - detailed API documentation for AgentClient methods.
+
+## See Also
+
+- [BackendClient](./backend.md) - For backend operations
+- [BaseClient](./base.md) - Base client implementation

synapse_sdk/devtools/docs/docs/api/clients/backend.md

@@ -0,0 +1,53 @@
+---
+id: backend
+title: BackendClient
+sidebar_position: 1
+---
+
+# BackendClient
+
+Main client for interacting with the Synapse backend API.
+
+## Overview
+
+The `BackendClient` provides comprehensive access to all backend operations including data management, plugin execution, annotations, and machine learning workflows. It aggregates functionality from multiple specialized mixins.
+
+## Constructor
+
+```python
+BackendClient(
+    base_url: str,
+    api_token: str = None,
+    agent_token: str = None,
+    timeout: dict = None
+)
+```
+
+### Parameters
+
+- **base_url** (`str`): The base URL of the Synapse backend API
+- **api_token** (`str`, optional): API authentication token. Can also be set via `SYNAPSE_API_TOKEN` environment variable
+- **agent_token** (`str`, optional): Agent authentication token. Can also be set via `SYNAPSE_AGENT_TOKEN` environment variable
+- **timeout** (`dict`, optional): Custom timeout settings. Defaults to `{'connect': 5, 'read': 30}`
+
+### Example
+
+```python
+from synapse_sdk.clients.backend import BackendClient
+
+# Create client with explicit token
+client = BackendClient(
+    base_url="https://api.synapse.sh",
+    api_token="your-api-token"
+)
+
+# Or use environment variables
+import os
+os.environ['SYNAPSE_API_TOKEN'] = "your-api-token"
+client = BackendClient(base_url="https://api.synapse.sh")
+```
+
+## See Also
+
+- [AgentClient](./agent.md) - For agent-specific operations
+- [BaseClient](./base.md) - Base client implementation

synapse_sdk/devtools/docs/docs/api/clients/base.md

@@ -0,0 +1,35 @@
+---
+id: base
+title: BaseClient
+sidebar_position: 3
+---
+
+# BaseClient
+
+Base class for all Synapse SDK clients.
+
+## Overview
+
+The `BaseClient` provides common functionality for HTTP operations, error handling, and request management used by all other clients.
+
+## Features
+
+- HTTP request handling with retry logic
+- Automatic timeout management
+- File upload/download capabilities
+- Pydantic model validation
+- Connection pooling
+
+## Usage
+
+```python
+from synapse_sdk.clients.base import BaseClient
+
+# BaseClient is typically not used directly
+# Use BackendClient or AgentClient instead
+```
+
+## See Also
+
+- [BackendClient](./backend.md) - Main client implementation
+- [AgentClient](./agent.md) - Agent-specific operations