synapse-sdk 1.0.0a23__py3-none-any.whl → 2025.12.3__py3-none-any.whl

synapse_sdk/clients/backend/data_collection.py

@@ -0,0 +1,186 @@
+from multiprocessing import Pool
+from pathlib import Path
+from typing import Dict, Optional, Union
+
+from tqdm import tqdm
+
+from synapse_sdk.clients.base import BaseClient
+from synapse_sdk.clients.utils import get_batched_list, get_default_url_conversion
+
+
+class DataCollectionClientMixin(BaseClient):
+    """Mixin class for data collection operations.
+
+    Provides methods for managing data collections, files, and units
+    in the Synapse backend. Supports both regular file uploads and
+    chunked uploads for large files.
+
+    This mixin extends BaseClient with data collection-specific functionality
+    including file upload capabilities, data unit management, and batch processing
+    operations for efficient data collection workflows.
+    """
+
+    def list_data_collection(self):
+        path = 'data_collections/'
+        return self._list(path)
+
+    def get_data_collection(self, data_collection_id):
+        """Get data_collection from synapse-backend.
+
+        Args:
+            data_collection_id: The data_collection id to get.
+        """
+        path = f'data_collections/{data_collection_id}/?expand=file_specifications'
+        return self._get(path)
+
+    def create_data_file(
+        self, file_path: Path, use_chunked_upload: bool = False
+    ) -> Union[Dict[str, Union[str, int]], str]:
+        """Create and upload a data file to the Synapse backend.
+
+        This method supports two upload strategies:
+        1. Direct file upload for smaller files (default)
+        2. Chunked upload for large files (automatic when flag is enabled)
+
+        Args:
+            file_path: Path object pointing to the file to upload.
+                Must be a valid file path that exists on the filesystem.
+            use_chunked_upload: Boolean flag to enable chunked upload for the file.
+                When True, automatically creates a chunked upload for the file
+                instead of uploading it directly. Defaults to False.
+
+        Returns:
+            Dictionary containing the created data file information including:
+                - id: The unique identifier of the created data file
+                - checksum: The MD5 checksum of the uploaded file
+                - size: The file size in bytes
+                - created_at: Timestamp of creation
+                - Additional metadata fields from the backend
+            Or a string response in case of non-JSON response.
+
+        Raises:
+            FileNotFoundError: If the specified file doesn't exist (for direct upload)
+            PermissionError: If the file can't be read due to permissions
+            ClientError: If the backend returns an error response
+            ValueError: If file_path is not a valid Path object
+
+        Examples:
+            Direct file upload for small files:
+            ```python
+            client = DataCollectionClientMixin(base_url='https://api.example.com')
+            file_path = Path('/path/to/small_file.csv')
+            result = client.create_data_file(file_path)
+            print(f"File uploaded with ID: {result['id']}")
+            ```
+
+            Using chunked upload for large files:
+            ```python
+            # Automatically create chunked upload for large file
+            file_path = Path('/path/to/large_file.zip')
+            result = client.create_data_file(file_path, use_chunked_upload=True)
+            print(f"Large file uploaded with ID: {result['id']}")
+            ```
+
+        Note:
+            - For files larger than 100MB, consider using chunked upload
+            - The chunked upload will be automatically created when the flag is enabled
+            - Chunked uploads provide better reliability for large files
+        """
+        path = 'data_files/'
+        if use_chunked_upload:
+            chunked_upload = self.create_chunked_upload(file_path)
+            data = {'chunked_upload': chunked_upload['id'], 'meta': {'filename': file_path.name}}
+            return self._post(path, data=data)
+        else:
+            return self._post(path, files={'file': file_path})
+
+    def get_data_unit(self, data_unit_id: int, params=None):
+        path = f'data_units/{data_unit_id}/'
+        return self._get(path, params=params)
+
+    def create_data_units(self, data):
+        """Create data units to synapse-backend.
+
+        Args:
+            data: The data bindings to upload from create_data_file interface.
+        """
+        path = 'data_units/'
+        return self._post(path, data=data)
+
+    def list_data_units(self, params=None, url_conversion=None, list_all=False):
+        path = 'data_units/'
+        url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
+        return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
+
+    def data_files_verify_checksums(self, checksums: list[str]):
+        """Check checksums from files are exists.
+
+        Args:
+            checksums: A list of MD5 checksums to verify.
+        """
+        path = 'data_files/verify_checksums/'
+        data = {'checksums': checksums}
+        return self._post(path, data=data)
+
+    def upload_data_collection(
+        self,
+        data_collection_id: int,
+        data_collection: Dict,
+        project_id: Optional[int] = None,
+        batch_size: int = 1000,
+        process_pool: int = 10,
+    ):
+        """Upload data_collection to synapse-backend.
+
+        Args:
+            data_collection_id: The data_collection id to upload the data to.
+            data_collection: The data_collection to upload.
+                * structure:
+                    - files: The files to upload. (key: file name, value: file pathlib object)
+                    - meta: The meta data to upload.
+            project_id: The project id to upload the data to.
+            batch_size: The batch size to upload the data.
+            process_pool: The process pool to upload the data.
+        """
+        # TODO validate data_collection with schema
+
+        params = [(data, data_collection_id) for data in data_collection]
+
+        with Pool(processes=process_pool) as pool:
+            data_collection = pool.starmap(self.upload_data_file, tqdm(params))
+
+        batches = get_batched_list(data_collection, batch_size)
+
+        for batch in tqdm(batches):
+            data_units = self.create_data_units(batch)
+
+            if project_id:
+                tasks_data = []
+                for data, data_unit in zip(batch, data_units):
+                    task_data = {'project': project_id, 'data_unit': data_unit['id']}
+                    # TODO: Additional logic needed here if task data storage is required during import.
+
+                    tasks_data.append(task_data)
+
+                self.create_tasks(tasks_data)
+
+    def upload_data_file(self, data: Dict, data_collection_id: int, use_chunked_upload: bool = False) -> Dict:
+        """Upload files to synapse-backend.
+
+        Args:
+            data: The data to upload.
+                * structure:
+                    - files: The files to upload. (key: file name, value: file pathlib object)
+                    - meta: The meta data to upload.
+            data_collection_id: The data_collection id to upload the data to.
+            use_chunked_upload: Whether to use chunked upload for large files.(default False)
+                Automatically determined based on file size threshold in upload plugin (default 50MB).
+
+        Returns:
+            Dict: The result of the upload.
+        """
+        for name, path in data['files'].items():
+            data_file = self.create_data_file(path, use_chunked_upload)
+            data['data_collection'] = data_collection_id
+            data['files'][name] = {'checksum': data_file['checksum'], 'path': str(path)}
+        return data

synapse_sdk/clients/backend/hitl.py

@@ -0,0 +1,17 @@
+from synapse_sdk.clients.base import BaseClient
+from synapse_sdk.clients.utils import get_default_url_conversion
+
+
+class HITLClientMixin(BaseClient):
+    def get_assignment(self, pk):
+        path = f'assignments/{pk}/'
+        return self._get(path)
+
+    def list_assignments(self, params=None, url_conversion=None, list_all=False):
+        path = 'sdk/assignments/'
+        url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
+        return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
+
+    def set_tags_assignments(self, data, params=None):
+        path = 'assignments/set_tags/'
+        return self._post(path, payload=data, params=params)

synapse_sdk/clients/backend/integration.py

@@ -1,4 +1,6 @@
+from synapse_sdk.clients.backend.models import Storage, UpdateJob
 from synapse_sdk.clients.base import BaseClient
+from synapse_sdk.utils.file import convert_file_to_base64
 
 
 class IntegrationClientMixin(BaseClient):
@@ -41,7 +43,7 @@ class IntegrationClientMixin(BaseClient):
 
     def update_job(self, pk, data):
         path = f'jobs/{pk}/'
-        return self._patch(path, data=data)
+        return self._patch(path, request_model=UpdateJob, data=data)
 
     def list_job_console_logs(self, pk):
         path = f'jobs/{pk}/console_logs/'
@@ -60,6 +62,13 @@ class IntegrationClientMixin(BaseClient):
 
     def create_logs(self, data):
         path = 'logs/'
+        if not isinstance(data, list):
+            data = [data]
+
+        for item in data:
+            if 'file' in item:
+                item['file'] = convert_file_to_base64(item['file'])
+
         return self._post(path, data=data)
 
     def create_serve_application(self, data):
@@ -69,3 +78,9 @@ class IntegrationClientMixin(BaseClient):
     def list_serve_applications(self, params=None, list_all=False):
         path = 'serve_applications/'
         return self._list(path, params=params, list_all=list_all)
+
+    def get_storage(self, pk):
+        """Get specific storage data from synapse backend."""
+        path = f'storages/{pk}/'
+        params = {'with_configuration': True}
+        return self._get(path, params=params, response_model=Storage)

synapse_sdk/clients/backend/ml.py

@@ -19,6 +19,10 @@ class MLClientMixin(BaseClient):
         return self._post(path, data=data)
 
     def list_ground_truth_events(self, params=None, url_conversion=None, list_all=False):
-        path = 'ground_truth_events/'
+        path = 'sdk/ground_truth_events/'
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
         return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
+
+    def get_ground_truth_version(self, pk):
+        path = f'ground_truth_dataset_versions/{pk}/'
+        return self._get(path)

synapse_sdk/clients/backend/models.py

@@ -0,0 +1,78 @@
+from enum import Enum
+from typing import Dict, Optional
+
+from pydantic import BaseModel
+
+
+class StorageCategory(str, Enum):
+    """Synapse Backend Storage Category Enum."""
+
+    INTERNAL = 'internal'
+    EXTERNAL = 'external'
+
+
+class StorageProvider(str, Enum):
+    """Synapse Backend Storage Provider Enum."""
+
+    AMAZON_S3 = 'amazon_s3'
+    AZURE = 'azure'
+    DIGITAL_OCEAN = 'digital_ocean'
+    FILE_SYSTEM = 'file_system'
+    FTP = 'ftp'
+    SFTP = 'sftp'
+    MINIO = 'minio'
+    GCP = 'gcp'
+
+
+class Storage(BaseModel):
+    """Synapse Backend Storage Model.
+
+    Attrs:
+        id (int): The storage pk.
+        name (str): The storage name.
+        category (str): The storage category. (ex: internal, external)
+        provider (str): The storage provider. (ex: s3, gcp)
+        configuration (Dict): The storage configuration.
+        is_default (bool): The storage is default for Synapse backend workspace.
+    """
+
+    id: int
+    name: str
+    category: StorageCategory
+    provider: StorageProvider
+    configuration: Dict
+    is_default: bool
+
+
+class JobStatus(str, Enum):
+    """Synapse Backend Job Status Enum."""
+
+    PENDING = 'pending'
+    RUNNING = 'running'
+    STOPPED = 'stopped'
+    SUCCEEDED = 'succeeded'
+    FAILED = 'failed'
+
+
+class UpdateJob(BaseModel):
+    """Synapse Backend Update Job Request Payload Model.
+
+    Attrs:
+        status (str): The job status. (ex: pending, running, stopped, succeeded, failed)
+        progress_record (Dict): The job progress record.
+        metrics_record (Dict): The job metrics record.
+        console_logs (Dict): The job console logs.
+        result (Dict): The job result.
+    """
+
+    status: Optional[JobStatus] = None
+    progress_record: Optional[Dict] = None
+    metrics_record: Optional[Dict] = None
+    console_logs: Optional[Dict | list] = None
+    result: Optional[Dict | list] = None
+
+    def model_dump(self, **kwargs):
+        data = super().model_dump(**kwargs)
+        if data.get('status') is not None:
+            data['status'] = data['status'].value
+        return data