synapse-sdk 1.0.0b12__tar.gz → 1.0.0b14__tar.gz

synapse_sdk-1.0.0b14/.github/workflows/pypi-publish.yml

@@ -0,0 +1,102 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Upload Python Package
+
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    name: Build distribution 📦
+    runs-on: runner-set
+    steps:
+      - uses: actions/checkout@v5
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
+      - name: Compile i18n messages
+        run: |
+          msgfmt locale/ko/LC_MESSAGES/messages.po -o locale/ko/LC_MESSAGES/messages.mo
+          msgfmt locale/en/LC_MESSAGES/messages.po -o locale/en/LC_MESSAGES/messages.mo
+      - name: Build package
+        run: python -m build
+
+      - name: Install AWS CLI
+        run: pip install awscli
+
+      - name: Store the distribution packages to MinIO ☁️
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.ARTIFACTS_S3_ACCESS_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.ARTIFACTS_S3_SECRET_KEY }}
+        run: |
+          # Create tarball of distribution packages
+          tar -czf dist.tar.gz -C dist .
+
+          # Upload to MinIO using AWS CLI
+          BUCKET_NAME="${{ vars.ARTIFACTS_S3_BUCKET_NAME }}"
+          OBJECT_KEY="dist.tar.gz"
+          
+          aws s3 cp dist.tar.gz s3://$BUCKET_NAME/$OBJECT_KEY --endpoint-url=https://datamaker-kr.cdn.datamaker.io
+
+          echo "Artifacts uploaded to: s3://$BUCKET_NAME/$OBJECT_KEY"
+          rm -f dist.tar.gz
+
+  publish-to-pypi:
+    name: Publish Python 🐍 distribution 📦 to PyPI
+    needs:
+      - build
+    environment: release
+    runs-on: runner-set
+    permissions:
+      # IMPORTANT: this permission is mandatory for trusted publishing
+      id-token: write
+    steps:
+      - name: Install AWS CLI
+        run: pip install awscli
+
+      - name: Retrieve distribution packages from MinIO ⬇️
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.ARTIFACTS_S3_ACCESS_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.ARTIFACTS_S3_SECRET_KEY }}
+        run: |
+          # Download from MinIO using AWS CLI
+          BUCKET_NAME="${{ vars.ARTIFACTS_S3_BUCKET_NAME }}"
+          OBJECT_KEY="dist.tar.gz"
+          
+          aws s3 cp s3://$BUCKET_NAME/$OBJECT_KEY dist.tar.gz --endpoint-url=https://datamaker-kr.cdn.datamaker.io
+
+          # Extract artifacts
+          mkdir -p dist/
+          tar -xzf dist.tar.gz -C dist/
+          rm -f dist.tar.gz
+
+          echo "Artifacts retrieved from: s3://$BUCKET_NAME/$OBJECT_KEY"
+
+      - name: Publish distribution 📦 to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+      - name: Cleanup artifacts 🧹
+        if: always()
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.ARTIFACTS_S3_ACCESS_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.ARTIFACTS_S3_SECRET_KEY }}
+        run: |
+          # Delete from MinIO using AWS CLI
+          BUCKET_NAME="${{ vars.ARTIFACTS_S3_BUCKET_NAME }}"
+          OBJECT_KEY="dist.tar.gz"
+          
+          aws s3 rm s3://$BUCKET_NAME/$OBJECT_KEY --endpoint-url=https://datamaker-kr.cdn.datamaker.io
+
+          echo "Artifacts cleaned up from: s3://$BUCKET_NAME/$OBJECT_KEY"

{synapse_sdk-1.0.0b12 → synapse_sdk-1.0.0b14}/AGENT.md

@@ -239,6 +239,24 @@ npm run typecheck
 - Add code examples in appropriate language blocks
 - Include cross-references using relative paths
 
+### Multi-Language Documentation Requirements
+
+**All documentation must be maintained in both English and Korean:**
+
+1. **Primary Language**: English is the primary language for all documentation
+2. **Required Translation**: Every new document must have a corresponding Korean translation
+3. **File Structure**: 
+   - English documents: `docs/[file].md`
+   - Korean documents: `docs/i18n/ko/docusaurus-plugin-content-docs/current/[file].md`
+4. **Content Consistency**: Korean translations must maintain the same structure, sections, and information as English versions
+5. **Technical Terms**: Keep technical terms like "Synapse SDK", "API", "CLI", "GitHub" in English within Korean documents
+6. **Code Examples**: All code examples, commands, and file paths must remain unchanged in Korean versions
+7. **Update Process**: When updating documentation:
+   - Update the English version first
+   - Update the corresponding Korean version immediately after
+   - Ensure both versions maintain content parity
+8. **Review Requirement**: Both English and Korean versions must be reviewed for accuracy and consistency
+
 ## 🔧 Code Formatting with Ruff
 
 Claude Code should format all Python code changes using **Ruff** to maintain consistent code style across the project.

{synapse_sdk-1.0.0b12 → synapse_sdk-1.0.0b14}/Makefile

@@ -59,6 +59,30 @@ run-docs:
 	cd synapse_sdk/devtools/docs && \
 		npx docusaurus start --host 0.0.0.0 --port 3330
 
+run-docs-ko:
+	cd synapse_sdk/devtools/docs && \
+		npx docusaurus start --locale ko --host 0.0.0.0 --port 3330
+
 init-docs:
 	cd synapse_sdk/devtools/docs && \
-		npm install	
+		npm install	
+	
+DOCS_PORT := 4444
+KILL_PORT_CMD := kill $$(lsof -t -i:$(DOCS_PORT)) 2>/dev/null || kill -9 $$(lsof -t -i:$(DOCS_PORT)) 2>/dev/null || true
+
+# Build Docusaurus Documentation
+build-docs:
+	cd synapse_sdk/devtools/docs && \
+		npm run docusaurus clear && \
+		npm run docusaurus build
+
+# Kill Docusaurus Documentation Server
+kill-docs:
+	$(KILL_PORT_CMD)
+
+# Serve Docusaurus Documentation
+serve-docs: 
+	cd synapse_sdk/devtools/docs && npx serve -s build -l $(DOCS_PORT) > /dev/null 2>&1 &
+
+# Build and Serve Docusaurus Documentation
+build-serve-docs: kill-docs build-docs serve-docs

{synapse_sdk-1.0.0b12 → synapse_sdk-1.0.0b14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0b12
+Version: 1.0.0b14
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0b12 → synapse_sdk-1.0.0b14}/synapse_sdk/clients/agent/ray.py

@@ -15,8 +15,7 @@ from synapse_sdk.utils.network import (
 
 
 class RayClientMixin(BaseClient):
-    """
-    Mixin class providing Ray cluster management and monitoring functionality.
+    """Mixin class providing Ray cluster management and monitoring functionality.
 
     This mixin extends BaseClient with Ray-specific operations for interacting with
     Apache Ray distributed computing clusters. It provides comprehensive job management,
@@ -95,8 +94,7 @@ class RayClientMixin(BaseClient):
         return self._get(path)
 
     def websocket_tail_job_logs(self, pk, stream_timeout=10):
-        """
-        Stream job logs in real-time using WebSocket protocol.
+        """Stream job logs in real-time using WebSocket protocol.
 
         Establishes a WebSocket connection to stream job logs as they are generated.
         This method provides the lowest latency for real-time log monitoring and is
@@ -153,18 +151,18 @@ class RayClientMixin(BaseClient):
 
         # Build WebSocket URL
         path = f'ray/jobs/{validated_pk}/logs/ws/'
-        url = self._get_url(path)
+        url = self._get_url(path, trailing_slash=True)
         ws_url = http_to_websocket_url(url)
 
         # Get headers and use WebSocket manager
         headers = self._get_headers()
+        headers['Agent-Token'] = f'Token {self.agent_token}'
         context = f'job {validated_pk}'
 
         return self._websocket_manager.stream_logs(ws_url, headers, validated_timeout, context)
 
     def stream_tail_job_logs(self, pk, stream_timeout=10):
-        """
-        Stream job logs in real-time using HTTP chunked transfer encoding.
+        """Stream job logs in real-time using HTTP chunked transfer encoding.
 
         Establishes an HTTP connection with chunked transfer encoding to stream
         job logs as they are generated. This method serves as a reliable fallback
@@ -225,17 +223,17 @@ class RayClientMixin(BaseClient):
         validated_timeout = validate_timeout(stream_timeout)
 
         # Build HTTP URL and prepare request
-        path = f'ray/jobs/{validated_pk}/logs/stream'
-        url = self._get_url(path)
+        path = f'ray/jobs/{validated_pk}/logs/stream/'
+        url = self._get_url(path, trailing_slash=True)
         headers = self._get_headers()
+        headers['Agent-Token'] = f'Token {self.agent_token}'
         timeout = (self.timeout['connect'], validated_timeout)
         context = f'job {validated_pk}'
 
         return self._http_manager.stream_logs(url, headers, timeout, context)
 
     def tail_job_logs(self, pk, stream_timeout=10, protocol='stream'):
-        """
-        Tail job logs using either WebSocket or HTTP streaming.
+        """Tail job logs using either WebSocket or HTTP streaming.
 
         Args:
             pk: Job primary key

{synapse_sdk-1.0.0b12 → synapse_sdk-1.0.0b14}/synapse_sdk/clients/backend/annotation.py

@@ -24,7 +24,7 @@ class AnnotationClientMixin(BaseClient):
         return self._list(path, params=params)
 
     def list_tasks(self, params=None, url_conversion=None, list_all=False):
-        path = 'tasks/'
+        path = 'sdk/tasks/'
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
         return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
 

synapse_sdk-1.0.0b14/synapse_sdk/clients/backend/core.py

@@ -0,0 +1,59 @@
+import hashlib
+import os
+from pathlib import Path
+
+from synapse_sdk.clients.base import BaseClient
+from synapse_sdk.utils.file import read_file_in_chunks
+
+
+class CoreClientMixin(BaseClient):
+    def create_chunked_upload(self, file_path):
+        """
+        Upload a file using chunked upload for efficient handling of large files.
+
+        This method breaks the file into chunks and uploads them sequentially to the server.
+        It calculates an MD5 hash of the entire file to ensure data integrity during upload.
+
+        Args:
+            file_path (str | Path): Path to the file to upload
+
+        Returns:
+            dict: Response from the server after successful upload completion,
+                  typically containing upload confirmation and file metadata
+
+        Raises:
+            FileNotFoundError: If the specified file doesn't exist
+            PermissionError: If the file can't be read due to permissions
+            ClientError: If there's an error during the upload process
+            OSError: If there's an OS-level error accessing the file
+
+        Example:
+            ```python
+            client = CoreClientMixin(base_url='https://api.example.com')
+            result = client.create_chunked_upload('/path/to/large_file.zip')
+            print(f"Upload completed: {result}")
+            ```
+
+        Note:
+            - Uses 50MB chunks by default for optimal upload performance
+            - Automatically resumes from the last successfully uploaded chunk
+            - Verifies upload integrity using MD5 checksum
+        """
+        file_path = Path(file_path)
+        size = os.path.getsize(file_path)
+        hash_md5 = hashlib.md5()
+
+        url = 'chunked_upload/'
+        offset = 0
+        for chunk in read_file_in_chunks(file_path):
+            hash_md5.update(chunk)
+            data = self._put(
+                url,
+                data={'filename': file_path.name},
+                files={'file': chunk},
+                headers={'Content-Range': f'bytes {offset}-{offset + len(chunk) - 1}/{size}'},
+            )
+            offset = data['offset']
+            url = data['url']
+
+        return self._post(url, data={'md5': hash_md5.hexdigest()})

{synapse_sdk-1.0.0b12 → synapse_sdk-1.0.0b14}/synapse_sdk/clients/backend/data_collection.py

@@ -1,6 +1,6 @@
 from multiprocessing import Pool
 from pathlib import Path
-from typing import Dict, Optional
+from typing import Dict, Optional, Union
 
 from tqdm import tqdm
 
@@ -9,6 +9,17 @@ from synapse_sdk.clients.utils import get_batched_list, get_default_url_conversi
 
 
 class DataCollectionClientMixin(BaseClient):
+    """Mixin class for data collection operations.
+
+    Provides methods for managing data collections, files, and units
+    in the Synapse backend. Supports both regular file uploads and
+    chunked uploads for large files.
+
+    This mixin extends BaseClient with data collection-specific functionality
+    including file upload capabilities, data unit management, and batch processing
+    operations for efficient data collection workflows.
+    """
+
     def list_data_collection(self):
         path = 'data_collections/'
         return self._list(path)
@@ -22,14 +33,66 @@ class DataCollectionClientMixin(BaseClient):
         path = f'data_collections/{data_collection_id}/?expand=file_specifications'
         return self._get(path)
 
-    def create_data_file(self, file_path: Path):
-        """Create data file to synapse-backend.
+    def create_data_file(
+        self, file_path: Path, use_chunked_upload: bool = False
+    ) -> Union[Dict[str, Union[str, int]], str]:
+        """Create and upload a data file to the Synapse backend.
+
+        This method supports two upload strategies:
+        1. Direct file upload for smaller files (default)
+        2. Chunked upload for large files (automatic when flag is enabled)
 
         Args:
-            file_path: The file pathlib object to upload.
+            file_path: Path object pointing to the file to upload.
+                Must be a valid file path that exists on the filesystem.
+            use_chunked_upload: Boolean flag to enable chunked upload for the file.
+                When True, automatically creates a chunked upload for the file
+                instead of uploading it directly. Defaults to False.
+
+        Returns:
+            Dictionary containing the created data file information including:
+                - id: The unique identifier of the created data file
+                - checksum: The MD5 checksum of the uploaded file
+                - size: The file size in bytes
+                - created_at: Timestamp of creation
+                - Additional metadata fields from the backend
+            Or a string response in case of non-JSON response.
+
+        Raises:
+            FileNotFoundError: If the specified file doesn't exist (for direct upload)
+            PermissionError: If the file can't be read due to permissions
+            ClientError: If the backend returns an error response
+            ValueError: If file_path is not a valid Path object
+
+        Examples:
+            Direct file upload for small files:
+            ```python
+            client = DataCollectionClientMixin(base_url='https://api.example.com')
+            file_path = Path('/path/to/small_file.csv')
+            result = client.create_data_file(file_path)
+            print(f"File uploaded with ID: {result['id']}")
+            ```
+
+            Using chunked upload for large files:
+            ```python
+            # Automatically create chunked upload for large file
+            file_path = Path('/path/to/large_file.zip')
+            result = client.create_data_file(file_path, use_chunked_upload=True)
+            print(f"Large file uploaded with ID: {result['id']}")
+            ```
+
+        Note:
+            - For files larger than 100MB, consider using chunked upload
+            - The chunked upload will be automatically created when the flag is enabled
+            - Chunked uploads provide better reliability for large files
         """
         path = 'data_files/'
-        return self._post(path, files={'file': file_path})
+        if use_chunked_upload:
+            chunked_upload = self.create_chunked_upload(file_path)
+            data = {'chunked_upload': chunked_upload['id']}
+            return self._post(path, data=data)
+        else:
+            return self._post(path, files={'file': file_path})
 
     def get_data_unit(self, data_unit_id: int, params=None):
         path = f'data_units/{data_unit_id}/'
@@ -49,6 +112,16 @@ class DataCollectionClientMixin(BaseClient):
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
         return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
 
+    def data_file_verify_checksums(self, checksums: list[str]):
+        """Check checksums from files are exists.
+
+        Args:
+            checksums: A list of MD5 checksums to verify.
+        """
+        path = 'data_file/verify_checksums/'
+        data = {'checksums': checksums}
+        return self._post(path, data=data)
+
     def upload_data_collection(
         self,
         data_collection_id: int,

{synapse_sdk-1.0.0b12 → synapse_sdk-1.0.0b14}/synapse_sdk/clients/backend/hitl.py

@@ -8,7 +8,7 @@ class HITLClientMixin(BaseClient):
         return self._get(path)
 
     def list_assignments(self, params=None, url_conversion=None, list_all=False):
-        path = 'assignments/'
+        path = 'sdk/assignments/'
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
         return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
 

{synapse_sdk-1.0.0b12 → synapse_sdk-1.0.0b14}/synapse_sdk/clients/backend/ml.py

@@ -19,7 +19,7 @@ class MLClientMixin(BaseClient):
         return self._post(path, data=data)
 
     def list_ground_truth_events(self, params=None, url_conversion=None, list_all=False):
-        path = 'ground_truth_events/'
+        path = 'sdk/ground_truth_events/'
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
         return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
 

{synapse_sdk-1.0.0b12 → synapse_sdk-1.0.0b14}/synapse_sdk/clients/base.py

@@ -39,10 +39,24 @@ class BaseClient:
 
         self.requests_session = requests_session
 
-    def _get_url(self, path):
-        if not path.startswith('http'):
-            return f'{self.base_url}/{path.lstrip("/")}'
-        return path
+    def _get_url(self, path, trailing_slash=False):
+        """Construct a full URL from a path.
+
+        Args:
+            path (str): URL path or full URL
+            trailing_slash (bool): Whether to ensure URL ends with trailing slash
+
+        Returns:
+            str: Complete URL
+        """
+        # Use the path as-is if it's already a full URL, otherwise construct from base_url and path
+        url = path if path.startswith(('http://', 'https://')) else f'{self.base_url}/{path.lstrip("/")}'
+
+        # Add trailing slash if requested and not present
+        if trailing_slash and not url.endswith('/'):
+            url += '/'
+
+        return url
 
     def _get_headers(self):
         return {}
@@ -125,8 +139,7 @@ class BaseClient:
             return response.text
 
     def _get(self, path, url_conversion=None, response_model=None, **kwargs):
-        """
-        Perform a GET request and optionally convert response to a pydantic model.
+        """Perform a GET request and optionally convert response to a pydantic model.
 
         Args:
             path (str): URL path to request.
@@ -152,8 +165,7 @@ class BaseClient:
         return response
 
     def _post(self, path, request_model=None, response_model=None, **kwargs):
-        """
-        Perform a POST request and optionally convert response to a pydantic model.
+        """Perform a POST request and optionally convert response to a pydantic model.
 
         Args:
             path (str): URL path to request.
@@ -173,8 +185,7 @@ class BaseClient:
             return response
 
     def _put(self, path, request_model=None, response_model=None, **kwargs):
-        """
-        Perform a PUT request to the specified path.
+        """Perform a PUT request to the specified path.
 
         Args:
             path (str): The URL path for the request.
@@ -196,8 +207,7 @@ class BaseClient:
             return response
 
     def _patch(self, path, request_model=None, response_model=None, **kwargs):
-        """
-        Perform a PATCH HTTP request to the specified path.
+        """Perform a PATCH HTTP request to the specified path.
 
         Args:
             path (str): The API endpoint path to make the request to.
@@ -210,7 +220,6 @@ class BaseClient:
             Union[dict, BaseModel]: If response_model is provided, returns an instance of that model.
                 Otherwise, returns the raw response data.
         """
-
         if kwargs.get('data') and request_model:
             kwargs['data'] = self._validate_request_body_with_pydantic_model(kwargs['data'], request_model)
         response = self._request('patch', path, **kwargs)
@@ -220,8 +229,7 @@ class BaseClient:
             return response
 
     def _delete(self, path, request_model=None, response_model=None, **kwargs):
-        """
-        Performs a DELETE request to the specified path.
+        """Performs a DELETE request to the specified path.
 
         Args:
             path (str): The API endpoint path to send the DELETE request to.
@@ -234,7 +242,6 @@ class BaseClient:
             Union[dict, BaseModel]: If response_model is provided, returns an instance of that model.
                                    Otherwise, returns the raw response data as a dictionary.
         """
-
         if kwargs.get('data') and request_model:
             kwargs['data'] = self._validate_request_body_with_pydantic_model(kwargs['data'], request_model)
         response = self._request('delete', path, **kwargs)