synapse-sdk 2025.10.3__py3-none-any.whl → 2025.10.5__py3-none-any.whl

synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/upload-plugins/upload-plugin-overview.md

@@ -97,6 +97,31 @@ sidebar_position: 1
 }
 ```
 
+**선택적 파일 사양:**
+
+다중 경로 모드에서는 데이터 컬렉션의 파일 사양 템플릿에서 파일 사양을 선택적으로 표시할 수 있습니다:
+
+- **필수 사양** (`is_required: true`): `assets` 매개변수에 자산 경로가 반드시 있어야 합니다
+- **선택적 사양** (`is_required: false`): `assets`에서 생략 가능 - 시스템이 건너뜁니다
+
+선택적 사양이 생략된 예제:
+
+```json
+{
+  "name": "다중 소스 업로드",
+  "use_single_path": false,
+  "assets": {
+    "pcd_1": {"path": "/sensors/lidar", "is_recursive": false},
+    "image_1": {"path": "/cameras/front", "is_recursive": true}
+    // "json_meta_1"은 선택사항이며 생략됨
+  },
+  "storage": 1,
+  "data_collection": 5
+}
+```
+
+시스템 로그: `"Skipping optional spec json_meta_1: no asset path configured"`
+
 ## 기본 사용법
 
 ### CLI 사용법

synapse_sdk/devtools/docs/sidebars.ts

@@ -54,6 +54,20 @@ const sidebars: SidebarsConfig = {
             'plugins/categories/upload-plugins/upload-plugin-template',
           ],
         },
+        {
+          type: 'category',
+          label: 'Pre-annotation Plugins',
+          link: {
+            type: 'doc',
+            id: 'plugins/categories/pre-annotation-plugins/pre-annotation-plugin-overview',
+          },
+          items: [
+            'plugins/categories/pre-annotation-plugins/pre-annotation-plugin-overview',
+            'plugins/categories/pre-annotation-plugins/to-task-overview',
+            'plugins/categories/pre-annotation-plugins/to-task-action-development',
+            'plugins/categories/pre-annotation-plugins/to-task-template-development',
+          ],
+        },
       ],
     },
     {

synapse_sdk/plugins/categories/export/actions/export/action.py

@@ -95,11 +95,11 @@ class ExportAction(Action):
             PydanticCustomError: If data retrieval fails
         """
         try:
-            result_list = handler.get_results(self.client, filters)
+            result_list = handler.get_results(self.client, filters, run=self.run)
             results = result_list[0]
             count = result_list[1]
         except ClientError:
-            raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset.'))
+            raise PydanticCustomError('client_error', _('Unable to get dataset.'))
         return results, count
 
     def start(self) -> Dict[str, Any]:
@@ -116,7 +116,12 @@ class ExportAction(Action):
         """
         self.run.log_message_with_code(LogCode.EXPORT_STARTED)
 
-        filters = {'expand': 'data', **self.params['filter']}
+        # Get expand setting from config, default to True (expand data)
+        filters = {**self.params['filter']}
+        data_expand = self.config.get('data_expand', True)
+        if data_expand:
+            filters['expand'] = 'data'
+
         target = self.params['target']
         handler = TargetHandlerFactory.get_handler(target)
 

synapse_sdk/plugins/categories/export/actions/export/utils.py

@@ -1,10 +1,12 @@
 from abc import ABC, abstractmethod
-from typing import Any
+from typing import Any, Optional
+import time
 
 from pydantic_core import PydanticCustomError
 
 from synapse_sdk.clients.exceptions import ClientError
 from synapse_sdk.i18n import gettext as _
+from synapse_sdk.shared.enums import Context
 
 
 class ExportTargetHandler(ABC):
@@ -15,6 +17,103 @@ class ExportTargetHandler(ABC):
     of methods to validate filters, retrieve results, and process collections of results.
     """
 
+    # TODO: This is a temporary workaround and needs improvement in the future
+    def _get_results_chunked(self, list_method, filters, chunk_size=100, max_retries=3, retry_delay=1, run=None):
+        """
+        Retrieve results in chunks to avoid memory and response size limits.
+
+        Args:
+            list_method: The client method to call (e.g., client.list_assignments)
+            filters (dict): The filter criteria to apply
+            chunk_size (int): Number of items to fetch per chunk
+            max_retries (int): Maximum number of retries for failed requests
+            retry_delay (int): Delay in seconds between retries
+
+        Returns:
+            tuple: A tuple containing the results generator and the total count
+        """
+        filters = filters.copy()
+        filters['page_size'] = chunk_size
+
+        page = 1
+        results = []
+        total_count = 0
+
+        try:
+            while True:
+                filters['page'] = page
+
+                # Retry logic for handling temporary server issues
+                for attempt in range(max_retries + 1):
+                    try:
+                        response = list_method(params=filters, list_all=False)
+                        break
+                    except ClientError as e:
+                        error_msg = str(e)
+
+                        # Use log_dev_event for better debugging and monitoring
+                        if run:
+                            run.log_dev_event(
+                                'Chunked data retrieval error',
+                                {
+                                    'page': page,
+                                    'attempt': attempt + 1,
+                                    'error_message': error_msg,
+                                    'chunk_size': chunk_size,
+                                },
+                                level=Context.WARNING,
+                            )
+
+                        # Check for JSON decode errors specifically
+                        if 'Expecting value' in error_msg or 'JSONDecodeError' in error_msg:
+                            if run:
+                                run.log_dev_event(
+                                    'JSON parsing error - skipping page',
+                                    {'page': page, 'error_type': 'JSON_DECODE_ERROR', 'error_details': error_msg},
+                                    level=Context.DANGER,
+                                )
+                            # Skip this page and continue with next
+                            page += 1
+                            break
+                        elif attempt < max_retries and ('503' in error_msg or 'connection' in error_msg.lower()):
+                            retry_delay_seconds = retry_delay * (2**attempt)
+                            if run:
+                                run.log_dev_event(
+                                    'Server issue - retrying with backoff',
+                                    {
+                                        'page': page,
+                                        'retry_attempt': attempt + 1,
+                                        'max_retries': max_retries,
+                                        'retry_delay_seconds': retry_delay_seconds,
+                                        'error_type': 'SERVER_ISSUE',
+                                    },
+                                    level=Context.INFO,
+                                )
+                            time.sleep(retry_delay_seconds)  # Exponential backoff
+                            continue
+                        else:
+                            raise
+
+                if page == 1:
+                    total_count = response['count']
+
+                current_results = response.get('results', [])
+                results.extend(current_results)
+
+                # Check if we've got all results or if there are no more results
+                if len(current_results) < chunk_size or not response.get('next'):
+                    break
+
+                page += 1
+
+                # Small delay between pages to avoid overwhelming the server
+                time.sleep(0.1)
+
+            return results, total_count
+        except Exception:
+            # Re-raise the exception to be handled by the calling method
+            raise
+
     @abstractmethod
     def validate_filter(self, value: dict, client: Any):
         """
@@ -33,13 +132,14 @@ class ExportTargetHandler(ABC):
         pass
 
     @abstractmethod
-    def get_results(self, client: Any, filters: dict):
+    def get_results(self, client: Any, filters: dict, run=None):
         """
         Retrieve original data from target sources.
 
         Args:
             client (Any): The client used to retrieve the results.
             filters (dict): The filter criteria to apply.
+            run: Optional ExportRun instance for logging.
 
         Returns:
             tuple: A tuple containing the results and the total count of results.
@@ -76,8 +176,8 @@ class AssignmentExportTargetHandler(ExportTargetHandler):
             raise PydanticCustomError('client_error', _('Unable to get Assignment.'))
         return value
 
-    def get_results(self, client: Any, filters: dict):
-        return client.list_assignments(params=filters, list_all=True)
+    def get_results(self, client: Any, filters: dict, run=None):
+        return self._get_results_chunked(client.list_assignments, filters, run=run)
 
     def get_export_item(self, results):
         for result in results:
@@ -104,9 +204,9 @@ class GroundTruthExportTargetHandler(ExportTargetHandler):
             raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset version.'))
         return value
 
-    def get_results(self, client: Any, filters: dict):
+    def get_results(self, client: Any, filters: dict, run=None):
         filters['ground_truth_dataset_versions'] = filters.pop('ground_truth_dataset_version')
-        return client.list_ground_truth_events(params=filters, list_all=True)
+        return self._get_results_chunked(client.list_ground_truth_events, filters, run=run)
 
     def get_export_item(self, results):
         for result in results:
@@ -134,9 +234,9 @@ class TaskExportTargetHandler(ExportTargetHandler):
             raise PydanticCustomError('client_error', _('Unable to get Task.'))
         return value
 
-    def get_results(self, client: Any, filters: dict):
+    def get_results(self, client: Any, filters: dict, run=None):
         filters['expand'] = ['data_unit', 'assignment', 'workshop']
-        return client.list_tasks(params=filters, list_all=True)
+        return self._get_results_chunked(client.list_tasks, filters, run=run)
 
     def get_export_item(self, results):
         for result in results:

synapse_sdk/plugins/categories/upload/actions/upload/action.py

@@ -173,11 +173,18 @@ class UploadAction(Action):
         organized_files = context.get('organized_files', [])
         file_specification_template = context.get('file_specification_template', {})
         pathlib_cwd = context.get('pathlib_cwd')
+        use_single_path = context.get_param('use_single_path', True)
 
-        if not organized_files or not file_specification_template or not pathlib_cwd:
+        # Validate required data based on mode
+        if not organized_files or not file_specification_template:
             raise ActionError('Required data not available from workflow steps')
 
+        # In single-path mode, pathlib_cwd is required
+        if use_single_path and not pathlib_cwd:
+            raise ActionError('pathlib_cwd is required in single-path mode')
+
         # CRITICAL: Integrate with existing uploader mechanism
+        # In multi-path mode, pathlib_cwd may be None, but uploader should still work
         uploader = self.get_uploader(pathlib_cwd, file_specification_template, organized_files, self.params)
         organized_files = uploader.handle_upload_files()
 

synapse_sdk/plugins/categories/upload/actions/upload/context.py

@@ -87,7 +87,6 @@ class UploadContext:
             'generated_data_units_count': len(self.data_units),
             'success': len(self.errors) == 0,
             'errors': self.errors,
-            'metrics': self.metrics,
         }
 
     def has_errors(self) -> bool:

synapse_sdk/plugins/categories/upload/actions/upload/steps/generate.py

@@ -31,7 +31,9 @@ class GenerateDataUnitsStep(BaseStep):
             context.run.log_message_with_code(LogCode.GENERATING_DATA_UNITS)
 
             # Initialize metrics
-            context.update_metrics('data_units', {'stand_by': upload_result_count, 'success': 0, 'failed': 0})
+            initial_metrics = {'stand_by': upload_result_count, 'success': 0, 'failed': 0}
+            context.update_metrics('data_units', initial_metrics)
+            context.run.set_metrics(initial_metrics, category='data_units')
 
             # Get batch size from parameters
             batch_size = context.get_param('creating_data_unit_batch_size', 1)
@@ -49,7 +51,9 @@ class GenerateDataUnitsStep(BaseStep):
                 )
 
             # Update final metrics
-            context.update_metrics('data_units', {'stand_by': 0, 'success': len(generated_data_units), 'failed': 0})
+            final_metrics = {'stand_by': 0, 'success': len(generated_data_units), 'failed': 0}
+            context.update_metrics('data_units', final_metrics)
+            context.run.set_metrics(final_metrics, category='data_units')
 
             # Complete progress
             context.run.set_progress(upload_result_count, upload_result_count, category='generate_data_units')

synapse_sdk/plugins/categories/upload/actions/upload/steps/initialize.py

@@ -29,19 +29,34 @@ class InitializeStep(BaseStep):
         except Exception as e:
             return self.create_error_result(f'Failed to get storage {storage_id}: {str(e)}')
 
-        # Get and validate path
+        # Check if we're in multi-path mode
+        use_single_path = context.get_param('use_single_path', True)
+
+        # Get and validate path (only required in single-path mode)
         path = context.get_param('path')
-        if path is None:
-            return self.create_error_result('Path parameter is required')
+        pathlib_cwd = None
 
-        try:
-            pathlib_cwd = get_pathlib(storage, path)
-            context.set_pathlib_cwd(pathlib_cwd)
-        except Exception as e:
-            return self.create_error_result(f'Failed to get path {path}: {str(e)}')
+        if use_single_path:
+            # Single-path mode: global path is required
+            if path is None:
+                return self.create_error_result('Path parameter is required in single-path mode')
+
+            try:
+                pathlib_cwd = get_pathlib(storage, path)
+                context.set_pathlib_cwd(pathlib_cwd)
+            except Exception as e:
+                return self.create_error_result(f'Failed to get path {path}: {str(e)}')
+        else:
+            # Multi-path mode: global path is optional (each asset has its own path)
+            if path:
+                try:
+                    pathlib_cwd = get_pathlib(storage, path)
+                    context.set_pathlib_cwd(pathlib_cwd)
+                except Exception as e:
+                    return self.create_error_result(f'Failed to get path {path}: {str(e)}')
 
         # Return success with rollback data
-        rollback_data = {'storage_id': storage_id, 'path': path}
+        rollback_data = {'storage_id': storage_id, 'path': path, 'use_single_path': use_single_path}
 
         return self.create_success_result(
             data={'storage': storage, 'pathlib_cwd': pathlib_cwd}, rollback_data=rollback_data

synapse_sdk/plugins/categories/upload/actions/upload/steps/metadata.py

@@ -61,9 +61,15 @@ class ProcessMetadataStep(BaseStep):
                 excel_metadata = metadata_strategy.extract(excel_path)
             else:
                 # Look for default metadata files (meta.xlsx, meta.xls)
-                excel_path = self._find_excel_metadata_file(context.pathlib_cwd)
-                if excel_path:
-                    excel_metadata = metadata_strategy.extract(excel_path)
+                # Only possible in single-path mode where pathlib_cwd is set
+                if context.pathlib_cwd:
+                    excel_path = self._find_excel_metadata_file(context.pathlib_cwd)
+                    if excel_path:
+                        excel_metadata = metadata_strategy.extract(excel_path)
+                else:
+                    context.run.log_message(
+                        'No Excel metadata specified and multi-path mode - skipping metadata processing'
+                    )
 
             # Validate extracted metadata
             if excel_metadata:
@@ -136,9 +142,13 @@ class ProcessMetadataStep(BaseStep):
         if path.exists() and path.is_file():
             return path, False
 
-        # Try relative to cwd
-        path = context.pathlib_cwd / excel_path_str
-        return (path, False) if path.exists() else (None, False)
+        # Try relative to cwd (only if pathlib_cwd is set)
+        if context.pathlib_cwd:
+            path = context.pathlib_cwd / excel_path_str
+            return (path, False) if path.exists() else (None, False)
+
+        # In multi-path mode without pathlib_cwd, can only use absolute paths
+        return (None, False)
 
     def _resolve_excel_path_from_base64(
         self, excel_config: dict | ExcelMetadataFile, context: UploadContext
@@ -179,7 +189,17 @@ class ProcessMetadataStep(BaseStep):
             return None, False
 
     def _find_excel_metadata_file(self, pathlib_cwd: Path) -> Path:
-        """Find default Excel metadata file."""
+        """Find default Excel metadata file.
+
+        Args:
+            pathlib_cwd: Working directory path (must not be None)
+
+        Returns:
+            Path to Excel metadata file, or None if not found
+        """
+        if not pathlib_cwd:
+            return None
+
         # Check .xlsx first as it's more common
         excel_path = pathlib_cwd / 'meta.xlsx'
         if excel_path.exists():

synapse_sdk/plugins/categories/upload/actions/upload/steps/organize.py

@@ -90,25 +90,40 @@ class OrganizeFilesStep(BaseStep):
         if not assets:
             return self.create_error_result('Multi-path mode requires assets configuration')
 
+        # Validate that all required specs have asset paths
+        required_specs = [spec['name'] for spec in context.file_specifications if spec.get('is_required', False)]
+        missing_required = [spec for spec in required_specs if spec not in assets]
+
+        if missing_required:
+            return self.create_error_result(
+                f'Multi-path mode requires asset paths for required specs: {", ".join(missing_required)}'
+            )
+
         context.run.log_message(f'Using multi-path mode with {len(assets)} asset configurations')
         context.run.log_message_with_code(LogCode.FILE_ORGANIZATION_STARTED)
 
-        all_organized_files = []
+        # Collect all files and specs first
+        all_files = []
         type_dirs = {}
+        specs_with_files = []
 
         for spec in context.file_specifications:
             spec_name = spec['name']
+            is_required = spec.get('is_required', False)
 
-            # Skip if no asset configuration for this spec
+            # Skip if no asset configuration for this spec (only allowed for optional specs)
             if spec_name not in assets:
-                context.run.log_message(f'Skipping {spec_name}: no asset path configured')
+                if is_required:
+                    # This should not happen due to validation above, but double-check
+                    return self.create_error_result(f'Required spec {spec_name} missing asset path')
+                context.run.log_message(f'Skipping optional spec {spec_name}: no asset path configured')
                 continue
 
             asset_config = assets[spec_name]
 
             # Get the asset path from storage
             try:
-                asset_path = get_pathlib(context.storage, asset_config.path)
+                asset_path = get_pathlib(context.storage, asset_config.get('path', ''))
                 type_dirs[spec_name] = asset_path
             except Exception as e:
                 context.run.log_message(f'Error accessing path for {spec_name}: {str(e)}', 'WARNING')
@@ -119,9 +134,9 @@ class OrganizeFilesStep(BaseStep):
                 continue
 
             # Discover files for this asset
-            is_recursive = asset_config.is_recursive
+            is_recursive = asset_config.get('is_recursive', True)
             context.run.log_message(
-                f'Discovering files for {spec_name} at {asset_config.path} (recursive={is_recursive})'
+                f'Discovering files for {spec_name} at {asset_config.get("path", "")} (recursive={is_recursive})'
             )
 
             files = file_discovery_strategy.discover(asset_path, is_recursive)
@@ -130,14 +145,23 @@ class OrganizeFilesStep(BaseStep):
                 context.run.log_message(f'No files found for {spec_name}', 'WARNING')
                 continue
 
-            # Organize files for this specific spec
-            organized = file_discovery_strategy.organize(files, [spec], context.metadata or {}, {spec_name: asset_path})
+            all_files.extend(files)
+            specs_with_files.append(spec)
+            context.run.log_message(f'Found {len(files)} files for {spec_name}')
 
-            all_organized_files.extend(organized)
-            context.run.log_message(f'Found {len(organized)} files for {spec_name}')
+        # Organize all files together to group by dataset_key
+        all_organized_files = []
+        if all_files and specs_with_files:
+            context.run.log_message(f'Organizing {len(all_files)} files across {len(specs_with_files)} specs')
+            context.run.log_message(f'Type directories: {list(type_dirs.keys())}')
+
+            all_organized_files = file_discovery_strategy.organize(
+                all_files, specs_with_files, context.metadata or {}, type_dirs
+            )
 
         if all_organized_files:
             context.run.log_message_with_code(LogCode.FILES_DISCOVERED, len(all_organized_files))
+            context.run.log_message(f'Created {len(all_organized_files)} data units from {len(all_files)} files')
             context.add_organized_files(all_organized_files)
         else:
             context.run.log_message_with_code(LogCode.NO_FILES_FOUND_WARNING)
@@ -159,8 +183,17 @@ class OrganizeFilesStep(BaseStep):
 
     def validate_prerequisites(self, context: UploadContext) -> None:
         """Validate prerequisites for file organization."""
-        if not context.pathlib_cwd:
-            raise ValueError('Working directory path not set')
+        use_single_path = context.get_param('use_single_path', True)
+
+        # In single-path mode, pathlib_cwd is required
+        if use_single_path and not context.pathlib_cwd:
+            raise ValueError('Working directory path not set in single-path mode')
+
+        # In multi-path mode, pathlib_cwd is optional (each asset has its own path)
+        if not use_single_path:
+            assets = context.get_param('assets', {})
+            if not assets:
+                raise ValueError('Multi-path mode requires assets configuration')
 
         if not context.file_specifications:
             raise ValueError('File specifications not available')

synapse_sdk/plugins/categories/upload/actions/upload/steps/upload.py

@@ -34,7 +34,9 @@ class UploadFilesStep(BaseStep):
             context.run.log_message_with_code(LogCode.UPLOADING_DATA_FILES)
 
             # Initialize metrics
-            context.update_metrics('data_files', {'stand_by': organized_files_count, 'success': 0, 'failed': 0})
+            initial_metrics = {'stand_by': organized_files_count, 'success': 0, 'failed': 0}
+            context.update_metrics('data_files', initial_metrics)
+            context.run.set_metrics(initial_metrics, category='data_files')
 
             # Create upload configuration
             upload_config = UploadConfig(
@@ -55,10 +57,13 @@ class UploadFilesStep(BaseStep):
                 context.run.log_data_file(uploaded_file, UploadStatus.SUCCESS)
 
             # Update final metrics
-            context.update_metrics(
-                'data_files',
-                {'stand_by': 0, 'success': len(uploaded_files), 'failed': organized_files_count - len(uploaded_files)},
-            )
+            final_metrics = {
+                'stand_by': 0,
+                'success': len(uploaded_files),
+                'failed': organized_files_count - len(uploaded_files),
+            }
+            context.update_metrics('data_files', final_metrics)
+            context.run.set_metrics(final_metrics, category='data_files')
 
             # Complete progress
             context.run.set_progress(organized_files_count, organized_files_count, category='upload_data_files')

synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/flat.py

@@ -10,7 +10,11 @@ class FlatFileDiscoveryStrategy(FileDiscoveryStrategy):
 
     def discover(self, path: Path, recursive: bool) -> List[Path]:
         """Discover files non-recursively in the given path."""
-        return [file_path for file_path in path.glob('*') if file_path.is_file()]
+        # Exclude system files
+        excluded_files = {'.DS_Store', 'Thumbs.db', 'desktop.ini'}
+        return [
+            file_path for file_path in path.glob('*') if file_path.is_file() and file_path.name not in excluded_files
+        ]
 
     def organize(self, files: List[Path], specs: Dict, metadata: Dict, type_dirs: Dict = None) -> List[Dict]:
         """Organize files according to specifications with metadata."""
@@ -39,9 +43,14 @@ class FlatFileDiscoveryStrategy(FileDiscoveryStrategy):
         # Performance optimization 2: Build metadata index for faster lookups
         metadata_index = self._build_metadata_index(metadata)
 
-        # Group files by dataset (flat discovery)
+        # Group files by dataset_key (stem-based matching) - flat discovery (no subdirectories)
+        # Strategy:
+        # 1. Group all files (required + optional) by their file stem
+        # 2. Only create data units for groups that have ALL required files
+        # 3. Optional files are automatically included if they match the stem
         dataset_files = {}
         required_specs = [spec['name'] for spec in specs if spec.get('is_required', False)]
+        optional_specs = [spec['name'] for spec in specs if not spec.get('is_required', False)]
 
         for file_path in files:
             # Determine which type directory this file belongs to
@@ -64,20 +73,36 @@ class FlatFileDiscoveryStrategy(FileDiscoveryStrategy):
                             # If stat fails, keep existing file
                             pass
 
-        # Create organized files for datasets with all required files
+        # Create organized files ONLY for datasets with ALL required files
+        # Optional files are included automatically if they match the stem
         for file_name, files_dict in sorted(dataset_files.items()):
-            if all(req in files_dict for req in required_specs):
-                # Calculate most common file extension (optimized)
+            # Check if all required files are present
+            has_all_required = all(req in files_dict for req in required_specs)
+
+            if has_all_required:
+                # Extract original file stem from actual file paths (more reliable)
+                # Collect stems from all files in the group
+                file_stems = {}
                 file_extensions = {}
+
                 for file_path in files_dict.values():
+                    stem = file_path.stem
                     ext = file_path.suffix.lower()
+
+                    # Count stems (to handle multiple files with slightly different names)
+                    if stem:
+                        file_stems[stem] = file_stems.get(stem, 0) + 1
+
+                    # Count extensions
                     if ext:
                         file_extensions[ext] = file_extensions.get(ext, 0) + 1
 
+                # Use the most common stem (usually they're all the same)
+                original_stem = max(file_stems, key=file_stems.get) if file_stems else file_name
                 origin_file_extension = max(file_extensions, key=file_extensions.get) if file_extensions else ''
 
                 meta_data = {
-                    'origin_file_stem': file_name,
+                    'origin_file_stem': original_stem,
                     'origin_file_extension': origin_file_extension,
                     'created_at': datetime.now().isoformat(),
                 }