synapse-sdk 2025.10.1__py3-none-any.whl → 2025.10.4__py3-none-any.whl

synapse_sdk/plugins/categories/upload/actions/upload/steps/metadata.py

@@ -1,8 +1,11 @@
+import base64
+import tempfile
 from pathlib import Path
 
 from ..context import StepResult, UploadContext
 from ..enums import LogCode
 from ..exceptions import ExcelParsingError, ExcelSecurityError
+from ..models import ExcelMetadataFile
 from .base import BaseStep
 
 
@@ -25,22 +28,36 @@ class ProcessMetadataStep(BaseStep):
             return self.create_success_result(data={'metadata': {}})
 
         excel_metadata = {}
+        temp_file_to_cleanup = None
 
         try:
-            # Check if Excel metadata path is specified
-            excel_metadata_path = context.get_param('excel_metadata_path')
-            if excel_metadata_path:
-                # Convert string to Path object
-                if isinstance(excel_metadata_path, str):
-                    excel_metadata_path = Path(excel_metadata_path)
-
-                if excel_metadata_path.exists() and excel_metadata_path.is_file():
-                    excel_path = excel_metadata_path
-                else:
-                    excel_path = context.pathlib_cwd / excel_metadata_path
-                if not excel_path.exists():
+            # Check if Excel metadata is specified - try both parameters
+            # TODO: Plan to deprecate excel_metadata_path in a few versions (backward compatibility)
+            excel_metadata_path_config = context.get_param('excel_metadata_path')
+            excel_metadata_config = context.get_param('excel_metadata')
+
+            if excel_metadata_path_config:
+                # Traditional path-based approach (will be deprecated in future)
+                excel_path, is_temp = self._resolve_excel_path_from_string(excel_metadata_path_config, context)
+
+                if not excel_path or not excel_path.exists():
                     context.run.log_message_with_code(LogCode.EXCEL_FILE_NOT_FOUND_PATH)
                     return self.create_success_result(data={'metadata': {}})
+
+                excel_metadata = metadata_strategy.extract(excel_path)
+
+            elif excel_metadata_config:
+                # Base64 encoded approach
+                excel_path, is_temp = self._resolve_excel_path_from_base64(excel_metadata_config, context)
+
+                if not excel_path or not excel_path.exists():
+                    context.run.log_message_with_code(LogCode.EXCEL_FILE_NOT_FOUND_PATH)
+                    return self.create_success_result(data={'metadata': {}})
+
+                # Track temp file for cleanup
+                if is_temp:
+                    temp_file_to_cleanup = excel_path
+
                 excel_metadata = metadata_strategy.extract(excel_path)
             else:
                 # Look for default metadata files (meta.xlsx, meta.xls)
@@ -65,9 +82,9 @@ class ProcessMetadataStep(BaseStep):
             return self.create_error_result(f'Excel security violation: {str(e)}')
 
         except ExcelParsingError as e:
-            # If excel_metadata_path was specified, this is an error
+            # If excel_metadata_path or excel_metadata was specified, this is an error
             # If we were just looking for default files, it's not an error
-            if context.get_param('excel_metadata_path'):
+            if context.get_param('excel_metadata_path') or context.get_param('excel_metadata'):
                 context.run.log_message_with_code(LogCode.EXCEL_PARSING_ERROR, str(e))
                 return self.create_error_result(f'Excel parsing error: {str(e)}')
             else:
@@ -77,6 +94,15 @@ class ProcessMetadataStep(BaseStep):
         except Exception as e:
             return self.create_error_result(f'Unexpected error processing metadata: {str(e)}')
 
+        finally:
+            # Clean up temporary file if it was created from base64
+            if temp_file_to_cleanup and temp_file_to_cleanup.exists():
+                try:
+                    temp_file_to_cleanup.unlink()
+                    context.run.log_message(f'Cleaned up temporary Excel file: {temp_file_to_cleanup}')
+                except Exception as e:
+                    context.run.log_message(f'Failed to clean up temporary file {temp_file_to_cleanup}: {str(e)}')
+
     def can_skip(self, context: UploadContext) -> bool:
         """Metadata step can be skipped if no metadata strategy is configured."""
         return 'metadata' not in context.strategies
@@ -86,6 +112,72 @@ class ProcessMetadataStep(BaseStep):
         # Clear any loaded metadata
         context.metadata.clear()
 
+    def _resolve_excel_path_from_string(self, excel_path_str: str, context: UploadContext) -> tuple[Path | None, bool]:
+        """Resolve Excel metadata path from a string path.
+
+        Note: This method supports the excel_metadata_path parameter which will be deprecated
+        in a future version. Consider using _resolve_excel_path_from_base64 instead.
+
+        Args:
+            excel_path_str: File path string to the Excel metadata file
+            context: Upload context for resolving relative paths
+
+        Returns:
+            Tuple of (resolved_path, is_temporary_file)
+            - resolved_path: Path object pointing to the Excel file, or None if resolution failed
+            - is_temporary_file: Always False for path-based approach
+
+        Examples:
+            >>> path, is_temp = self._resolve_excel_path_from_string("/data/meta.xlsx", context)
+        """
+        # TODO: Plan to deprecate this method in a few versions (backward compatibility)
+        # Try absolute path first
+        path = Path(excel_path_str)
+        if path.exists() and path.is_file():
+            return path, False
+
+        # Try relative to cwd
+        path = context.pathlib_cwd / excel_path_str
+        return (path, False) if path.exists() else (None, False)
+
+    def _resolve_excel_path_from_base64(
+        self, excel_config: dict | ExcelMetadataFile, context: UploadContext
+    ) -> tuple[Path | None, bool]:
+        """Resolve Excel metadata path from base64 encoded data.
+
+        Args:
+            excel_config: Either a dict or an ExcelMetadataFile object with base64 data
+            context: Upload context for logging
+
+        Returns:
+            Tuple of (resolved_path, is_temporary_file)
+            - resolved_path: Path object pointing to the temporary Excel file, or None if decoding failed
+            - is_temporary_file: Always True for base64 approach (requires cleanup)
+
+        Examples:
+            >>> config = ExcelMetadataFile(data="UEsDB...", filename="meta.xlsx")
+            >>> path, is_temp = self._resolve_excel_path_from_base64(config, context)
+        """
+        if isinstance(excel_config, dict):
+            excel_config = ExcelMetadataFile(**excel_config)
+
+        try:
+            # Decode base64 data
+            decoded_data = base64.b64decode(excel_config.data, validate=True)
+
+            # Create temp file
+            temp_dir = Path(tempfile.gettempdir())
+            filename = excel_config.filename
+            temp_file = temp_dir / filename
+            temp_file.write_bytes(decoded_data)
+
+            context.run.log_message(f'Decoded base64 Excel metadata to temporary file: {temp_file}')
+            return temp_file, True
+
+        except Exception as e:
+            context.run.log_message(f'Failed to decode base64 Excel metadata: {str(e)}')
+            return None, False
+
     def _find_excel_metadata_file(self, pathlib_cwd: Path) -> Path:
         """Find default Excel metadata file."""
         # Check .xlsx first as it's more common

synapse_sdk/plugins/categories/upload/actions/upload/steps/organize.py

@@ -24,51 +24,128 @@ class OrganizeFilesStep(BaseStep):
             return self.create_error_result('File specifications not available')
 
         try:
-            # Create type directories mapping
-            type_dirs = {}
-            for spec in context.file_specifications:
-                spec_name = spec['name']
-                spec_dir = context.pathlib_cwd / spec_name
-                if spec_dir.exists() and spec_dir.is_dir():
-                    type_dirs[spec_name] = spec_dir
-
-            if type_dirs:
-                context.run.log_message_with_code(LogCode.TYPE_DIRECTORIES_FOUND, list(type_dirs.keys()))
+            # Check which mode we're in
+            use_single_path = context.get_param('use_single_path', True)
+
+            if use_single_path:
+                # Single path mode: all assets use same base path
+                return self._execute_single_path_mode(context, file_discovery_strategy)
             else:
-                context.run.log_message_with_code(LogCode.NO_TYPE_DIRECTORIES)
-                return self.create_success_result(data={'organized_files': []})
+                # Multi-path mode: each asset has its own path
+                return self._execute_multi_path_mode(context, file_discovery_strategy)
 
-            context.run.log_message_with_code(LogCode.TYPE_STRUCTURE_DETECTED)
-            context.run.log_message_with_code(LogCode.FILE_ORGANIZATION_STARTED)
+        except Exception as e:
+            return self.create_error_result(f'File organization failed: {str(e)}')
 
-            # Discover files in type directories
-            all_files = []
-            is_recursive = context.get_param('is_recursive', True)
+    def _execute_single_path_mode(self, context: UploadContext, file_discovery_strategy) -> StepResult:
+        """Execute file organization in single path mode (traditional)."""
+        # Create type directories mapping
+        type_dirs = {}
+        for spec in context.file_specifications:
+            spec_name = spec['name']
+            spec_dir = context.pathlib_cwd / spec_name
+            if spec_dir.exists() and spec_dir.is_dir():
+                type_dirs[spec_name] = spec_dir
+
+        if type_dirs:
+            context.run.log_message_with_code(LogCode.TYPE_DIRECTORIES_FOUND, list(type_dirs.keys()))
+        else:
+            context.run.log_message_with_code(LogCode.NO_TYPE_DIRECTORIES)
+            return self.create_success_result(data={'organized_files': []})
+
+        context.run.log_message_with_code(LogCode.TYPE_STRUCTURE_DETECTED)
+        context.run.log_message_with_code(LogCode.FILE_ORGANIZATION_STARTED)
+
+        # Discover files in type directories
+        all_files = []
+        is_recursive = context.get_param('is_recursive', True)
+
+        for spec_name, dir_path in type_dirs.items():
+            files_in_dir = file_discovery_strategy.discover(dir_path, is_recursive)
+            all_files.extend(files_in_dir)
+
+        if not all_files:
+            context.run.log_message_with_code(LogCode.NO_FILES_FOUND_WARNING)
+            return self.create_success_result(data={'organized_files': []})
+
+        # Organize files using strategy
+        organized_files = file_discovery_strategy.organize(
+            all_files, context.file_specifications, context.metadata or {}, type_dirs
+        )
+
+        if organized_files:
+            context.run.log_message_with_code(LogCode.FILES_DISCOVERED, len(organized_files))
+            context.add_organized_files(organized_files)
+
+        return self.create_success_result(
+            data={'organized_files': organized_files},
+            rollback_data={'files_count': len(organized_files), 'type_dirs': list(type_dirs.keys())},
+        )
+
+    def _execute_multi_path_mode(self, context: UploadContext, file_discovery_strategy) -> StepResult:
+        """Execute file organization in multi-path mode (each asset has own path)."""
+        from synapse_sdk.utils.storage import get_pathlib
+
+        assets = context.get_param('assets', {})
+        if not assets:
+            return self.create_error_result('Multi-path mode requires assets configuration')
+
+        context.run.log_message(f'Using multi-path mode with {len(assets)} asset configurations')
+        context.run.log_message_with_code(LogCode.FILE_ORGANIZATION_STARTED)
+
+        all_organized_files = []
+        type_dirs = {}
+
+        for spec in context.file_specifications:
+            spec_name = spec['name']
+
+            # Skip if no asset configuration for this spec
+            if spec_name not in assets:
+                context.run.log_message(f'Skipping {spec_name}: no asset path configured')
+                continue
+
+            asset_config = assets[spec_name]
+
+            # Get the asset path from storage
+            try:
+                asset_path = get_pathlib(context.storage, asset_config.path)
+                type_dirs[spec_name] = asset_path
+            except Exception as e:
+                context.run.log_message(f'Error accessing path for {spec_name}: {str(e)}', 'WARNING')
+                continue
+
+            if not asset_path.exists():
+                context.run.log_message(f'Path does not exist for {spec_name}: {asset_config.path}', 'WARNING')
+                continue
+
+            # Discover files for this asset
+            is_recursive = asset_config.is_recursive
+            context.run.log_message(
+                f'Discovering files for {spec_name} at {asset_config.path} (recursive={is_recursive})'
+            )
 
-            for spec_name, dir_path in type_dirs.items():
-                files_in_dir = file_discovery_strategy.discover(dir_path, is_recursive)
-                all_files.extend(files_in_dir)
+            files = file_discovery_strategy.discover(asset_path, is_recursive)
 
-            if not all_files:
-                context.run.log_message_with_code(LogCode.NO_FILES_FOUND_WARNING)
-                return self.create_success_result(data={'organized_files': []})
+            if not files:
+                context.run.log_message(f'No files found for {spec_name}', 'WARNING')
+                continue
 
-            # Organize files using strategy
-            organized_files = file_discovery_strategy.organize(
-                all_files, context.file_specifications, context.metadata or {}, type_dirs
-            )
+            # Organize files for this specific spec
+            organized = file_discovery_strategy.organize(files, [spec], context.metadata or {}, {spec_name: asset_path})
 
-            if organized_files:
-                context.run.log_message_with_code(LogCode.FILES_DISCOVERED, len(organized_files))
-                context.add_organized_files(organized_files)
+            all_organized_files.extend(organized)
+            context.run.log_message(f'Found {len(organized)} files for {spec_name}')
 
-            return self.create_success_result(
-                data={'organized_files': organized_files},
-                rollback_data={'files_count': len(organized_files), 'type_dirs': list(type_dirs.keys())},
-            )
+        if all_organized_files:
+            context.run.log_message_with_code(LogCode.FILES_DISCOVERED, len(all_organized_files))
+            context.add_organized_files(all_organized_files)
+        else:
+            context.run.log_message_with_code(LogCode.NO_FILES_FOUND_WARNING)
 
-        except Exception as e:
-            return self.create_error_result(f'File organization failed: {str(e)}')
+        return self.create_success_result(
+            data={'organized_files': all_organized_files},
+            rollback_data={'files_count': len(all_organized_files), 'type_dirs': list(type_dirs.keys())},
+        )
 
     def can_skip(self, context: UploadContext) -> bool:
         """File organization cannot be skipped."""

synapse_sdk/plugins/categories/upload/templates/README.md

@@ -0,0 +1,365 @@
+# Upload Plugin
+
+The Upload Plugin provides comprehensive file and data upload functionality with support for various storage backends, flexible asset path configuration, and Excel metadata integration.
+
+## Quick Start Usage
+
+### CLI Usage Examples
+
+#### Standard Upload (Single Directory)
+```bash
+synapse plugin run upload '{
+  "name": "Dataset Upload",
+  "storage": 1,
+  "collection": 2,
+  "use_single_path": false,
+  "assets": {
+    "path": "/data/dataset",
+    "recursive": true
+  }
+}'
+```
+
+#### Multi-Path Upload (Different Locations)
+```bash
+synapse plugin run upload '{
+  "name": "Complex Dataset Upload",
+  "storage": 1,
+  "collection": 2,
+  "use_single_path": true,
+  "assets": {
+    "images": {"path": "/images", "recursive": true},
+    "pointclouds": {"path": "/pcd", "recursive": false},
+    "annotations": {"path": "/labels", "recursive": true}
+  },
+  "excel_metadata_path": "/metadata/dataset_info.xlsx"
+}' --debug
+```
+
+### Common Use Cases
+
+#### 1. Simple Dataset Upload
+```json
+{
+  "name": "Training Dataset",
+  "storage": 1,
+  "collection": 2,
+  "assets": {
+    "path": "/datasets/training",
+    "recursive": true
+  }
+}
+```
+
+#### 2. Multi-Source Dataset Upload
+```json
+{
+  "name": "Multi-Camera Dataset",
+  "storage": 1,
+  "collection": 2,
+  "use_single_path": true,
+  "assets": {
+    "front_camera": {"path": "/cameras/front", "recursive": true},
+    "rear_camera": {"path": "/cameras/rear", "recursive": true},
+    "lidar": {"path": "/sensors/lidar", "recursive": false}
+  }
+}
+```
+
+#### 3. Dataset with Metadata
+```json
+{
+  "name": "Annotated Dataset",
+  "storage": 1,
+  "collection": 2,
+  "assets": {
+    "path": "/data/annotated",
+    "recursive": true
+  },
+  "excel_metadata_path": "/data/metadata.xlsx"
+}
+```
+
+## Configuration Parameters
+
+### Required Parameters
+
+| Parameter | Type | Description | Example |
+|-----------|------|-------------|---------|
+| `name` | string | Display name for the upload | `"My Dataset"` |
+| `storage` | integer | Storage backend ID | `1` |
+| `collection` | integer | Collection ID defining file specs | `2` |
+| `assets` | object | Path configuration (varies by mode) | See examples below |
+
+### Optional Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `description` | string | `null` | Upload description |
+| `project` | integer | `null` | Project ID to associate |
+| `use_single_path` | boolean | `false` | Enable individual path mode |
+| `is_recursive` | boolean | `false` | Global recursive setting |
+| `excel_metadata_path` | `string` | `null` | File path to Excel metadata file (will be deprecated in future, consider using `excel_metadata`) |
+| `excel_metadata` | `object` | `null` | Base64 encoded Excel metadata (recommended) |
+
+## Excel Metadata Support
+
+The upload plugin provides advanced Excel metadata processing with flexible header support, comprehensive filename matching, and two distinct input methods.
+
+### Input Methods
+
+There are two separate parameters for providing Excel metadata:
+
+#### 1. File Path Method (`excel_metadata_path`)
+
+:::info Future Deprecation Notice
+This parameter will be deprecated in a future version.
+We recommend using the `excel_metadata` parameter with base64 encoding for new implementations.
+:::
+
+**Use case:** Traditional file-based uploads where the Excel file exists on the server's file system.
+
+Simple string path to an Excel file:
+
+```json
+{
+  "excel_metadata_path": "/data/metadata.xlsx"
+}
+```
+
+**Advantages:**
+- Backward compatible with existing implementations
+- Simple and straightforward
+- Direct file system access
+
+**Note:** Consider migrating to base64 encoded method for future compatibility (see Method 2 below)
+
+#### 2. Base64 Encoded Method (`excel_metadata`) - **RECOMMENDED**
+
+**Use case:** Web frontends, APIs, and cloud integrations where files are transmitted as encoded data.
+
+Send Excel file as base64-encoded data with original filename:
+
+```json
+{
+  "excel_metadata": {
+    "data": "UEsDBBQABgAIAAAAIQDd4Z...",
+    "filename": "metadata.xlsx"
+  }
+}
+```
+
+**Advantages:**
+- No intermediate file storage required
+- Perfect for web upload forms
+- API-friendly JSON payload
+- Automatic temporary file cleanup
+- **This is the recommended method going forward**
+
+**Important:** You cannot use both `excel_metadata_path` and `excel_metadata` at the same time
+
+**Future-Proof Migration Example:**
+```python
+import base64
+
+# Current approach (will be deprecated in future)
+params = {
+    "excel_metadata_path": "/data/metadata.xlsx"
+}
+
+# Recommended approach for new implementations
+with open("/data/metadata.xlsx", "rb") as f:
+    encoded = base64.b64encode(f.read()).decode("utf-8")
+params = {
+    "excel_metadata": {
+        "data": encoded,
+        "filename": "metadata.xlsx"
+    }
+}
+```
+
+### Excel Format Example
+| filename | category | quality | notes |
+|----------|----------|---------|-------|
+| sample001 | vehicle | high | Clear visibility |
+| sample002 | pedestrian | medium | Partial occlusion |
+
+### Security Limits
+- Max file size: 10MB
+- Max rows: 10,000
+- Max columns: 50
+
+## File Matching Logic
+
+Files are matched by **stem name** (filename without extension):
+- `sample001.jpg` → stem: "sample001"
+- `sample001.pcd` → stem: "sample001"
+- `sample001.json` → stem: "sample001"
+
+These files form a single dataset named "sample001".
+
+## Troubleshooting Guide
+
+### Common Issues
+
+#### "No Files Found" Error
+```bash
+# Check path exists and is readable
+ls -la /path/to/data
+test -r /path/to/data && echo "Readable" || echo "Not readable"
+
+# Verify files exist
+find /path/to/data -name "*.jpg" | head -10
+```
+
+#### Excel Processing Errors
+```bash
+# Check file format and size
+file /path/to/metadata.xlsx
+ls -lh /path/to/metadata.xlsx
+
+# Validate Excel content
+python -c "
+from openpyxl import load_workbook
+wb = load_workbook('/path/to/metadata.xlsx')
+print(f'Sheets: {wb.sheetnames}')
+print(f'Rows: {wb.active.max_row}')
+"
+```
+
+#### Upload Failures
+```bash
+# Test storage connection
+synapse storage test --storage-id 1
+
+# Verify collection configuration
+synapse collection show --id 2
+
+# Run with debug mode
+synapse plugin run upload '{}' --debug
+```
+
+## Best Practices
+
+### Directory Organization
+- Use clear, descriptive directory names
+- Keep reasonable directory sizes (< 10,000 files)
+- Use absolute paths for reliability
+
+### Performance Optimization
+- Enable recursive only when needed
+- Keep Excel files under 5MB
+- Organize files in balanced directory structures
+
+### Security Considerations
+- Validate all paths before processing
+- Use read-only permissions for source data
+- Set appropriate Excel size limits
+
+## Advanced Features
+
+### Batch Processing
+The plugin automatically optimizes batch sizes based on dataset size:
+- Small datasets (< 50 files): batch size 50
+- Large datasets: dynamic batch size (10-100)
+
+### Progress Tracking
+Real-time progress updates with categories:
+- Collection analysis: 2%
+- File upload: 38%
+- Data unit generation: 60%
+
+### Error Handling
+Comprehensive validation at multiple levels:
+- Parameter validation (Pydantic)
+- Runtime path validation
+- File format validation
+- Excel security checks
+
+## Environment Variables
+
+Configure Excel processing limits:
+```bash
+# File size limits
+EXCEL_MAX_FILE_SIZE_MB=10
+EXCEL_MAX_MEMORY_MB=30
+
+# Content limits
+EXCEL_MAX_ROWS=10000
+EXCEL_MAX_COLUMNS=50
+
+# String length limits
+EXCEL_MAX_FILENAME_LENGTH=255
+EXCEL_MAX_METADATA_VALUE_LENGTH=1000
+```
+
+## Migration Guide
+
+### Upgrading from Previous Versions
+All existing configurations continue to work. New features are additive:
+
+#### Test Current Configuration
+```bash
+synapse plugin run upload '{}' --debug
+```
+
+#### Convert to Explicit Mode
+```python
+# Add explicit mode setting
+config["use_single_path"] = False  # or True for single path mode
+```
+
+#### Gradual Migration to Single Path Mode
+```python
+# Start with subset
+test_config = {
+    "use_single_path": True,
+    "assets": {
+        "test_images": {"path": "/existing/path/images", "recursive": True}
+    }
+}
+
+# Then migrate all assets
+production_config = {
+    "use_single_path": True,
+    "assets": {
+        "images": {"path": "/optimized/path1", "recursive": True},
+        "annotations": {"path": "/optimized/path2", "recursive": False}
+    }
+}
+```
+
+## Storage Backend Support
+
+The plugin supports multiple storage backends:
+- **Local filesystem**: Optimized for high I/O
+- **S3/GCS**: Cloud storage with retry logic
+- **SFTP**: Connection pooling for remote servers
+- **HTTP**: Streaming uploads for large files
+
+## API Reference
+
+### Plugin Class
+```python
+from synapse import Plugin
+
+plugin = Plugin("upload")
+result = plugin.run(config, debug=True)
+```
+
+### Result Structure
+```python
+{
+    "status": "success",
+    "uploaded_files": 150,
+    "data_units_created": 50,
+    "errors": [],
+    "metadata": {}
+}
+```
+
+## Support and Resources
+
+- **Documentation**: Full API documentation at [synapse-docs]
+- **Issues**: Report bugs at [issue-tracker]
+- **Examples**: More examples at [examples-repo]