synapse-sdk 2025.9.5__py3-none-any.whl → 2025.10.6__py3-none-any.whl

synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/recursive.py

@@ -10,7 +10,14 @@ class RecursiveFileDiscoveryStrategy(FileDiscoveryStrategy):
 
     def discover(self, path: Path, recursive: bool) -> List[Path]:
         """Discover files recursively in the given path."""
-        return [file_path for file_path in path.rglob('*') if file_path.is_file()]
+        # Exclude system directories
+        excluded_dirs = {'@eaDir', '.@__thumb', '@Recycle', '#recycle', '.DS_Store', 'Thumbs.db', '.synology'}
+
+        def exclude_dirs(file_path: Path) -> bool:
+            """Check if file path contains excluded directories."""
+            return any(excluded_dir in file_path.parts for excluded_dir in excluded_dirs)
+
+        return [file_path for file_path in path.rglob('*') if file_path.is_file() and not exclude_dirs(file_path)]
 
     def organize(self, files: List[Path], specs: Dict, metadata: Dict, type_dirs: Dict = None) -> List[Dict]:
         """Organize files according to specifications with metadata."""
@@ -42,57 +49,78 @@ class RecursiveFileDiscoveryStrategy(FileDiscoveryStrategy):
         # Performance optimization 2: Build metadata index for faster lookups
         metadata_index = self._build_metadata_index(metadata)
 
-        # Group files by dataset
+        # Group files by dataset_key (stem-based matching)
+        # Strategy:
+        # 1. Group all files (required + optional) by their file stem
+        # 2. Only create data units for groups that have ALL required files
+        # 3. Optional files are automatically included if they match the stem
         dataset_files = {}
         required_specs = [spec['name'] for spec in specs if spec.get('is_required', False)]
 
         for file_path in files:
             # Determine which type directory this file belongs to
-            file_path_str = str(file_path)
+            matched = False
             for spec_name, dir_path in type_dirs.items():
-                dir_path_str = path_cache[dir_path]
-                if file_path_str.startswith(dir_path_str):
-                    # Create unique dataset key using relative path from spec directory
+                # Check if file is under this spec's directory
+                # Use try/except for relative_to to ensure proper path matching
+                try:
                     relative_path = file_path.relative_to(dir_path)
-                    # Use parent directory + stem as unique key to group related files
-                    if relative_path.parent != Path('.'):
-                        dataset_key = f'{relative_path.parent}_{file_path.stem}'
-                    else:
-                        dataset_key = file_path.stem
-
-                    if dataset_key not in dataset_files:
-                        dataset_files[dataset_key] = {}
-
-                    if spec_name not in dataset_files[dataset_key]:
-                        dataset_files[dataset_key][spec_name] = file_path
-                    else:
-                        # Keep the most recent file - only stat when needed
-                        existing_file = dataset_files[dataset_key][spec_name]
-                        try:
-                            if file_path.stat().st_mtime > existing_file.stat().st_mtime:
-                                dataset_files[dataset_key][spec_name] = file_path
-                        except (OSError, IOError):
-                            # If stat fails, keep existing file
-                            pass
-
-        # Create organized files for datasets with all required files
-        for dataset_key, files_dict in sorted(dataset_files.items()):
-            if all(req in files_dict for req in required_specs):
-                # Extract original file stem from dataset_key
-                # If dataset_key contains path info (parent_stem), extract just the stem part
-                if '_' in dataset_key and len(dataset_key.split('_')) >= 2:
-                    # Extract the last part after the last underscore as the original stem
-                    original_stem = dataset_key.split('_')[-1]
+                    matched = True
+                except ValueError:
+                    # File is not under this directory
+                    continue
+
+                # Create unique dataset key using relative path from spec directory
+                # Use parent directory + stem as unique key to group related files
+                if relative_path.parent != Path('.'):
+                    dataset_key = f'{relative_path.parent}_{file_path.stem}'
+                else:
+                    dataset_key = file_path.stem
+
+                if dataset_key not in dataset_files:
+                    dataset_files[dataset_key] = {}
+
+                if spec_name not in dataset_files[dataset_key]:
+                    dataset_files[dataset_key][spec_name] = file_path
                 else:
-                    original_stem = dataset_key
+                    # Keep the most recent file - only stat when needed
+                    existing_file = dataset_files[dataset_key][spec_name]
+                    try:
+                        if file_path.stat().st_mtime > existing_file.stat().st_mtime:
+                            dataset_files[dataset_key][spec_name] = file_path
+                    except (OSError, IOError):
+                        # If stat fails, keep existing file
+                        pass
+
+                # Found matching directory, move to next file
+                break
+
+        # Create organized files ONLY for datasets with ALL required files
+        # Optional files are included automatically if they match the stem
+        for dataset_key, files_dict in sorted(dataset_files.items()):
+            # Check if all required files are present
+            has_all_required = all(req in files_dict for req in required_specs)
 
-                # Calculate most common file extension (optimized)
+            if has_all_required:
+                # Extract original file stem from actual file paths (more reliable than parsing dataset_key)
+                # Collect stems from all files in the group
+                file_stems = {}
                 file_extensions = {}
+
                 for file_path in files_dict.values():
+                    stem = file_path.stem
                     ext = file_path.suffix.lower()
+
+                    # Count stems (to handle multiple files with slightly different names)
+                    if stem:
+                        file_stems[stem] = file_stems.get(stem, 0) + 1
+
+                    # Count extensions
                     if ext:
                         file_extensions[ext] = file_extensions.get(ext, 0) + 1
 
+                # Use the most common stem (usually they're all the same)
+                original_stem = max(file_stems, key=file_stems.get) if file_stems else dataset_key
                 origin_file_extension = max(file_extensions, key=file_extensions.get) if file_extensions else ''
 
                 meta_data = {

synapse_sdk/plugins/categories/upload/actions/upload/strategies/validation/default.py

@@ -12,12 +12,24 @@ class DefaultValidationStrategy(ValidationStrategy):
         """Validate action parameters."""
         errors = []
 
-        # Check required parameters
-        required_params = ['storage', 'data_collection', 'path', 'name']
+        # Check required parameters (common to all modes)
+        required_params = ['storage', 'data_collection', 'name']
         for param in required_params:
             if param not in params:
                 errors.append(f'Missing required parameter: {param}')
 
+        # Check mode-specific requirements
+        use_single_path = params.get('use_single_path', True)
+
+        if use_single_path:
+            # Single-path mode: 'path' is required
+            if 'path' not in params:
+                errors.append("Missing required parameter 'path' in single-path mode")
+        else:
+            # Multi-path mode: 'assets' is required
+            if 'assets' not in params:
+                errors.append("Missing required parameter 'assets' in multi-path mode")
+
         # Check parameter types
         if 'storage' in params and not isinstance(params['storage'], int):
             errors.append("Parameter 'storage' must be an integer")
@@ -28,6 +40,9 @@ class DefaultValidationStrategy(ValidationStrategy):
         if 'is_recursive' in params and not isinstance(params['is_recursive'], bool):
             errors.append("Parameter 'is_recursive' must be a boolean")
 
+        if 'use_single_path' in params and not isinstance(params['use_single_path'], bool):
+            errors.append("Parameter 'use_single_path' must be a boolean")
+
         return ValidationResult(valid=len(errors) == 0, errors=errors)
 
     def validate_files(self, files: List[Dict], specs: Dict) -> ValidationResult:

synapse_sdk/plugins/categories/upload/templates/README.md

@@ -0,0 +1,394 @@
+# Upload Plugin
+
+The Upload Plugin provides comprehensive file and data upload functionality with support for various storage backends, flexible asset path configuration, and Excel metadata integration.
+
+## Quick Start Usage
+
+### CLI Usage Examples
+
+#### Standard Upload (Single Directory)
+
+```bash
+synapse plugin run upload '{
+  "name": "Dataset Upload",
+  "storage": 1,
+  "collection": 2,
+  "use_single_path": false,
+  "assets": {
+    "path": "/data/dataset",
+    "recursive": true
+  }
+}'
+```
+
+#### Multi-Path Upload (Different Locations)
+
+```bash
+synapse plugin run upload '{
+  "name": "Complex Dataset Upload",
+  "storage": 1,
+  "collection": 2,
+  "use_single_path": true,
+  "assets": {
+    "images": {"path": "/images", "recursive": true},
+    "pointclouds": {"path": "/pcd", "recursive": false},
+    "annotations": {"path": "/labels", "recursive": true}
+  },
+  "excel_metadata_path": "/metadata/dataset_info.xlsx"
+}' --debug
+```
+
+### Common Use Cases
+
+#### 1. Simple Dataset Upload
+
+```json
+{
+  "name": "Training Dataset",
+  "storage": 1,
+  "collection": 2,
+  "assets": {
+    "path": "/datasets/training",
+    "recursive": true
+  }
+}
+```
+
+#### 2. Multi-Source Dataset Upload
+
+```json
+{
+  "name": "Multi-Camera Dataset",
+  "storage": 1,
+  "collection": 2,
+  "use_single_path": true,
+  "assets": {
+    "front_camera": { "path": "/cameras/front", "recursive": true },
+    "rear_camera": { "path": "/cameras/rear", "recursive": true },
+    "lidar": { "path": "/sensors/lidar", "recursive": false }
+  }
+}
+```
+
+#### 3. Dataset with Metadata
+
+```json
+{
+  "name": "Annotated Dataset",
+  "storage": 1,
+  "collection": 2,
+  "assets": {
+    "path": "/data/annotated",
+    "recursive": true
+  },
+  "excel_metadata_path": "/data/metadata.xlsx"
+}
+```
+
+## Configuration Parameters
+
+### Required Parameters
+
+| Parameter    | Type    | Description                         | Example            |
+| ------------ | ------- | ----------------------------------- | ------------------ |
+| `name`       | string  | Display name for the upload         | `"My Dataset"`     |
+| `storage`    | integer | Storage backend ID                  | `1`                |
+| `collection` | integer | Collection ID defining file specs   | `2`                |
+| `assets`     | object  | Path configuration (varies by mode) | See examples below |
+
+### Optional Parameters
+
+| Parameter             | Type     | Default | Description                                                                      |
+| --------------------- | -------- | ------- | -------------------------------------------------------------------------------- |
+| `description`         | string   | `null`  | Upload description                                                               |
+| `project`             | integer  | `null`  | Project ID to associate                                                          |
+| `use_single_path`     | boolean  | `false` | Enable individual path mode                                                      |
+| `is_recursive`        | boolean  | `false` | Global recursive setting                                                         |
+| `excel_metadata_path` | `string` | `null`  | **DEPRECATED** - File path to Excel metadata file (use `excel_metadata` instead) |
+| `excel_metadata`      | `object` | `null`  | Base64 encoded Excel metadata (recommended)                                      |
+
+## Excel Metadata Support
+
+The upload plugin provides advanced Excel metadata processing with flexible header support, comprehensive filename matching, and two distinct input methods.
+
+### Input Methods
+
+There are two separate parameters for providing Excel metadata:
+
+#### 1. File Path Method (`excel_metadata_path`) - **DEPRECATED**
+
+:::warning Deprecation Notice
+This parameter is **deprecated** and will be removed in a future version.
+Please migrate to using the `excel_metadata` parameter with base64 encoding instead.
+:::
+
+**Use case:** Traditional file-based uploads where the Excel file exists on the server's file system.
+
+Simple string path to an Excel file:
+
+```json
+{
+  "excel_metadata_path": "/data/metadata.xlsx"
+}
+```
+
+**Advantages:**
+
+- Backward compatible with existing implementations
+- Simple and straightforward
+- Direct file system access
+
+#### 2. Base64 Encoded Method (`excel_metadata`)
+
+**Use case:** Web frontends, APIs, and cloud integrations where files are transmitted as encoded data.
+
+Send Excel file as base64-encoded data with original filename:
+
+```json
+{
+  "excel_metadata": {
+    "data": "UEsDBBQABgAIAAAAIQDd4Z...",
+    "filename": "metadata.xlsx"
+  }
+}
+```
+
+**Advantages:**
+
+- No intermediate file storage required
+- Perfect for web upload forms
+- API-friendly JSON payload
+- Automatic temporary file cleanup
+- **This is the recommended method going forward**
+
+**Important:** You cannot use both `excel_metadata_path` and `excel_metadata` at the same time
+
+**Migration Example:**
+
+```python
+import base64
+
+# Old way (deprecated)
+params = {
+    "excel_metadata_path": "/data/metadata.xlsx"
+}
+
+# New way (recommended)
+with open("/data/metadata.xlsx", "rb") as f:
+    encoded = base64.b64encode(f.read()).decode("utf-8")
+params = {
+    "excel_metadata": {
+        "data": encoded,
+        "filename": "metadata.xlsx"
+    }
+}
+```
+
+### Excel Format Example
+
+| filename  | category   | quality | notes             |
+| --------- | ---------- | ------- | ----------------- |
+| sample001 | vehicle    | high    | Clear visibility  |
+| sample002 | pedestrian | medium  | Partial occlusion |
+
+### Security Limits
+
+- Max file size: 10MB
+- Max rows: 10,000
+- Max columns: 50
+
+## File Matching Logic
+
+Files are matched by **stem name** (filename without extension):
+
+- `sample001.jpg` → stem: "sample001"
+- `sample001.pcd` → stem: "sample001"
+- `sample001.json` → stem: "sample001"
+
+These files form a single dataset named "sample001".
+
+## Troubleshooting Guide
+
+### Common Issues
+
+#### "No Files Found" Error
+
+```bash
+# Check path exists and is readable
+ls -la /path/to/data
+test -r /path/to/data && echo "Readable" || echo "Not readable"
+
+# Verify files exist
+find /path/to/data -name "*.jpg" | head -10
+```
+
+#### Excel Processing Errors
+
+```bash
+# Check file format and size
+file /path/to/metadata.xlsx
+ls -lh /path/to/metadata.xlsx
+
+# Validate Excel content
+python -c "
+from openpyxl import load_workbook
+wb = load_workbook('/path/to/metadata.xlsx')
+print(f'Sheets: {wb.sheetnames}')
+print(f'Rows: {wb.active.max_row}')
+"
+```
+
+#### Upload Failures
+
+```bash
+# Test storage connection
+synapse storage test --storage-id 1
+
+# Verify collection configuration
+synapse collection show --id 2
+
+# Run with debug mode
+synapse plugin run upload '{}' --debug
+```
+
+## Best Practices
+
+### Directory Organization
+
+- Use clear, descriptive directory names
+- Keep reasonable directory sizes (< 10,000 files)
+- Use absolute paths for reliability
+
+### Performance Optimization
+
+- Enable recursive only when needed
+- Keep Excel files under 5MB
+- Organize files in balanced directory structures
+
+### Security Considerations
+
+- Validate all paths before processing
+- Use read-only permissions for source data
+- Set appropriate Excel size limits
+
+## Advanced Features
+
+### Batch Processing
+
+The plugin automatically optimizes batch sizes based on dataset size:
+
+- Small datasets (< 50 files): batch size 50
+- Large datasets: dynamic batch size (10-100)
+
+### Progress Tracking
+
+Real-time progress updates with categories:
+
+- Collection analysis: 2%
+- File upload: 38%
+- Data unit generation: 60%
+
+### Error Handling
+
+Comprehensive validation at multiple levels:
+
+- Parameter validation (Pydantic)
+- Runtime path validation
+- File format validation
+- Excel security checks
+
+## Environment Variables
+
+Configure Excel processing limits:
+
+```bash
+# File size limits
+EXCEL_MAX_FILE_SIZE_MB=10
+EXCEL_MAX_MEMORY_MB=30
+
+# Content limits
+EXCEL_MAX_ROWS=10000
+EXCEL_MAX_COLUMNS=50
+
+# String length limits
+EXCEL_MAX_FILENAME_LENGTH=255
+EXCEL_MAX_METADATA_VALUE_LENGTH=1000
+```
+
+## Migration Guide
+
+### Upgrading from Previous Versions
+
+All existing configurations continue to work. New features are additive:
+
+#### Test Current Configuration
+
+```bash
+synapse plugin run upload '{}' --debug
+```
+
+#### Convert to Explicit Mode
+
+```python
+# Add explicit mode setting
+config["use_single_path"] = False  # or True for single path mode
+```
+
+#### Gradual Migration to Single Path Mode
+
+```python
+# Start with subset
+test_config = {
+    "use_single_path": True,
+    "assets": {
+        "test_images": {"path": "/existing/path/images", "recursive": True}
+    }
+}
+
+# Then migrate all assets
+production_config = {
+    "use_single_path": True,
+    "assets": {
+        "images": {"path": "/optimized/path1", "recursive": True},
+        "annotations": {"path": "/optimized/path2", "recursive": False}
+    }
+}
+```
+
+## Storage Backend Support
+
+The plugin supports multiple storage backends:
+
+- **Local filesystem**: Optimized for high I/O
+- **S3/GCS**: Cloud storage with retry logic
+- **SFTP**: Connection pooling for remote servers
+- **HTTP**: Streaming uploads for large files
+
+## API Reference
+
+### Plugin Class
+
+```python
+from synapse import Plugin
+
+plugin = Plugin("upload")
+result = plugin.run(config, debug=True)
+```
+
+### Result Structure
+
+```python
+{
+    "status": "success",
+    "uploaded_files": 150,
+    "data_units_created": 50,
+    "errors": [],
+    "metadata": {}
+}
+```
+
+## Support and Resources
+
+- **Documentation**: Full API documentation at [synapse-docs]
+- **Issues**: Report bugs at [issue-tracker]
+- **Examples**: More examples at [examples-repo]

synapse_sdk/plugins/models.py

@@ -1,7 +1,10 @@
 import os
+from datetime import datetime
 from functools import cached_property
 from typing import Any, Dict
 
+from pydantic import BaseModel
+
 from synapse_sdk.clients.backend import BackendClient
 from synapse_sdk.devtools.config import get_backend_config
 from synapse_sdk.loggers import BackendLogger, ConsoleLogger
@@ -131,6 +134,26 @@ class Run:
     context = None
     client = None
 
+    class DevLog(BaseModel):
+        """Model for developer log entries.
+
+        Records custom events and information that plugin developers want to track
+        during plugin execution for debugging and monitoring purposes.
+
+        Attributes:
+            event_type (str): Type/category of the development event
+            message (str): Descriptive message about the event
+            data (dict | None): Optional additional data/context
+            level (Context): Event status/severity level
+            created (str): Timestamp when event occurred
+        """
+
+        event_type: str
+        message: str
+        data: dict | None = None
+        level: Context
+        created: str
+
     def __init__(self, job_id, context=None):
         self.job_id = job_id
         self.context = context or {}
@@ -177,5 +200,44 @@ class Run:
     def log_message(self, message, context=Context.INFO.value):
         self.logger.log('message', {'context': context, 'content': message})
 
+    def log_dev_event(self, message: str, data: dict | None = None, level: Context = Context.INFO):
+        """Log development event for plugin developers.
+
+        This function allows plugin developers to log custom events and information
+        during plugin execution for debugging, monitoring, and development purposes.
+        The event_type is automatically constructed as '{action_name}_dev_log' and cannot
+        be modified by plugin developers.
+
+        Args:
+            message (str): Descriptive message about the event
+            data (dict | None): Optional additional data or context to include
+            level (Context): Event severity level (INFO, WARNING, DANGER, SUCCESS)
+
+        Example:
+            >>> run = Run(job_id, context)
+            >>> run.log_dev_event('Data validation completed', {'records_count': 100})
+            >>> run.log_dev_event('Processing time recorded', {'duration_ms': 1500})
+            >>> run.log_dev_event('Variable state at checkpoint', {'variable_x': 42}, level=Context.WARNING)
+        """
+        # Construct event_type from action name - this cannot be modified by developers
+        action_name = self.context.get('action_name', 'unknown')
+        event_type = f'{action_name}_dev_log'
+
+        # Log the message for basic logging
+        self.log_message(f'[{event_type.upper()}] {message}', context=level.value)
+
+        # Also log the structured event for development tracking
+        now = datetime.now().isoformat()
+        self.log(
+            'dev_event',
+            self.DevLog(
+                event_type=event_type,
+                message=message,
+                data=data,
+                level=level,
+                created=now,
+            ).model_dump(),
+        )
+
     def end_log(self):
         self.log_message('Plugin run is complete.')