synapse-sdk 1.0.0a13__py3-none-any.whl → 2025.11.7__py3-none-any.whl

synapse_sdk/plugins/categories/upload/actions/upload/strategies/metadata/excel.py

@@ -0,0 +1,174 @@
+from io import BytesIO
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from openpyxl import load_workbook
+from openpyxl.utils.exceptions import InvalidFileException
+
+from ...exceptions import ExcelParsingError, ExcelSecurityError
+from ...utils import ExcelMetadataUtils, ExcelSecurityConfig
+from ..base import MetadataStrategy, ValidationResult
+
+
+class ExcelMetadataStrategy(MetadataStrategy):
+    """Excel metadata extraction strategy."""
+
+    def __init__(self):
+        self.excel_config = ExcelSecurityConfig()
+        self.excel_utils = ExcelMetadataUtils(self.excel_config)
+
+    def extract(self, source_path: Path) -> Dict[str, Dict[str, Any]]:
+        """Extract metadata from Excel file."""
+        try:
+            excel_stream = self._prepare_excel_file(source_path)
+            workbook = load_workbook(excel_stream, read_only=True, data_only=True)
+            try:
+                return self._process_excel_worksheet(workbook.active)
+            finally:
+                workbook.close()
+
+        except ExcelSecurityError:
+            raise
+        except ExcelParsingError:
+            raise
+        except InvalidFileException as e:
+            raise ExcelParsingError(f'Invalid Excel file format: {str(e)}')
+        except MemoryError:
+            raise ExcelSecurityError('Excel file exceeds memory limits')
+        except (OSError, IOError) as e:
+            raise ExcelParsingError(f'File access error: {str(e)}')
+        except Exception as e:
+            # Handle ZIP file errors (Excel files are ZIP archives)
+            if 'zip file' in str(e).lower():
+                raise ExcelParsingError(f'Invalid Excel file format: {str(e)}')
+            raise ExcelParsingError(f'Unexpected error: {str(e)}')
+
+    def validate(self, metadata: Dict) -> ValidationResult:
+        """Validate extracted metadata."""
+        errors = []
+
+        if not isinstance(metadata, dict):
+            errors.append('Metadata must be a dictionary')
+            return ValidationResult(valid=False, errors=errors)
+
+        # Validate each file's metadata
+        for file_name, file_metadata in metadata.items():
+            if not isinstance(file_metadata, dict):
+                errors.append(f"Metadata for file '{file_name}' must be a dictionary")
+                continue
+
+            # Check filename length
+            if not self.excel_utils.is_valid_filename_length(file_name):
+                errors.append(f"Filename '{file_name}' exceeds maximum length")
+
+        return ValidationResult(valid=len(errors) == 0, errors=errors)
+
+    def _validate_excel_security(self, excel_path: Path) -> None:
+        """Validate Excel file security constraints."""
+        file_size = excel_path.stat().st_size
+        if file_size > self.excel_config.MAX_FILE_SIZE_BYTES:
+            raise ExcelSecurityError(
+                f'Excel file too large: {file_size} bytes (max: {self.excel_config.MAX_FILE_SIZE_BYTES})'
+            )
+
+        estimated_memory = file_size * 3
+        if estimated_memory > self.excel_config.MAX_MEMORY_USAGE_BYTES:
+            raise ExcelSecurityError(
+                f'Excel file may consume too much memory: ~{estimated_memory} bytes '
+                f'(max: {self.excel_config.MAX_MEMORY_USAGE_BYTES})'
+            )
+
+    def _prepare_excel_file(self, excel_path: Path) -> BytesIO:
+        """Prepare Excel file for processing."""
+        self._validate_excel_security(excel_path)
+        excel_bytes = excel_path.read_bytes()
+        return BytesIO(excel_bytes)
+
+    def _process_excel_headers(self, headers: tuple) -> tuple:
+        """Process Excel headers."""
+        if len(headers) < 2:
+            raise ExcelParsingError('Excel file must have at least 2 columns (file name and metadata)')
+
+        # Validate first column header (filename column)
+        first_header = str(headers[0]).strip().lower() if headers[0] else ''
+        valid_filename_headers = ['filename', 'file_name']
+
+        if first_header not in valid_filename_headers:
+            raise ExcelParsingError(
+                f'First column header must be "filename" or "file_name", got: "{headers[0]}". '
+                f'Valid options: {", ".join(valid_filename_headers)}'
+            )
+
+        self._validate_excel_content(headers, 0)
+        return headers
+
+    def _process_excel_data_row(self, row: tuple, headers: tuple) -> Optional[Dict[str, Any]]:
+        """Process a single Excel data row."""
+        if not row[0] or str(row[0]).strip() == '':
+            return None
+
+        file_name = str(row[0]).strip()
+        if not self.excel_utils.is_valid_filename_length(file_name):
+            return None
+
+        file_metadata: Dict[str, Any] = {}
+        for i, value in enumerate(row[1:], start=1):
+            if i < len(headers):  # Include empty strings, exclude only None values
+                header_value = headers[i]
+                column_name = str(header_value).strip() if header_value is not None else f'column_{i}'
+
+                column_name = self.excel_utils.validate_and_truncate_string(
+                    column_name, self.excel_config.MAX_COLUMN_NAME_LENGTH
+                )
+
+                # Convert None to empty string, otherwise convert to string
+                value_str = '' if value is None else str(value)
+                str_value = self.excel_utils.validate_and_truncate_string(
+                    value_str, self.excel_config.MAX_METADATA_VALUE_LENGTH
+                )
+                file_metadata[column_name] = str_value
+
+        return {file_name: file_metadata} if file_metadata else None
+
+    def _process_excel_worksheet(self, worksheet) -> Dict[str, Dict[str, Any]]:
+        """Process Excel worksheet."""
+        if worksheet is None:
+            raise ExcelParsingError('Excel file has no active worksheet')
+
+        metadata_dict: Dict[str, Dict[str, Any]] = {}
+        headers: Optional[tuple] = None
+        data_row_count = 0
+        validation_interval = getattr(self.excel_config, 'VALIDATION_CHECK_INTERVAL', 1000)
+
+        for row_idx, row in enumerate(worksheet.iter_rows(values_only=True)):
+            # Quick check: if row is empty or first cell is empty, skip
+            if not row or not row[0] or str(row[0]).strip() == '':
+                continue
+
+            if row_idx == 0:
+                headers = self._process_excel_headers(row)
+                continue
+
+            if headers is None:
+                raise ExcelParsingError('Excel file missing header row')
+
+            data_row_count += 1
+
+            if data_row_count % validation_interval == 0:
+                self._validate_excel_content(headers, data_row_count)
+
+            row_result = self._process_excel_data_row(row, headers)
+            if row_result:
+                metadata_dict.update(row_result)
+
+        self._validate_excel_content(headers or (), data_row_count)
+
+        return metadata_dict
+
+    def _validate_excel_content(self, headers: tuple, row_count: int) -> None:
+        """Validate Excel content constraints."""
+        if len(headers) > self.excel_config.MAX_COLUMNS:
+            raise ExcelParsingError(f'Too many columns: {len(headers)} (max: {self.excel_config.MAX_COLUMNS})')
+
+        if row_count > self.excel_config.MAX_ROWS:
+            raise ExcelParsingError(f'Too many rows: {row_count} (max: {self.excel_config.MAX_ROWS})')

synapse_sdk/plugins/categories/upload/actions/upload/strategies/metadata/none.py

@@ -0,0 +1,16 @@
+from pathlib import Path
+from typing import Any, Dict
+
+from ..base import MetadataStrategy, ValidationResult
+
+
+class NoneMetadataStrategy(MetadataStrategy):
+    """Metadata strategy that returns no metadata."""
+
+    def extract(self, source_path: Path) -> Dict[str, Dict[str, Any]]:
+        """Return empty metadata."""
+        return {}
+
+    def validate(self, metadata: Dict) -> ValidationResult:
+        """Always validates successfully for empty metadata."""
+        return ValidationResult(valid=True)

synapse_sdk/plugins/categories/upload/actions/upload/strategies/upload/__init__.py

@@ -0,0 +1 @@
+# Upload strategy implementations

synapse_sdk/plugins/categories/upload/actions/upload/strategies/upload/sync.py

@@ -0,0 +1,84 @@
+from pathlib import Path
+from typing import Any, Dict, List
+
+from ...enums import LogCode, UploadStatus
+from ..base import UploadConfig, UploadStrategy
+
+
+class SyncUploadStrategy(UploadStrategy):
+    """Synchronous upload strategy."""
+
+    def __init__(self, context):
+        self.context = context
+
+    def upload(self, files: List[Dict], config: UploadConfig) -> List[Dict]:
+        """Upload files synchronously.
+
+        Integrates plugin's uploader for file processing before upload.
+        """
+        # Process files through plugin uploader if available
+        processed_files = self._process_files_with_uploader(files)
+
+        uploaded_files = []
+        client = self.context.client
+        collection_id = self.context.get_param('data_collection')
+
+        for organized_file in processed_files:
+            try:
+                use_chunked_upload = self._requires_chunked_upload(organized_file, config)
+                uploaded_data_file = client.upload_data_file(organized_file, collection_id, use_chunked_upload)
+                self.context.run.log_data_file(organized_file, UploadStatus.SUCCESS)
+                uploaded_files.append(uploaded_data_file)
+            except Exception as e:
+                self.context.run.log_data_file(organized_file, UploadStatus.FAILED)
+                self.context.run.log_message_with_code(LogCode.FILE_UPLOAD_FAILED, str(e))
+                # Continue with other files instead of failing completely
+
+        return uploaded_files
+
+    def _requires_chunked_upload(self, organized_file: Dict[str, Any], config: UploadConfig) -> bool:
+        """Determine if chunked upload is required based on file sizes."""
+        max_file_size_mb = config.chunked_threshold_mb
+        for file_path in organized_file.get('files', {}).values():
+            if isinstance(file_path, Path) and self._get_file_size_mb(file_path) > max_file_size_mb:
+                return True
+        return False
+
+    def _get_file_size_mb(self, file_path: Path) -> float:
+        """Get file size in megabytes."""
+        return file_path.stat().st_size / (1024 * 1024)
+
+    def _process_files_with_uploader(self, files: List[Dict]) -> List[Dict]:
+        """Process files through plugin uploader if available.
+
+        Args:
+            files: List of organized files to process
+
+        Returns:
+            Processed files (filtered, transformed) ready for upload
+        """
+        action = self.context._action
+        if not (hasattr(action, 'entrypoint') and action.entrypoint):
+            # No plugin uploader, return files as-is
+            return files
+
+        try:
+            use_single_path = self.context.get_param('use_single_path', True)
+            pathlib_cwd = self.context.get('pathlib_cwd')
+            uploader_path = pathlib_cwd if use_single_path else None
+
+            # Get uploader instance from plugin
+            uploader = action.get_uploader(
+                path=uploader_path,
+                file_specification=self.context.file_specifications,
+                organized_files=files,
+                params=self.context.params,
+            )
+
+            # Process files through uploader (includes extension validation, transformation, etc.)
+            return uploader.handle_upload_files()
+
+        except Exception as e:
+            self.context.run.log_message_with_code(LogCode.FILE_UPLOAD_FAILED, f'Uploader processing failed: {str(e)}')
+            # Return original files if uploader fails
+            return files

synapse_sdk/plugins/categories/upload/actions/upload/strategies/validation/__init__.py

@@ -0,0 +1 @@
+# Validation strategy implementations

synapse_sdk/plugins/categories/upload/actions/upload/strategies/validation/default.py

@@ -0,0 +1,60 @@
+from typing import Dict, List
+
+from synapse_sdk.clients.validators.collections import FileSpecificationValidator
+
+from ..base import ValidationResult, ValidationStrategy
+
+
+class DefaultValidationStrategy(ValidationStrategy):
+    """Default validation strategy for upload operations."""
+
+    def validate_params(self, params: Dict) -> ValidationResult:
+        """Validate action parameters."""
+        errors = []
+
+        # Check required parameters (common to all modes)
+        required_params = ['storage', 'data_collection', 'name']
+        for param in required_params:
+            if param not in params:
+                errors.append(f'Missing required parameter: {param}')
+
+        # Check mode-specific requirements
+        use_single_path = params.get('use_single_path', True)
+
+        if use_single_path:
+            # Single-path mode: 'path' is required
+            if 'path' not in params:
+                errors.append("Missing required parameter 'path' in single-path mode")
+        else:
+            # Multi-path mode: 'assets' is required
+            if 'assets' not in params:
+                errors.append("Missing required parameter 'assets' in multi-path mode")
+
+        # Check parameter types
+        if 'storage' in params and not isinstance(params['storage'], int):
+            errors.append("Parameter 'storage' must be an integer")
+
+        if 'data_collection' in params and not isinstance(params['data_collection'], int):
+            errors.append("Parameter 'data_collection' must be an integer")
+
+        if 'is_recursive' in params and not isinstance(params['is_recursive'], bool):
+            errors.append("Parameter 'is_recursive' must be a boolean")
+
+        if 'use_single_path' in params and not isinstance(params['use_single_path'], bool):
+            errors.append("Parameter 'use_single_path' must be a boolean")
+
+        return ValidationResult(valid=len(errors) == 0, errors=errors)
+
+    def validate_files(self, files: List[Dict], specs: Dict) -> ValidationResult:
+        """Validate organized files against specifications."""
+        try:
+            validator = FileSpecificationValidator(specs, files)
+            is_valid = validator.validate()
+
+            if is_valid:
+                return ValidationResult(valid=True)
+            else:
+                return ValidationResult(valid=False, errors=['File specification validation failed'])
+
+        except Exception as e:
+            return ValidationResult(valid=False, errors=[f'Validation error: {str(e)}'])

synapse_sdk/plugins/categories/upload/actions/upload/utils.py

@@ -0,0 +1,250 @@
+import json
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel, Field, model_validator
+
+
+class PathAwareJSONEncoder(json.JSONEncoder):
+    """Custom JSON encoder that handles Path objects and datetime objects.
+
+    Extends the default JSON encoder to properly serialize Path objects
+    and datetime objects that are commonly used in upload operations.
+
+    Supported object types:
+    - Path objects (converts to string using __fspath__ or as_posix)
+    - Datetime objects (converts using isoformat)
+    - All other standard JSON-serializable types
+
+    Example:
+        >>> data = {"path": Path("/tmp/file.txt"), "timestamp": datetime.now()}
+        >>> json.dumps(data, cls=PathAwareJSONEncoder)
+        '{"path": "/tmp/file.txt", "timestamp": "2023-01-01T12:00:00"}'
+    """
+
+    def default(self, obj):
+        if hasattr(obj, '__fspath__'):
+            return obj.__fspath__()
+        elif hasattr(obj, 'as_posix'):
+            return obj.as_posix()
+        elif hasattr(obj, 'isoformat'):
+            return obj.isoformat()
+        return super().default(obj)
+
+
+class ExcelSecurityConfig(BaseModel):
+    """Security configuration for Excel file processing using Pydantic.
+
+    Defines essential security limits for Excel file processing to prevent
+    resource exhaustion attacks and ensure safe handling of potentially malicious files.
+
+    Attributes:
+        max_file_size_mb (int): Maximum file size in megabytes (default: 10)
+        max_rows (int): Maximum number of rows allowed (default: 100000)
+        max_columns (int): Maximum number of columns allowed (default: 50)
+    """
+
+    max_file_size_mb: int = Field(
+        default=10,
+        ge=1,
+        le=1000,
+        description='Maximum file size in megabytes',
+    )
+
+    max_rows: int = Field(
+        default=100000,
+        ge=1,
+        le=100000,
+        description='Maximum number of rows allowed',
+    )
+
+    max_columns: int = Field(
+        default=50,
+        ge=1,
+        le=16384,  # Excel's column limit
+        description='Maximum number of columns allowed',
+    )
+
+    max_memory_usage_mb: int = Field(
+        default=30,
+        ge=1,
+        le=1000,
+        description='Maximum memory usage in megabytes',
+    )
+
+    max_filename_length: int = Field(
+        default=255,
+        ge=1,
+        le=1000,
+        description='Maximum filename length',
+    )
+
+    max_column_name_length: int = Field(
+        default=100,
+        ge=1,
+        le=500,
+        description='Maximum column name length',
+    )
+
+    max_metadata_value_length: int = Field(
+        default=1000,
+        ge=1,
+        le=10000,
+        description='Maximum metadata value length',
+    )
+
+    validation_check_interval: int = Field(
+        default=1000,
+        ge=100,
+        le=10000,
+        description='Validation check interval for processing',
+    )
+
+    model_config = {'validate_assignment': True, 'extra': 'forbid'}
+
+    @model_validator(mode='after')
+    def validate_resource_limits(self) -> 'ExcelSecurityConfig':
+        """Validate that resource limits are reasonable."""
+        # Check for unreasonable combinations
+        estimated_cells = self.max_rows * self.max_columns
+        if estimated_cells > 50000000:  # 50 million cells
+            raise ValueError(
+                f'Combination of max_rows ({self.max_rows}) and max_columns ({self.max_columns}) '
+                f'would allow too many cells ({estimated_cells:,})'
+            )
+
+        return self
+
+    @property
+    def max_file_size_bytes(self) -> int:
+        """Get maximum file size in bytes."""
+        return self.max_file_size_mb * 1024 * 1024
+
+    @property
+    def max_memory_usage_bytes(self) -> int:
+        """Get maximum memory usage in bytes."""
+        return self.max_memory_usage_mb * 1024 * 1024
+
+    # Backward compatibility properties (uppercase versions)
+    @property
+    def MAX_FILE_SIZE_MB(self) -> int:
+        """Backward compatibility property."""
+        return self.max_file_size_mb
+
+    @property
+    def MAX_FILE_SIZE_BYTES(self) -> int:
+        """Backward compatibility property."""
+        return self.max_file_size_bytes
+
+    @property
+    def MAX_MEMORY_USAGE_MB(self) -> int:
+        """Backward compatibility property."""
+        return self.max_memory_usage_mb
+
+    @property
+    def MAX_MEMORY_USAGE_BYTES(self) -> int:
+        """Backward compatibility property."""
+        return self.max_memory_usage_bytes
+
+    @property
+    def MAX_ROWS(self) -> int:
+        """Backward compatibility property."""
+        return self.max_rows
+
+    @property
+    def MAX_COLUMNS(self) -> int:
+        """Backward compatibility property."""
+        return self.max_columns
+
+    @property
+    def MAX_FILENAME_LENGTH(self) -> int:
+        """Backward compatibility property."""
+        return self.max_filename_length
+
+    @property
+    def MAX_COLUMN_NAME_LENGTH(self) -> int:
+        """Backward compatibility property."""
+        return self.max_column_name_length
+
+    @property
+    def MAX_METADATA_VALUE_LENGTH(self) -> int:
+        """Backward compatibility property."""
+        return self.max_metadata_value_length
+
+    @property
+    def VALIDATION_CHECK_INTERVAL(self) -> int:
+        """Backward compatibility property."""
+        return self.validation_check_interval
+
+    @classmethod
+    def from_action_config(cls, action_config: Optional[Dict[str, Any]]) -> 'ExcelSecurityConfig':
+        """Create ExcelSecurityConfig from plugin action configuration (config.yaml).
+
+        Args:
+            action_config: Action configuration dictionary from config.yaml
+
+        Returns:
+            New ExcelSecurityConfig instance with config.yaml values
+
+        Example config.yaml:
+            actions:
+              upload:
+                excel_config:
+                  max_file_size_mb: 25
+                  max_rows: 50000
+                  max_columns: 100
+        """
+        if not action_config or 'excel_config' not in action_config:
+            return cls()
+
+        excel_config = action_config['excel_config']
+
+        return cls(
+            max_file_size_mb=excel_config.get('max_file_size_mb', 10),
+            max_rows=excel_config.get('max_rows', 100000),
+            max_columns=excel_config.get('max_columns', 50),
+            max_memory_usage_mb=excel_config.get('max_memory_usage_mb', 30),
+            max_filename_length=excel_config.get('max_filename_length', 255),
+            max_column_name_length=excel_config.get('max_column_name_length', 100),
+            max_metadata_value_length=excel_config.get('max_metadata_value_length', 1000),
+            validation_check_interval=excel_config.get('validation_check_interval', 1000),
+        )
+
+
+class ExcelMetadataUtils:
+    """Utility class for Excel metadata processing."""
+
+    def __init__(self, config: ExcelSecurityConfig):
+        """Initialize with Excel security configuration."""
+        self.config = config
+
+    def is_valid_filename_length(self, filename: str) -> bool:
+        """Check if filename length is within limits."""
+        return len(filename) <= self.config.max_filename_length
+
+    def validate_and_truncate_string(self, value: str, max_length: int) -> str:
+        """Validate and truncate string to maximum length."""
+        if not isinstance(value, str):
+            value = str(value)
+
+        # Strip whitespace
+        value = value.strip()
+
+        # Truncate if too long
+        if len(value) > max_length:
+            value = value[:max_length]
+
+        return value
+
+    def is_valid_column_name(self, column_name: str) -> bool:
+        """Check if column name is valid."""
+        if not column_name or not isinstance(column_name, str):
+            return False
+        return len(column_name.strip()) <= self.config.max_column_name_length
+
+    def is_valid_metadata_value(self, value: str) -> bool:
+        """Check if metadata value is valid."""
+        if value is None:
+            return True
+        if not isinstance(value, str):
+            value = str(value)
+        return len(value) <= self.config.max_metadata_value_length