synapse-sdk 1.0.0b24__py3-none-any.whl → 2025.9.3__py3-none-any.whl

synapse_sdk/plugins/categories/upload/actions/upload/utils.py

@@ -1,5 +1,7 @@
 import json
-import os
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel, Field, model_validator
 
 
 class PathAwareJSONEncoder(json.JSONEncoder):
@@ -29,111 +31,220 @@ class PathAwareJSONEncoder(json.JSONEncoder):
         return super().default(obj)
 
 
-class ExcelSecurityConfig:
-    """Configuration class for Excel file security limits.
+class ExcelSecurityConfig(BaseModel):
+    """Security configuration for Excel file processing using Pydantic.
 
-    Manages security constraints for Excel file processing to prevent
-    resource exhaustion and security vulnerabilities. All limits can
-    be configured via environment variables.
+    Defines essential security limits for Excel file processing to prevent
+    resource exhaustion attacks and ensure safe handling of potentially malicious files.
 
     Attributes:
-        MAX_FILE_SIZE_MB (int): Maximum file size in megabytes
-        MAX_FILE_SIZE_BYTES (int): Maximum file size in bytes
-        MAX_MEMORY_USAGE_MB (int): Maximum memory usage in megabytes
-        MAX_MEMORY_USAGE_BYTES (int): Maximum memory usage in bytes
-        MAX_ROWS (int): Maximum number of rows allowed
-        MAX_COLUMNS (int): Maximum number of columns allowed
-        MAX_FILENAME_LENGTH (int): Maximum filename length
-        MAX_COLUMN_NAME_LENGTH (int): Maximum column name length
-        MAX_METADATA_VALUE_LENGTH (int): Maximum metadata value length
-
-    Environment Variables:
-        EXCEL_MAX_FILE_SIZE_MB: Override default file size limit (default: 10)
-        EXCEL_MAX_MEMORY_MB: Override default memory limit (default: 30)
-        EXCEL_MAX_ROWS: Override default row limit (default: 10000)
-        EXCEL_MAX_COLUMNS: Override default column limit (default: 50)
-        EXCEL_MAX_FILENAME_LENGTH: Override filename length limit (default: 255)
-        EXCEL_MAX_COLUMN_NAME_LENGTH: Override column name length (default: 100)
-        EXCEL_MAX_METADATA_VALUE_LENGTH: Override metadata value length (default: 1000)
+        max_file_size_mb (int): Maximum file size in megabytes (default: 10)
+        max_rows (int): Maximum number of rows allowed (default: 100000)
+        max_columns (int): Maximum number of columns allowed (default: 50)
     """
 
-    def __init__(self):
-        self.MAX_FILE_SIZE_MB = int(os.getenv('EXCEL_MAX_FILE_SIZE_MB', '10'))
-        self.MAX_FILE_SIZE_BYTES = self.MAX_FILE_SIZE_MB * 1024 * 1024
-
-        self.MAX_MEMORY_USAGE_MB = int(os.getenv('EXCEL_MAX_MEMORY_MB', '30'))
-        self.MAX_MEMORY_USAGE_BYTES = self.MAX_MEMORY_USAGE_MB * 1024 * 1024
+    max_file_size_mb: int = Field(
+        default=10,
+        ge=1,
+        le=1000,
+        description='Maximum file size in megabytes',
+    )
+
+    max_rows: int = Field(
+        default=100000,
+        ge=1,
+        le=100000,
+        description='Maximum number of rows allowed',
+    )
+
+    max_columns: int = Field(
+        default=50,
+        ge=1,
+        le=16384,  # Excel's column limit
+        description='Maximum number of columns allowed',
+    )
+
+    max_memory_usage_mb: int = Field(
+        default=30,
+        ge=1,
+        le=1000,
+        description='Maximum memory usage in megabytes',
+    )
+
+    max_filename_length: int = Field(
+        default=255,
+        ge=1,
+        le=1000,
+        description='Maximum filename length',
+    )
+
+    max_column_name_length: int = Field(
+        default=100,
+        ge=1,
+        le=500,
+        description='Maximum column name length',
+    )
+
+    max_metadata_value_length: int = Field(
+        default=1000,
+        ge=1,
+        le=10000,
+        description='Maximum metadata value length',
+    )
+
+    validation_check_interval: int = Field(
+        default=1000,
+        ge=100,
+        le=10000,
+        description='Validation check interval for processing',
+    )
+
+    model_config = {'validate_assignment': True, 'extra': 'forbid'}
+
+    @model_validator(mode='after')
+    def validate_resource_limits(self) -> 'ExcelSecurityConfig':
+        """Validate that resource limits are reasonable."""
+        # Check for unreasonable combinations
+        estimated_cells = self.max_rows * self.max_columns
+        if estimated_cells > 50000000:  # 50 million cells
+            raise ValueError(
+                f'Combination of max_rows ({self.max_rows}) and max_columns ({self.max_columns}) '
+                f'would allow too many cells ({estimated_cells:,})'
+            )
+
+        return self
+
+    @property
+    def max_file_size_bytes(self) -> int:
+        """Get maximum file size in bytes."""
+        return self.max_file_size_mb * 1024 * 1024
+
+    @property
+    def max_memory_usage_bytes(self) -> int:
+        """Get maximum memory usage in bytes."""
+        return self.max_memory_usage_mb * 1024 * 1024
+
+    # Backward compatibility properties (uppercase versions)
+    @property
+    def MAX_FILE_SIZE_MB(self) -> int:
+        """Backward compatibility property."""
+        return self.max_file_size_mb
+
+    @property
+    def MAX_FILE_SIZE_BYTES(self) -> int:
+        """Backward compatibility property."""
+        return self.max_file_size_bytes
+
+    @property
+    def MAX_MEMORY_USAGE_MB(self) -> int:
+        """Backward compatibility property."""
+        return self.max_memory_usage_mb
+
+    @property
+    def MAX_MEMORY_USAGE_BYTES(self) -> int:
+        """Backward compatibility property."""
+        return self.max_memory_usage_bytes
+
+    @property
+    def MAX_ROWS(self) -> int:
+        """Backward compatibility property."""
+        return self.max_rows
+
+    @property
+    def MAX_COLUMNS(self) -> int:
+        """Backward compatibility property."""
+        return self.max_columns
+
+    @property
+    def MAX_FILENAME_LENGTH(self) -> int:
+        """Backward compatibility property."""
+        return self.max_filename_length
+
+    @property
+    def MAX_COLUMN_NAME_LENGTH(self) -> int:
+        """Backward compatibility property."""
+        return self.max_column_name_length
+
+    @property
+    def MAX_METADATA_VALUE_LENGTH(self) -> int:
+        """Backward compatibility property."""
+        return self.max_metadata_value_length
+
+    @property
+    def VALIDATION_CHECK_INTERVAL(self) -> int:
+        """Backward compatibility property."""
+        return self.validation_check_interval
+
+    @classmethod
+    def from_action_config(cls, action_config: Optional[Dict[str, Any]]) -> 'ExcelSecurityConfig':
+        """Create ExcelSecurityConfig from plugin action configuration (config.yaml).
 
-        self.MAX_ROWS = int(os.getenv('EXCEL_MAX_ROWS', '10000'))
-        self.MAX_COLUMNS = int(os.getenv('EXCEL_MAX_COLUMNS', '50'))
-
-        self.MAX_FILENAME_LENGTH = int(os.getenv('EXCEL_MAX_FILENAME_LENGTH', '255'))
-        self.MAX_COLUMN_NAME_LENGTH = int(os.getenv('EXCEL_MAX_COLUMN_NAME_LENGTH', '100'))
-        self.MAX_METADATA_VALUE_LENGTH = int(os.getenv('EXCEL_MAX_METADATA_VALUE_LENGTH', '1000'))
+        Args:
+            action_config: Action configuration dictionary from config.yaml
 
+        Returns:
+            New ExcelSecurityConfig instance with config.yaml values
+
+        Example config.yaml:
+            actions:
+              upload:
+                excel_config:
+                  max_file_size_mb: 25
+                  max_rows: 50000
+                  max_columns: 100
+        """
+        if not action_config or 'excel_config' not in action_config:
+            return cls()
 
-class ExcelMetadataUtils:
-    """Utility class for Excel metadata processing and validation.
+        excel_config = action_config['excel_config']
 
-    Provides helper methods for validating and processing Excel metadata
-    while respecting security constraints defined in ExcelSecurityConfig.
+        return cls(
+            max_file_size_mb=excel_config.get('max_file_size_mb', 10),
+            max_rows=excel_config.get('max_rows', 100000),
+            max_columns=excel_config.get('max_columns', 50),
+            max_memory_usage_mb=excel_config.get('max_memory_usage_mb', 30),
+            max_filename_length=excel_config.get('max_filename_length', 255),
+            max_column_name_length=excel_config.get('max_column_name_length', 100),
+            max_metadata_value_length=excel_config.get('max_metadata_value_length', 1000),
+            validation_check_interval=excel_config.get('validation_check_interval', 1000),
+        )
 
-    Args:
-        config (ExcelSecurityConfig): Security configuration instance
 
-    Example:
-        >>> config = ExcelSecurityConfig()
-        >>> utils = ExcelMetadataUtils(config)
-        >>> safe_value = utils.validate_and_truncate_string("long text", 10)
-        >>> is_valid = utils.is_valid_filename_length("file.xlsx")
-    """
+class ExcelMetadataUtils:
+    """Utility class for Excel metadata processing."""
 
     def __init__(self, config: ExcelSecurityConfig):
+        """Initialize with Excel security configuration."""
         self.config = config
 
-    def validate_and_truncate_string(self, value: str, max_length: int) -> str:
-        """Validate and truncate string to maximum length.
-
-        Converts non-string values to strings, trims whitespace, and
-        truncates to the specified maximum length if necessary.
-
-        Args:
-            value (str): Value to validate and truncate
-            max_length (int): Maximum allowed length
-
-        Returns:
-            str: Validated and truncated string
+    def is_valid_filename_length(self, filename: str) -> bool:
+        """Check if filename length is within limits."""
+        return len(filename) <= self.config.max_filename_length
 
-        Example:
-            >>> utils.validate_and_truncate_string("  long text  ", 5)
-            'long '
-        """
+    def validate_and_truncate_string(self, value: str, max_length: int) -> str:
+        """Validate and truncate string to maximum length."""
         if not isinstance(value, str):
             value = str(value)
 
+        # Strip whitespace
         value = value.strip()
 
+        # Truncate if too long
         if len(value) > max_length:
-            return value[:max_length]
+            value = value[:max_length]
 
         return value
 
-    def is_valid_filename_length(self, filename: str) -> bool:
-        """Check if filename length is within security limits.
-
-        Validates that the filename (after trimming whitespace) does not
-        exceed the maximum filename length configured in security settings.
+    def is_valid_column_name(self, column_name: str) -> bool:
+        """Check if column name is valid."""
+        if not column_name or not isinstance(column_name, str):
+            return False
+        return len(column_name.strip()) <= self.config.max_column_name_length
 
-        Args:
-            filename (str): Filename to validate
-
-        Returns:
-            bool: True if filename length is valid, False otherwise
-
-        Example:
-            >>> utils.is_valid_filename_length("file.xlsx")
-            True
-            >>> utils.is_valid_filename_length("x" * 300)
-            False
-        """
-        return len(filename.strip()) <= self.config.MAX_FILENAME_LENGTH
+    def is_valid_metadata_value(self, value: str) -> bool:
+        """Check if metadata value is valid."""
+        if value is None:
+            return True
+        if not isinstance(value, str):
+            value = str(value)
+        return len(value) <= self.config.max_metadata_value_length

synapse_sdk/plugins/categories/upload/templates/config.yaml

@@ -3,6 +3,10 @@ actions:
     entrypoint: plugin.upload.Uploader
     options:
     supported_data_type: image # A primary data type of synapse backend collection. (e.g. 'image', 'text', 'video', 'pcd', 'audio')
+    excel_config: # Configuration for Excel file uploads
+      max_file_size_mb: 10
+      max_rows: 100000
+      max_columns: 50
     ui_schema: # UI schema for the input of extra params
       - $formkit: "radio"
         name: "file_format"

synapse_sdk/plugins/categories/upload/templates/plugin/__init__.py

@@ -0,0 +1,269 @@
+from pathlib import Path
+from typing import Dict, List
+
+
+class BaseUploader:
+    """Base class for upload plugins with common functionality.
+
+    This class handles common tasks like file organization, validation, and metadata
+    that are shared across all upload plugins. Plugin developers should inherit
+    from this class and implement the required methods for their specific logic.
+
+    Core Methods:
+        handle_upload_files(): Main upload method - handles the complete upload workflow
+        organize_files(): Handle file organization logic (can be overridden)
+        validate_files(): Handle file validation logic (can be overridden)
+
+    Required Methods (should be implemented by subclasses):
+        process_files(): Transform/process files during upload
+
+    Optional Methods (can be overridden by subclasses):
+        before_process(): Pre-process files before main processing
+        after_process(): Post-process files after main processing
+        setup_directories(): Setup custom directories
+        validate_file_types(): Custom file type validation
+
+    Helper Methods:
+        _log_validation_warning(): Log validation warnings
+        _log_conversion_warning(): Log conversion warnings
+        _filter_valid_files(): Filter files based on validation
+
+    Auto-provided Utilities:
+        Logging via self.run.log_message() and other run methods
+        File path utilities via self.path
+        Specification access via self.file_specification
+    """
+
+    def __init__(
+        self,
+        run,
+        path: Path,
+        file_specification: List = None,
+        organized_files: List = None,
+        extra_params: Dict = None,
+    ):
+        """Initialize the base upload class.
+
+        Args:
+            run: Plugin run object with logging capabilities.
+            path: Path object pointing to the upload target directory.
+            file_specification: List of specifications that define the structure of files to be uploaded.
+            organized_files: List of pre-organized files based on the default logic.
+            extra_params: Additional parameters for customization.
+        """
+        self.run = run
+        self.path = path
+        self.file_specification = file_specification or []
+        self.organized_files = organized_files or []
+        self.extra_params = extra_params or {}
+
+    def _log_validation_warning(self, spec_name: str, invalid_extensions: List[str], expected_extensions: List[str]):
+        """Log validation warning for invalid file extensions."""
+        self.run.log_message(
+            f"Validation warning in '{spec_name}': File extensions {invalid_extensions} do not match expected extensions {expected_extensions}. These files will be excluded from upload."
+        )
+
+    def _log_conversion_warning(self, spec_name: str, extension: str, recommended_formats: str):
+        """Log conversion warning for file formats that may need conversion."""
+        self.run.log_message(
+            f"Conversion warning in '{spec_name}': File extension '{extension}' may require conversion to [{recommended_formats}]."
+        )
+
+    def _filter_valid_files(self, files_to_validate: List) -> List:
+        """Filter files based on validation criteria.
+
+        Args:
+            files_to_validate: List of organized file dictionaries to validate
+
+        Returns:
+            List: Filtered list containing only valid files
+        """
+        return files_to_validate  # Default: return all files
+
+    def get_file_extensions_config(self) -> Dict[str, List[str]]:
+        """Get allowed file extensions configuration.
+
+        Returns:
+            Dict mapping file categories to allowed extensions
+        """
+        return {
+            'pcd': ['.pcd'],
+            'text': ['.txt', '.html'],
+            'audio': ['.wav', '.mp3'],
+            'data': ['.bin', '.json', '.fbx'],
+            'image': ['.jpg', '.jpeg', '.png'],
+            'video': ['.mp4'],
+        }
+
+    def get_conversion_warnings_config(self) -> Dict[str, str]:
+        """Get file conversion warnings configuration.
+
+        Returns:
+            Dict mapping problematic extensions to recommended formats
+        """
+        return {
+            '.tif': ' .jpg, .png',
+            '.tiff': ' .jpg, .png',
+            '.avi': ' .mp4',
+            '.mov': ' .mp4',
+            '.mkv': ' .mp4',
+            '.wmv': ' .mp4',
+        }
+
+    # Abstract methods that should be implemented by subclasses
+    def process_files(self, organized_files: List) -> List:
+        """Process files. Should be implemented by subclasses."""
+        return organized_files
+
+    def before_process(self, organized_files: List) -> List:
+        """Pre-process files before main processing. Can be overridden by subclasses."""
+        return organized_files
+
+    def after_process(self, processed_files: List) -> List:
+        """Post-process files after main processing. Can be overridden by subclasses."""
+        return processed_files
+
+    def organize_files(self, files: List) -> List:
+        """Organize files. Can be overridden by subclasses."""
+        return files
+
+    def validate_files(self, files: List) -> List:
+        """Validate files. Can be overridden by subclasses."""
+        return self._filter_valid_files(files)
+
+    def setup_directories(self) -> None:
+        """Setup custom directories. Can be overridden by subclasses."""
+        pass
+
+    def validate_file_types(self, organized_files: List) -> List:
+        """Validate file types against specifications with comprehensive filtering logic.
+
+        This method implements the complete validation logic from legacy code,
+        filtering out files that don't match their expected specifications.
+
+        Args:
+            organized_files: List of organized file dictionaries
+
+        Returns:
+            List: Filtered list containing only valid files that match specifications
+        """
+        if not organized_files or not self.file_specification:
+            return organized_files
+
+        valid_files = []
+        allowed_extensions = self.get_file_extensions_config()
+        conversion_warnings = self.get_conversion_warnings_config()
+        warning_extensions = list(conversion_warnings.keys())
+        all_violation_case = {}
+
+        for file_group in organized_files:
+            files_dict = file_group.get('files', {})
+            invalid_case = {}
+            warning_case = {}
+
+            for spec_name, file_path in files_dict.items():
+                # Find the specification for this file type
+                file_spec = next((s for s in self.file_specification if s['name'] == spec_name), None)
+                if not file_spec:
+                    continue
+
+                # Handle file path lists
+                if isinstance(file_path, list):
+                    file_path = file_path[0] if len(file_path) == 1 else file_path
+
+                # Extract file information
+                file_category = spec_name.split('_')[0]
+                file_type = file_spec['file_type']
+                file_extension = file_path.suffix.lower()
+
+                # Check if file needs conversion warning (these files will be excluded)
+                if file_extension in warning_extensions:
+                    case = invalid_case.get(spec_name, {})
+                    case['warning'] = case.get('warning', []) + [file_extension]
+                    warning_case[spec_name] = case
+                    break
+
+                # Validate against file category (e.g., 'image', 'data', etc.)
+                if file_category in allowed_extensions.keys():
+                    if file_extension in allowed_extensions[file_category]:
+                        continue  # Valid file
+                    else:
+                        case = invalid_case.get(spec_name, {})
+                        case['invalid'] = case.get('invalid', []) + [file_extension]
+                        case['expected'] = allowed_extensions[file_category]
+                        invalid_case[spec_name] = case
+                        break
+
+                # Validate against file type from specification
+                if file_type in allowed_extensions.keys():
+                    if file_extension in allowed_extensions[file_type]:
+                        continue  # Valid file
+                    else:
+                        case = invalid_case.get(spec_name, {})
+                        case['invalid'] = case.get('invalid', []) + [file_extension]
+                        case['expected'] = allowed_extensions[file_type]
+                        invalid_case[spec_name] = case
+                        break
+
+            # If violations found, exclude this file group
+            if invalid_case or warning_case:
+                all_violation_case[spec_name] = {
+                    'invalid': invalid_case.get(spec_name, {}),
+                    'warning': warning_case.get(spec_name, {}),
+                }
+                continue  # Skip this file group
+
+            # No violations - add to valid files
+            valid_files.append(file_group)
+
+        # Log all violations found during validation
+        self._log_all_violations(all_violation_case, conversion_warnings)
+
+        return valid_files
+
+    def _log_all_violations(self, all_violation_case: Dict, conversion_warnings: Dict):
+        """Log all validation violations found during file validation."""
+        for spec_name, violation_info in all_violation_case.items():
+            if violation_info['invalid']:
+                self.run.log_message(
+                    f"Validation warning in '{spec_name}': File extensions {violation_info['invalid']['invalid']} do not match expected extensions {violation_info['invalid']['expected']}. These files will be excluded from upload."
+                )
+            if violation_info['warning']:
+                for warning in violation_info['warning']['warning']:
+                    if warning in conversion_warnings:
+                        self.run.log_message(
+                            f"Conversion warning in '{spec_name}': File extension '{warning}' may require conversion to [{conversion_warnings[warning]}]."
+                        )
+
+    def handle_upload_files(self) -> List:
+        """Main upload method that handles the complete upload workflow.
+
+        This method provides the core workflow for upload plugins:
+        setup_directories -> organize_files -> before_process -> process_files ->
+        after_process -> validate_files
+
+        Returns:
+            List: The final processed and validated list of files ready for upload.
+        """
+        # Setup any required directories
+        self.setup_directories()
+
+        # Start with organized files from the workflow
+        current_files = self.organized_files
+
+        # Apply organization logic
+        current_files = self.organize_files(current_files)
+
+        # Pre-process files
+        current_files = self.before_process(current_files)
+
+        # Main processing step
+        current_files = self.process_files(current_files)
+
+        # Post-process files
+        current_files = self.after_process(current_files)
+
+        # Final validation
+        current_files = self.validate_files(current_files)
+
+        return current_files