synapse-sdk 1.0.0b5__py3-none-any.whl → 2025.12.3__py3-none-any.whl

synapse_sdk/plugins/categories/upload/templates/plugin/__init__.py

@@ -0,0 +1,310 @@
+from pathlib import Path
+from typing import Dict, List
+
+
+class BaseUploader:
+    """Base class for upload plugins with common functionality.
+
+    This class handles common tasks like file organization, validation, and metadata
+    that are shared across all upload plugins. Plugin developers should inherit
+    from this class and implement the required methods for their specific logic.
+
+    Important: Plugin extensions work with already-organized files from the main upload workflow.
+    Whether single-path or multi-path mode is used is transparent to plugin developers - you
+    simply process the organized_files list provided to you.
+
+    Core Methods:
+        handle_upload_files(): Main upload method - handles the complete upload workflow
+        organize_files(): Handle file organization logic (can be overridden)
+        validate_files(): Handle file validation logic (can be overridden)
+
+    Required Methods (should be implemented by subclasses):
+        process_files(): Transform/process files during upload
+
+    Optional Methods (can be overridden by subclasses):
+        before_process(): Pre-process files before main processing
+        after_process(): Post-process files after main processing
+        setup_directories(): Setup custom directories
+        validate_file_types(): Custom file type validation
+
+    Helper Methods:
+        _log_validation_warning(): Log validation warnings
+        _log_conversion_warning(): Log conversion warnings
+        _filter_valid_files(): Filter files based on validation
+
+    Auto-provided Utilities:
+        Logging via self.run.log_message() and other run methods
+        File path utilities via self.path
+        Specification access via self.file_specification
+
+    Customization:
+        To restrict file extensions, modify get_file_extensions_config() in this file:
+
+        Example - Allow only MP4 videos:
+            def get_file_extensions_config(self):
+                return {
+                    'video': ['.mp4'],  # Only MP4 allowed
+                    'image': ['.jpg', '.png'],
+                    # ... other types
+                }
+    """
+
+    def __init__(
+        self,
+        run,
+        path: Path,
+        file_specification: List = None,
+        organized_files: List = None,
+        extra_params: Dict = None,
+    ):
+        """Initialize the base upload class.
+
+        Args:
+            run: Plugin run object with logging capabilities.
+            path: Path object pointing to the upload target directory.
+                  - In single-path mode: Base directory path (Path object)
+                  - In multi-path mode: None (not needed - use self.assets_config instead)
+                  Files have already been discovered from their respective asset paths.
+            file_specification: List of specifications that define the structure of files to be uploaded.
+            organized_files: List of pre-organized files based on the default logic.
+                            Plugin extensions work with these already-organized files regardless of
+                            whether single-path or multi-path mode was used.
+            extra_params: Additional parameters for customization.
+        """
+        self.run = run
+        self.path = path
+        self.file_specification = file_specification or []
+        self.organized_files = organized_files or []
+        self.extra_params = extra_params or {}
+
+    def get_file_extensions_config(self) -> Dict[str, List[str]]:
+        """Get allowed file extensions configuration.
+
+        Modify this dictionary to restrict file extensions per file type.
+        Extensions are case-insensitive and must include the dot prefix.
+
+        Example:
+            To allow only MP4 videos::
+
+                def get_file_extensions_config(self):
+                    return {
+                        'video': ['.mp4'],
+                        'image': ['.jpg', '.png'],
+                    }
+
+        Returns:
+            Dict[str, List[str]]: Mapping of file types to allowed extensions.
+                Each key is a file type (e.g., 'video', 'image') and each value
+                is a list of allowed extensions (e.g., ['.mp4', '.avi']).
+        """
+        # Configure allowed extensions here
+        # Extensions should include the dot (e.g., '.mp4', not 'mp4')
+        return {
+            'video': ['.mp4', '.avi', '.mov', '.mkv', '.webm', '.flv', '.wmv'],
+            'image': ['.jpg', '.jpeg', '.png'],
+            'pcd': ['.pcd'],
+            'text': ['.txt', '.html'],
+            'audio': ['.mp3', '.wav'],
+            'data': ['.xml', '.bin', '.json', '.fbx'],
+        }
+
+    def _log_validation_warning(self, spec_name: str, invalid_extensions: List[str], expected_extensions: List[str]):
+        """Log validation warning for invalid file extensions."""
+        self.run.log_message(
+            f"Validation warning in '{spec_name}': File extensions {invalid_extensions} do not match expected extensions {expected_extensions}. These files will be excluded from upload."
+        )
+
+    def _log_conversion_warning(self, spec_name: str, extension: str, recommended_formats: str):
+        """Log conversion warning for file formats that may need conversion."""
+        self.run.log_message(
+            f"Conversion warning in '{spec_name}': File extension '{extension}' may require conversion to [{recommended_formats}]."
+        )
+
+    def _filter_valid_files(self, files_to_validate: List) -> List:
+        """Filter files based on validation criteria.
+
+        Args:
+            files_to_validate: List of organized file dictionaries to validate
+
+        Returns:
+            List: Filtered list containing only valid files
+        """
+        return files_to_validate  # Default: return all files
+
+    # Abstract methods that should be implemented by subclasses
+    def process_files(self, organized_files: List) -> List:
+        """Process files. Should be implemented by subclasses."""
+        return organized_files
+
+    def before_process(self, organized_files: List) -> List:
+        """Pre-process files before main processing. Can be overridden by subclasses."""
+        return organized_files
+
+    def after_process(self, processed_files: List) -> List:
+        """Post-process files after main processing. Can be overridden by subclasses."""
+        return processed_files
+
+    def organize_files(self, files: List) -> List:
+        """Organize files. Can be overridden by subclasses."""
+        return files
+
+    def validate_files(self, files: List) -> List:
+        """Validate files against allowed extensions and custom rules.
+
+        This method first validates file types against get_file_extensions_config(),
+        then applies custom filtering via _filter_valid_files().
+
+        Override this method for complete custom validation, or override
+        _filter_valid_files() to add additional filtering after extension validation.
+        """
+        # First, validate file extensions
+        files = self.validate_file_types(files)
+
+        # Then apply custom filtering
+        return self._filter_valid_files(files)
+
+    def setup_directories(self) -> None:
+        """Setup custom directories. Can be overridden by subclasses."""
+        pass
+
+    def validate_file_types(self, organized_files: List) -> List:
+        """Validate file types against allowed extensions configuration.
+
+        Filters files based on their extensions according to get_file_extensions_config().
+        Main files (is_required=True) are uploaded if they have valid extensions,
+        even when sub files fail validation. Sub files are filtered individually.
+
+        Args:
+            organized_files (List[Dict]): List of organized file dictionaries.
+                Each dict contains a 'files' key mapping spec names to file paths.
+
+        Returns:
+            List[Dict]: Filtered list containing only files with valid extensions.
+                Groups are included if they have at least one valid main file.
+                Files with disallowed extensions are removed and logged as WARNING.
+
+        Note:
+            Extension matching is case-insensitive (.mp4 == .MP4).
+            Filtered files are logged using LogCode.FILES_FILTERED_BY_EXTENSION.
+        """
+        if not organized_files or not self.file_specification:
+            return organized_files
+
+        valid_files = []
+        allowed_extensions_config = self.get_file_extensions_config()
+        filtered_by_type = {}  # Track filtered files per type
+
+        for file_group in organized_files:
+            files_dict = file_group.get('files', {})
+            valid_files_dict = {}  # Track valid files individually
+
+            for spec_name, file_path in files_dict.items():
+                # Find the specification for this file type
+                file_spec = next((s for s in self.file_specification if s['name'] == spec_name), None)
+                if not file_spec:
+                    continue
+
+                # Handle file path lists
+                if isinstance(file_path, list):
+                    file_path = file_path[0] if file_path else None
+
+                if file_path is None:
+                    continue
+
+                # Get file type and extension
+                file_type = file_spec['file_type']
+                file_extension = file_path.suffix.lower()
+
+                # Check if this file type has allowed extensions
+                if file_type in allowed_extensions_config:
+                    allowed_exts = [ext.lower() for ext in allowed_extensions_config[file_type]]
+
+                    if file_extension not in allowed_exts:
+                        # Track filtered extension
+                        if file_type not in filtered_by_type:
+                            filtered_by_type[file_type] = {'extensions': set(), 'count': 0}
+                        filtered_by_type[file_type]['extensions'].add(
+                            file_extension if file_extension else '(no extension)'
+                        )
+                        filtered_by_type[file_type]['count'] += 1
+                        # Skip this file and continue checking others
+                        continue
+
+                # File is valid - add to valid files dict
+                valid_files_dict[spec_name] = file_path
+
+            # Include group if it has ALL required files valid
+            if valid_files_dict:
+                # Get all required spec names
+                required_specs = [spec['name'] for spec in self.file_specification if spec.get('is_required', False)]
+
+                # Check if ALL required specs are present and valid
+                has_all_required = all(spec_name in valid_files_dict for spec_name in required_specs)
+
+                if has_all_required:
+                    # Create new file group with only valid files
+                    # Preserve all fields from original file_group (e.g., 'group', 'meta', custom fields)
+                    # Only replace 'files' with the validated files
+                    valid_group = {**file_group, 'files': valid_files_dict}
+                    valid_files.append(valid_group)
+
+        # Log filtered files by type
+        self._log_filtered_files(filtered_by_type, allowed_extensions_config)
+
+        return valid_files
+
+    def _log_filtered_files(self, filtered_by_type: Dict, allowed_config: Dict):
+        """Log filtered files by type with detailed information.
+
+        Args:
+            filtered_by_type (Dict[str, Dict]): Filtered file information per type.
+                Each entry contains 'extensions' (set) and 'count' (int).
+            allowed_config (Dict[str, List[str]]): The allowed extensions configuration
+                mapping file types to allowed extension lists.
+        """
+        from synapse_sdk.plugins.categories.upload.actions.upload.enums import LogCode
+
+        for file_type, info in filtered_by_type.items():
+            if info['count'] > 0:
+                extensions_str = ', '.join(sorted(info['extensions']))
+                allowed_str = ', '.join(allowed_config.get(file_type, []))
+                self.run.log_message_with_code(
+                    LogCode.FILES_FILTERED_BY_EXTENSION,
+                    info['count'],
+                    file_type,
+                    extensions_str,
+                    allowed_str,
+                )
+
+    def handle_upload_files(self) -> List:
+        """Main upload method that handles the complete upload workflow.
+
+        This method provides the core workflow for upload plugins:
+        setup_directories -> organize_files -> before_process -> process_files ->
+        after_process -> validate_files
+
+        Returns:
+            List: The final processed and validated list of files ready for upload.
+        """
+        # Setup any required directories
+        self.setup_directories()
+
+        # Start with organized files from the workflow
+        current_files = self.organized_files
+
+        # Apply organization logic
+        current_files = self.organize_files(current_files)
+
+        # Pre-process files
+        current_files = self.before_process(current_files)
+
+        # Main processing step
+        current_files = self.process_files(current_files)
+
+        # Post-process files
+        current_files = self.after_process(current_files)
+
+        # Final validation
+        current_files = self.validate_files(current_files)
+
+        return current_files

synapse_sdk/plugins/categories/upload/templates/plugin/upload.py

@@ -1,40 +1,102 @@
 from pathlib import Path
-from typing import List
+from typing import Any, Dict, List
 
+from . import BaseUploader
 
-class Uploader:
+
+class Uploader(BaseUploader):
     """Plugin upload action interface for organizing files.
 
-    This class provides a minimal interface for plugin developers to implement
-    their own file organization logic.
+    This class provides a template for plugin developers to implement
+    their own file organization logic by inheriting from BaseUploader.
+
+    Important: This plugin extension works with already-organized files from the
+    main upload workflow. Files are provided via organized_files parameter regardless
+    of whether single-path or multi-path mode was used in the upload configuration.
+
+    Example usage:
+        Override process_files() to implement custom file processing logic.
+        Override validate_file_types() to implement custom validation rules.
+        Override setup_directories() to create custom directory structures.
     """
 
-    def __init__(self, run, path: Path, file_specification: List = None, organized_files: List = None):
-        """Initialize the plugin upload action class.
+    def __init__(
+        self, run, path: Path, file_specification: List = None, organized_files: List = None, extra_params: Dict = None
+    ):
+        """Initialize the uploader with required parameters.
 
         Args:
             run: Plugin run object with logging capabilities.
             path: Path object pointing to the upload target directory.
+                  - In single-path mode: Base directory path (Path object)
+                  - In multi-path mode: None (use self.assets_config instead)
             file_specification: List of specifications that define the structure of files to be uploaded.
-                Each specification contains details like file name, type, and requirements.
+            organized_files: List of pre-organized files from the main upload workflow.
+                            Works transparently with both single-path and multi-path modes.
+            extra_params: Additional parameters for customization.
+        """
+        super().__init__(run, path, file_specification, organized_files, extra_params)
+
+    def process_files(self, organized_files: List) -> List:
+        """Process and transform files during upload.
+
+        Override this method to implement custom file processing logic.
+        This is the main method where plugin-specific logic should be implemented.
+
+        Args:
+            organized_files: List of organized file dictionaries from the workflow.
+
+        Returns:
+            List: The processed list of files ready for upload.
+        """
+        # Default implementation: return files as-is
+        # Plugin developers should override this method for custom logic
+        return organized_files
+
+    def validate_file_types(self, organized_files: List) -> List:
+        """Validate file types against specifications.
+
+        This example shows how to use the BaseUploader's comprehensive validation logic.
+        You can override this method for custom validation or call super() to use the base implementation.
+
+        Args:
+            organized_files: List of organized file dictionaries to validate.
+
+        Returns:
+            List: Filtered list containing only valid files that match specifications.
+        """
+        return super().validate_file_types(organized_files)
+
+    def handle_upload_files(self) -> List[Dict[str, Any]]:
+        """Executes the upload task using the base class implementation.
+
+        Returns:
+            List: The final list of organized files ready for upload
+        """
+        return super().handle_upload_files()
+
+    def organize_files(self, organized_files: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Transform and organize files based on plugin logic.
+
+        Override this method to implement custom file organization logic.
+
+        Args:
+            organized_files: List of organized files from the default logic
+
+        Returns:
+            List of transformed organized files
         """
-        self.run = run
-        self.path = path
-        self.file_specification = file_specification
-        self.organized_files = organized_files
+        return organized_files
 
-    def handle_upload_files(self) -> List:
-        """Customize the organization of files for upload.
+    def filter_files(self, organized_file: Dict[str, Any]) -> bool:
+        """Filter files based on custom criteria.
 
-        This method provides a hook for plugin developers to modify the default file organization.
-        You can override this method to filter files, transform data, or add custom metadata
-        based on your specific requirements.
+        Override this method to implement custom filtering logic.
 
         Args:
-            organized_files (List): The default organized files structure.
-                Each item is a dictionary with 'files' and 'meta' keys.
+            organized_file: Single organized file to filter
 
         Returns:
-            List: The modified list of organized files to be uploaded.
+            bool: True to include the file, False to filter it out
         """
-        return self.organized_files
+        return True

synapse_sdk/plugins/models.py

@@ -1,11 +1,15 @@
 import os
+from datetime import datetime
 from functools import cached_property
 from typing import Any, Dict
 
+from pydantic import BaseModel
+
 from synapse_sdk.clients.backend import BackendClient
 from synapse_sdk.devtools.config import get_backend_config
 from synapse_sdk.loggers import BackendLogger, ConsoleLogger
 from synapse_sdk.plugins.utils import read_plugin_config
+from synapse_sdk.shared import needs_sentry_init
 from synapse_sdk.shared.enums import Context
 from synapse_sdk.utils.storage import get_storage
 from synapse_sdk.utils.string import hash_text
@@ -48,8 +52,26 @@ class PluginRelease:
 
     @cached_property
     def package_manager_options(self):
-        options = {'pip': {}, 'uv': {'uv_pip_install_options': []}}
-        return options[self.package_manager]
+        # Get user-defined options from config
+        user_options = self.config.get('package_manager_options', [])
+
+        if self.package_manager == 'uv':
+            defaults = ['--no-cache']
+            # Add defaults if not already present
+            options_list = defaults.copy()
+            for option in user_options:
+                if option not in options_list:
+                    options_list.append(option)
+            return {'uv_pip_install_options': options_list}
+        else:
+            # For pip, use pip_install_options with --upgrade flag to ensure
+            # packages from requirements.txt (like synapse-sdk) override pre-installed versions
+            defaults = ['--upgrade']
+            options_list = defaults.copy()
+            for option in user_options:
+                if option not in options_list:
+                    options_list.append(option)
+            return {'pip_install_options': options_list}
 
     @cached_property
     def checksum(self):
@@ -68,6 +90,11 @@ class PluginRelease:
         def warm_up():
             pass
 
+        extra_runtime_env = {}
+
+        if needs_sentry_init():
+            extra_runtime_env['worker_process_setup_hook'] = 'synapse_sdk.shared.worker_process_setup_hook'
+
         nodes = list_nodes(address=self.envs['RAY_DASHBOARD_URL'])
         node_ids = [n['node_id'] for n in nodes]
         for node_id in node_ids:
@@ -80,6 +107,7 @@ class PluginRelease:
                         ** self.package_manager_options
                     },
                     'working_dir': self.get_url(self.envs['SYNAPSE_PLUGIN_STORAGE']),
+                    **extra_runtime_env,
                 },
                 scheduling_strategy=strategy,
             ).remote()
@@ -110,9 +138,29 @@ class Run:
     context = None
     client = None
 
-    def __init__(self, job_id, context):
+    class DevLog(BaseModel):
+        """Model for developer log entries.
+
+        Records custom events and information that plugin developers want to track
+        during plugin execution for debugging and monitoring purposes.
+
+        Attributes:
+            event_type (str): Type/category of the development event
+            message (str): Descriptive message about the event
+            data (dict | None): Optional additional data/context
+            level (Context): Event status/severity level
+            created (str): Timestamp when event occurred
+        """
+
+        event_type: str
+        message: str
+        data: dict | None = None
+        level: Context
+        created: str
+
+    def __init__(self, job_id, context=None):
         self.job_id = job_id
-        self.context = context
+        self.context = context or {}
         config = get_backend_config()
         if config:
             self.client = BackendClient(
@@ -120,17 +168,23 @@ class Run:
                 access_token=config['token'],
             )
         else:
+            # Handle missing environment variables for test environments
+            envs = self.context.get('envs', {})
+            host = envs.get('SYNAPSE_PLUGIN_RUN_HOST', os.getenv('SYNAPSE_PLUGIN_RUN_HOST', 'http://localhost:8000'))
+            token = envs.get('SYNAPSE_PLUGIN_RUN_USER_TOKEN', os.getenv('SYNAPSE_PLUGIN_RUN_USER_TOKEN'))
+            tenant = envs.get('SYNAPSE_PLUGIN_RUN_TENANT', os.getenv('SYNAPSE_PLUGIN_RUN_TENANT'))
+
             self.client = BackendClient(
-                self.context['envs']['SYNAPSE_PLUGIN_RUN_HOST'],
-                token=self.context['envs'].get('SYNAPSE_PLUGIN_RUN_USER_TOKEN'),
-                tenant=self.context['envs'].get('SYNAPSE_PLUGIN_RUN_TENANT'),
+                host,
+                token=token,
+                tenant=tenant,
             )
         self.set_logger()
 
     def set_logger(self):
         kwargs = {
-            'progress_categories': self.context['progress_categories'],
-            'metrics_categories': self.context['metrics_categories'],
+            'progress_categories': self.context.get('progress_categories'),
+            'metrics_categories': self.context.get('metrics_categories'),
         }
 
         if self.job_id:
@@ -141,6 +195,17 @@ class Run:
     def set_progress(self, current, total, category=''):
         self.logger.set_progress(current, total, category)
 
+    def set_progress_failed(self, category: str | None = None):
+        """Mark progress as failed with elapsed time but no completion.
+
+        This method should be called when an operation fails to indicate that
+        no progress was made, but still track how long the operation ran before failing.
+
+        Args:
+            category (str | None): progress category
+        """
+        self.logger.set_progress_failed(category)
+
     def set_metrics(self, value: Dict[Any, Any], category: str):
         self.logger.set_metrics(value, category)
 
@@ -150,5 +215,42 @@ class Run:
     def log_message(self, message, context=Context.INFO.value):
         self.logger.log('message', {'context': context, 'content': message})
 
+    def log_dev_event(self, message: str, data: dict | None = None, level: Context = Context.INFO):
+        """Log development event for plugin developers.
+
+        This function allows plugin developers to log custom events and information
+        during plugin execution for debugging, monitoring, and development purposes.
+        The event_type is automatically constructed as '{action_name}_dev_log' and cannot
+        be modified by plugin developers.
+
+        Args:
+            message (str): Descriptive message about the event
+            data (dict | None): Optional additional data or context to include
+            level (Context): Event severity level (INFO, WARNING, DANGER, SUCCESS)
+
+        Example:
+            >>> run = Run(job_id, context)
+            >>> run.log_dev_event('Data validation completed', {'records_count': 100})
+            >>> run.log_dev_event('Processing time recorded', {'duration_ms': 1500})
+            >>> run.log_dev_event('Variable state at checkpoint', {'variable_x': 42}, level=Context.WARNING)
+        """
+        # Construct event_type from action name - this cannot be modified by developers
+        action_name = self.context.get('action_name', 'unknown')
+        event_type = f'{action_name}_dev_log'
+
+        # Log the structured event for development tracking only
+        # Do NOT use log_message to avoid showing debug logs to end users
+        now = datetime.now().isoformat()
+        self.log(
+            'dev_event',
+            self.DevLog(
+                event_type=event_type,
+                message=message,
+                data=data,
+                level=level,
+                created=now,
+            ).model_dump(),
+        )
+
     def end_log(self):
         self.log_message('Plugin run is complete.')

synapse_sdk/plugins/templates/plugin-config-schema.json

@@ -51,6 +51,13 @@
       "enum": ["pip", "uv"],
       "default": "pip"
     },
+    "package_manager_options": {
+      "type": "array",
+      "description": "User-provided options for package manager install command. Currently only supported for uv in Ray 2.44.1 (defaults to ['--no-cache']). pip_install_options requires newer Ray versions.",
+      "items": {
+        "type": "string"
+      }
+    },
     "data_type": {
       "type": "string",
       "description": "Primary data type the plugin works with",

synapse_sdk/plugins/templates/schema.json

@@ -44,6 +44,13 @@
         "enum": ["pip", "uv"],
         "default": "pip"
       },
+      "package_manager_options": {
+        "type": "array",
+        "description": "User-provided options for package manager install command. Currently only supported for uv in Ray 2.44.1 (defaults to ['--no-cache']). pip_install_options requires newer Ray versions.",
+        "items": {
+          "type": "string"
+        }
+      },
       "data_type": {
         "type": "string",
         "description": "Primary data type handled by the plugin",

synapse_sdk/plugins/utils/__init__.py

@@ -15,6 +15,7 @@ from .config import (
 
 # Import legacy functions for backward compatibility
 from .legacy import read_requirements, run_plugin
+from .ray_gcs import convert_http_to_ray_gcs
 from .registry import (
     get_category_display_name,
     get_plugin_categories,
@@ -37,6 +38,8 @@ __all__ = [
     'get_plugin_categories',
     'is_valid_category',
     'get_category_display_name',
+    # Ray utilities
+    'convert_http_to_ray_gcs',
     # Legacy utilities for backward compatibility
     'read_requirements',
     'run_plugin',