synapse-sdk 1.0.0a35__py3-none-any.whl → 2025.11.7__py3-none-any.whl

synapse_sdk/plugins/categories/base.py

@@ -11,7 +11,8 @@ from synapse_sdk.clients.ray import RayClient
 from synapse_sdk.plugins.enums import RunMethod
 from synapse_sdk.plugins.exceptions import ActionError
 from synapse_sdk.plugins.models import PluginRelease, Run
-from synapse_sdk.plugins.upload import archive_and_upload, build_and_upload, download_and_upload
+from synapse_sdk.plugins.upload import archive_and_upload, build_and_upload
+from synapse_sdk.shared import init_sentry, needs_sentry_init
 from synapse_sdk.utils.module_loading import import_string
 from synapse_sdk.utils.pydantic.errors import pydantic_to_drf_error
 
@@ -25,7 +26,8 @@ class Action:
         method (RunMethod): The method to run of the action.
         run_class (Run): The class to run the action.
         params_model (BaseModel): The model to validate the params.
-        progress_categories (List[str]): The categories to update the progress.
+        progress_categories (Dict[str] | None): The categories to update the progress.
+        metrics_categories (Dict[str] | None): The categories to update the metrics.
         params (Dict): The params to run the action.
         plugin_config (Dict): The plugin config.
         plugin_release (PluginRelease): The plugin release.
@@ -48,6 +50,7 @@ class Action:
     run_class = Run
     params_model = None
     progress_categories = None
+    metrics_categories = None
 
     # init 변수
     params = None
@@ -61,6 +64,7 @@ class Action:
     envs = None
     run = None
 
+    # TODO: Refactor to use Synapse Access Token instead of SYNAPSE_PLUGIN_RUN_USER_TOKEN and SYNAPSE_PLUGIN_RUN_TENANT
     REQUIRED_ENVS = [
         'RAY_ADDRESS',
         'RAY_DASHBOARD_URL',
@@ -105,15 +109,37 @@ class Action:
     def plugin_url(self):
         if self.debug:
             plugin_path = self.envs.get('SYNAPSE_DEBUG_PLUGIN_PATH') or '.'
-            if plugin_path.startswith('https://'):  # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
-                plugin_url = plugin_path
-            elif plugin_path.startswith('http://'):
-                plugin_url = download_and_upload(plugin_path, self.plugin_storage_url)
-            else:
+
+            # For HTTP/HTTPS URLs in debug mode, convert to Ray GCS (Global Control Store) URL
+            if plugin_path.startswith(('http://', 'https://')):
+                try:
+                    from synapse_sdk.plugins.utils import convert_http_to_ray_gcs
+
+                    plugin_url = convert_http_to_ray_gcs(plugin_path)
+                except (ImportError, RuntimeError):
+                    plugin_url = plugin_path
+
+            elif self.envs.get('SYNAPSE_PLUGIN_STORAGE'):
                 plugin_url = archive_and_upload(plugin_path, self.plugin_storage_url)
+            else:
+                plugin_url = plugin_path
+
             self.envs['SYNAPSE_DEBUG_PLUGIN_PATH'] = plugin_url
             return plugin_url
-        return self.plugin_release.get_url(self.plugin_storage_url)
+
+        # Production path: get URL from storage provider
+        url = self.plugin_release.get_url(self.plugin_storage_url)
+
+        # Convert HTTP URLs to Ray GCS URLs if needed
+        if url.startswith(('http://', 'https://')):
+            try:
+                from synapse_sdk.plugins.utils import convert_http_to_ray_gcs
+
+                url = convert_http_to_ray_gcs(url)
+            except (ImportError, RuntimeError):
+                pass
+
+        return url
 
     @property
     def debug_modules(self):
@@ -133,13 +159,38 @@ class Action:
     def plugin_package_manager(self):
         return self.plugin_config.get('package_manager', 'pip')
 
+    @property
+    def package_manager_options(self):
+        # Get user-defined options from plugin config
+        user_options = self.plugin_config.get('package_manager_options', [])
+
+        if self.plugin_package_manager == 'uv':
+            defaults = ['--no-cache']
+            # Add defaults if not already present
+            options_list = defaults.copy()
+            for option in user_options:
+                if option not in options_list:
+                    options_list.append(option)
+            return {'uv_pip_install_options': options_list}
+        else:
+            # For pip, use pip_install_options with --upgrade flag to ensure
+            # packages from requirements.txt (like synapse-sdk) override pre-installed versions
+            defaults = ['--upgrade']
+            options_list = defaults.copy()
+            for option in user_options:
+                if option not in options_list:
+                    options_list.append(option)
+            return {'pip_install_options': options_list}
+
     def get_run(self):
         context = {
             'plugin_release': self.plugin_release,
             'progress_categories': self.progress_categories,
+            'metrics_categories': self.metrics_categories,
             'params': self.params,
             'envs': self.envs,
             'debug': self.debug,
+            'action_name': self.name,
         }
         return self.run_class(self.job_id, context)
 
@@ -147,13 +198,20 @@ class Action:
         return {env: os.environ[env] for env in self.REQUIRED_ENVS if env in os.environ}
 
     def get_runtime_env(self):
-        runtime_env = {self.plugin_package_manager: [], 'working_dir': self.plugin_url}
+        runtime_env = {self.plugin_package_manager: {'packages': []}, 'working_dir': self.plugin_url}
 
         if self.requirements:
-            runtime_env[self.plugin_package_manager] += self.requirements
+            runtime_env[self.plugin_package_manager]['packages'] += self.requirements
 
         if self.debug:
-            runtime_env[self.plugin_package_manager] += self.debug_modules
+            runtime_env[self.plugin_package_manager]['packages'] += self.debug_modules
+
+        for key, value in self.package_manager_options.items():
+            runtime_env[self.plugin_package_manager][key] = value
+
+        # Sentry init if SENTRY_DSN is set
+        if needs_sentry_init():
+            runtime_env['worker_process_setup_hook'] = 'synapse_sdk.shared.worker_process_setup_hook'
 
         # 맨 마지막에 진행되어야 함
         runtime_env['env_vars'] = self.envs
@@ -231,6 +289,8 @@ class Action:
 
         * Executes the entrypoint with Ray job. Ray job manages the entrypoint execution and stores the results.
         """
+        self.ray_init()
+
         main_options = []
         options = ['run', '--direct']
         arguments = [self.name, f'{json.dumps(json.dumps(self.params))}']
@@ -285,5 +345,7 @@ class Action:
     def ray_init(self):
         import ray
 
+        init_sentry()
+
         if not ray.is_initialized():
             ray.init(address=self.envs['RAY_ADDRESS'], ignore_reinit_error=True)

synapse_sdk/plugins/categories/data_validation/actions/validation.py

@@ -1,10 +1,82 @@
+from enum import Enum
+from typing import Any, Dict, List
+
+from pydantic import BaseModel
+
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
 from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 
 
+class ValidationDataStatus(str, Enum):
+    """Validation data status enumeration.
+
+    Represents the possible status values for validation operations.
+
+    Attributes:
+        SUCCESS: Validation completed successfully with no errors.
+        FAILED: Validation failed with one or more errors.
+    """
+
+    SUCCESS = 'success'
+    FAILED = 'failed'
+
+
+class ValidationResult(BaseModel):
+    """Validation result model.
+
+    Args:
+        status: The validation status.
+        errors: List of validation errors.
+    """
+
+    status: ValidationDataStatus
+    errors: List[str]
+
+
+class CriticalError(Exception):
+    """Critical error exception for validation processing.
+
+    Raised when a critical error occurs during validation that prevents
+    the validation process from continuing normally.
+
+    Args:
+        message: Custom error message. Defaults to a standard critical error message.
+
+    Attributes:
+        message: The error message associated with this exception.
+    """
+
+    def __init__(self, message: str = 'Critical error occured while processing validation'):
+        self.message = message
+        super().__init__(self.message)
+
+
+class ValidationParams(BaseModel):
+    """Validation action parameters.
+
+    Args:
+        data (dict): The validation data.
+    """
+
+    data: Dict[str, Any]
+
+
 @register_action
 class ValidationAction(Action):
+    """Validation action for data validation processing.
+
+    This action handles the process of validating data with assignment IDs.
+    It supports validation methods and provides structured logging.
+
+    Attrs:
+        name (str): Action name, set to 'validation'.
+        category (PluginCategory): Plugin category, set to DATA_VALIDATION.
+        method (RunMethod): Execution method, set to TASK.
+        params_model (Type[ValidationParams]): Parameter validation model.
+    """
+
     name = 'validation'
     category = PluginCategory.DATA_VALIDATION
     method = RunMethod.TASK
+    params_model = ValidationParams

synapse_sdk/plugins/categories/data_validation/templates/plugin/validation.py

@@ -1,5 +1,33 @@
-def validate(data, assignment_id, **kwargs):
-    errors = []
-    if data.get('errors'):
-        errors += data['errors']
-    return errors
+from typing import Any, Dict, List, Optional
+
+from synapse_sdk.plugins.categories.data_validation.actions.validation import ValidationDataStatus, ValidationResult
+
+
+def validate(data: Optional[Dict[str, Any]] = None, **kwargs: Any) -> Dict[str, Any]:
+    """Validate data with assignment data.
+
+    * Custom validation logic can be added here.
+    * Error messages can be added to the errors list if errors exist in data.
+    * The validation result will be returned as a dict with ValidationResult structure.
+
+    Args:
+        data: The data to validate.
+        **kwargs: Additional arguments.
+
+    Returns:
+        Dict[str, Any]: The validation result as a dictionary with ValidationResult structure.
+    """
+    errors: List[str] = []
+
+    # Add custom validation logic here
+
+    # Add error messages into errors list if errors exist in data
+
+    # Determine status based on errors
+    status = ValidationDataStatus.FAILED if errors else ValidationDataStatus.SUCCESS
+
+    # DO NOT MODIFY BELOW THIS LINE - Validation result should be returned as a dumped ValidationResult.
+    validation_result = ValidationResult(status=status, errors=errors)
+    result_dict = validation_result.model_dump()
+    result_dict['status'] = status.value
+    return result_dict

synapse_sdk/plugins/categories/export/actions/__init__.py

@@ -0,0 +1,3 @@
+from .export import ExportAction
+
+__all__ = ['ExportAction']

synapse_sdk/plugins/categories/export/actions/export/__init__.py

@@ -0,0 +1,28 @@
+from .action import ExportAction
+from .enums import ExportStatus, LogCode
+from .exceptions import ExportError, ExportTargetError, ExportValidationError
+from .models import ExportParams
+from .run import ExportRun
+from .utils import (
+    AssignmentExportTargetHandler,
+    ExportTargetHandler,
+    GroundTruthExportTargetHandler,
+    TargetHandlerFactory,
+    TaskExportTargetHandler,
+)
+
+__all__ = [
+    'ExportAction',
+    'ExportStatus',
+    'LogCode',
+    'ExportError',
+    'ExportTargetError',
+    'ExportValidationError',
+    'ExportParams',
+    'ExportRun',
+    'ExportTargetHandler',
+    'AssignmentExportTargetHandler',
+    'GroundTruthExportTargetHandler',
+    'TaskExportTargetHandler',
+    'TargetHandlerFactory',
+]

synapse_sdk/plugins/categories/export/actions/export/action.py

@@ -0,0 +1,165 @@
+from itertools import tee
+from typing import Any, Dict
+
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.utils.storage import get_pathlib
+
+from .enums import LogCode
+from .models import ExportParams
+from .run import ExportRun
+from .utils import TargetHandlerFactory
+
+
+@register_action
+class ExportAction(Action):
+    """Main export action for processing and exporting data from various targets.
+
+    Handles export operations including target validation, data retrieval,
+    and file generation. Supports export from assignment, ground_truth, and task
+    targets with comprehensive progress tracking and error handling.
+
+    Features:
+    - Multiple target source support (assignment, ground_truth, task)
+    - Filter validation and data retrieval
+    - Original file and data file export options
+    - Progress tracking with detailed metrics
+    - Comprehensive error logging
+    - Project configuration handling
+
+    Class Attributes:
+        name (str): Action identifier ('export')
+        category (PluginCategory): EXPORT category
+        method (RunMethod): JOB execution method
+        run_class (type): ExportRun for specialized logging
+        params_model (type): ExportParams for parameter validation
+        progress_categories (dict): Progress tracking configuration
+        metrics_categories (dict): Metrics collection configuration
+
+    Example:
+        >>> action = ExportAction(
+        ...     params={
+        ...         'name': 'Assignment Export',
+        ...         'storage': 1,
+        ...         'path': '/exports/assignments',
+        ...         'target': 'assignment',
+        ...         'filter': {'project': 123}
+        ...     },
+        ...     plugin_config=config
+        ... )
+        >>> result = action.start()
+    """
+
+    name = 'export'
+    category = PluginCategory.EXPORT
+    method = RunMethod.JOB
+    params_model = ExportParams
+    run_class = ExportRun
+    progress_categories = {
+        'dataset_conversion': {
+            'proportion': 100,
+        }
+    }
+    metrics_categories = {
+        'data_file': {
+            'stand_by': 0,
+            'failed': 0,
+            'success': 0,
+        },
+        'original_file': {
+            'stand_by': 0,
+            'failed': 0,
+            'success': 0,
+        },
+    }
+
+    def get_filtered_results(self, filters, handler):
+        """Get filtered target results.
+
+        Retrieves data from the specified target using the provided filters
+        through the appropriate target handler.
+
+        Args:
+            filters (dict): Filter criteria to apply
+            handler (ExportTargetHandler): Target-specific handler
+
+        Returns:
+            tuple: (results, count) where results is the data and count is total
+
+        Raises:
+            PydanticCustomError: If data retrieval fails
+        """
+        try:
+            result_list = handler.get_results(self.client, filters)
+            results = result_list[0]
+            count = result_list[1]
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get dataset.'))
+        return results, count
+
+    def start(self) -> Dict[str, Any]:
+        """Start the export process.
+
+        Main entry point for export operations. Handles parameter preparation,
+        target handler selection, data retrieval, and export execution.
+
+        Returns:
+            Dict[str, Any]: Export results from the entrypoint
+
+        Raises:
+            Various exceptions based on validation and processing failures
+        """
+        self.run.log_message_with_code(LogCode.EXPORT_STARTED)
+
+        # Get expand setting from config, default to True (expand data)
+        filters = {**self.params['filter']}
+        data_expand = self.config.get('data_expand', True)
+        if data_expand:
+            filters['expand'] = 'data'
+
+        target = self.params['target']
+        handler = TargetHandlerFactory.get_handler(target)
+
+        self.params['results'], self.params['count'] = self.get_filtered_results(filters, handler)
+
+        if self.params['count'] == 0:
+            self.run.log_message_with_code(LogCode.NO_RESULTS_FOUND)
+        else:
+            self.run.log_message_with_code(LogCode.RESULTS_RETRIEVED, self.params['count'])
+
+        # For the 'ground_truth' target, retrieve project information from the first result and add configuration
+        if target == 'ground_truth':
+            try:
+                # Split generator into two using tee()
+                peek_iter, main_iter = tee(self.params['results'])
+                first_result = next(peek_iter)  # Peek first value only
+                project_pk = first_result['project']
+                project_info = self.client.get_project(project_pk)
+                self.params['project_id'] = project_pk
+                self.params['configuration'] = project_info.get('configuration', {})
+                self.params['results'] = main_iter  # Keep original generator intact
+            except (StopIteration, KeyError):
+                self.params['configuration'] = {}
+        # For the 'assignment' and 'task' targets, retrieve the project from the filter as before
+        elif target in ['assignment', 'task'] and 'project' in self.params['filter']:
+            project_pk = self.params['filter']['project']
+            project_info = self.client.get_project(project_pk)
+            self.params['configuration'] = project_info.get('configuration', {})
+
+        export_items = handler.get_export_item(self.params['results'])
+        storage = self.client.get_storage(self.params['storage'])
+        pathlib_cwd = get_pathlib(storage, self.params['path'])
+        exporter = self.entrypoint(self.run, export_items, pathlib_cwd, **self.params)
+
+        try:
+            result = exporter.export()
+            self.run.log_message_with_code(LogCode.EXPORT_COMPLETED)
+            return result
+        except Exception as e:
+            self.run.log_message_with_code(LogCode.EXPORT_FAILED, str(e))
+            raise

synapse_sdk/plugins/categories/export/actions/export/enums.py

@@ -0,0 +1,113 @@
+from enum import Enum
+
+from synapse_sdk.shared.enums import Context
+
+
+class ExportStatus(str, Enum):
+    """Export processing status enumeration.
+
+    Defines the possible states for export operations, data files, and export items
+    throughout the export process.
+
+    Attributes:
+        SUCCESS: Export completed successfully
+        FAILED: Export failed with errors
+        STAND_BY: Export waiting to be processed
+    """
+
+    SUCCESS = 'success'
+    FAILED = 'failed'
+    STAND_BY = 'stand_by'
+
+
+class LogCode(str, Enum):
+    """Type-safe logging codes for export operations.
+
+    Enumeration of all possible log events during export processing. Each code
+    corresponds to a specific event or error state with predefined message
+    templates and log levels.
+
+    The codes are organized by category:
+    - Validation codes (VALIDATION_FAILED, STORAGE_VALIDATION_FAILED, etc.)
+    - Export processing codes (EXPORT_STARTED, EXPORT_COMPLETED, etc.)
+    - File processing codes (ORIGINAL_FILE_EXPORTED, DATA_FILE_EXPORTED, etc.)
+    - Error handling codes (TARGET_HANDLER_ERROR, EXPORT_FAILED, etc.)
+
+    Each code maps to a configuration in LOG_MESSAGES with message template
+    and appropriate log level.
+    """
+
+    STORAGE_VALIDATION_FAILED = 'STORAGE_VALIDATION_FAILED'
+    FILTER_VALIDATION_FAILED = 'FILTER_VALIDATION_FAILED'
+    TARGET_VALIDATION_FAILED = 'TARGET_VALIDATION_FAILED'
+    VALIDATION_FAILED = 'VALIDATION_FAILED'
+    EXPORT_STARTED = 'EXPORT_STARTED'
+    EXPORT_COMPLETED = 'EXPORT_COMPLETED'
+    EXPORT_FAILED = 'EXPORT_FAILED'
+    NO_RESULTS_FOUND = 'NO_RESULTS_FOUND'
+    RESULTS_RETRIEVED = 'RESULTS_RETRIEVED'
+    ORIGINAL_FILE_EXPORTED = 'ORIGINAL_FILE_EXPORTED'
+    DATA_FILE_EXPORTED = 'DATA_FILE_EXPORTED'
+    FILE_EXPORT_FAILED = 'FILE_EXPORT_FAILED'
+    TARGET_HANDLER_ERROR = 'TARGET_HANDLER_ERROR'
+    NULL_DATA_DETECTED = 'NULL_DATA_DETECTED'
+
+
+LOG_MESSAGES = {
+    LogCode.STORAGE_VALIDATION_FAILED: {
+        'message': 'Storage validation failed.',
+        'level': Context.DANGER,
+    },
+    LogCode.FILTER_VALIDATION_FAILED: {
+        'message': 'Filter validation failed.',
+        'level': Context.DANGER,
+    },
+    LogCode.TARGET_VALIDATION_FAILED: {
+        'message': 'Target validation failed.',
+        'level': Context.DANGER,
+    },
+    LogCode.VALIDATION_FAILED: {
+        'message': 'Validation failed.',
+        'level': Context.DANGER,
+    },
+    LogCode.EXPORT_STARTED: {
+        'message': 'Export process started.',
+        'level': None,
+    },
+    LogCode.EXPORT_COMPLETED: {
+        'message': 'Export process completed.',
+        'level': None,
+    },
+    LogCode.EXPORT_FAILED: {
+        'message': 'Export process failed: {}',
+        'level': Context.DANGER,
+    },
+    LogCode.NO_RESULTS_FOUND: {
+        'message': 'No results found for export.',
+        'level': Context.WARNING,
+    },
+    LogCode.RESULTS_RETRIEVED: {
+        'message': 'Retrieved {} results for export',
+        'level': None,
+    },
+    LogCode.ORIGINAL_FILE_EXPORTED: {
+        'message': 'Original file exported successfully.',
+        'level': None,
+    },
+    LogCode.DATA_FILE_EXPORTED: {
+        'message': 'Data file exported successfully.',
+        'level': None,
+    },
+    LogCode.FILE_EXPORT_FAILED: {
+        'message': 'Failed to export file: {}',
+        'level': Context.DANGER,
+    },
+    LogCode.TARGET_HANDLER_ERROR: {
+        'message': 'Target handler error: {}',
+        'level': Context.DANGER,
+    },
+    LogCode.NULL_DATA_DETECTED: {
+        'message': 'Data is null for export item',
+        'level': Context.WARNING,
+    },
+}

synapse_sdk/plugins/categories/export/actions/export/exceptions.py

@@ -0,0 +1,53 @@
+class ExportError(Exception):
+    """Base exception for export-related errors.
+
+    This exception is raised when an export operation encounters errors
+    that prevent successful completion. It serves as the base class for
+    more specific export-related exceptions.
+
+    Used during export processing to handle various error conditions
+    such as validation failures, data access errors, or processing issues.
+
+    Example:
+        >>> if not validate_export_data(data):
+        ...     raise ExportError("Export data validation failed")
+    """
+
+    pass
+
+
+class ExportValidationError(ExportError):
+    """Exception raised when export parameter validation fails.
+
+    This exception is raised when export parameters or configuration
+    fail validation checks, preventing the export operation from starting.
+
+    Used during parameter validation to distinguish validation errors
+    from other types of export failures.
+
+    Example:
+        >>> if not storage_exists(storage_id):
+        ...     raise ExportValidationError(f"Storage {storage_id} does not exist")
+    """
+
+    pass
+
+
+class ExportTargetError(ExportError):
+    """Exception raised when export target handling encounters errors.
+
+    This exception is raised when target-specific operations (assignment,
+    ground_truth, task) fail due to data access issues, filter problems,
+    or target-specific validation failures.
+
+    Used during target data retrieval and processing to handle target-specific
+    errors separately from general export errors.
+
+    Example:
+        >>> try:
+        ...     results = client.list_assignments(params=filters)
+        ... except ClientError as e:
+        ...     raise ExportTargetError(f"Failed to retrieve assignments: {e}")
+    """
+
+    pass