synapse-sdk 1.0.0b5__py3-none-any.whl → 2025.12.3__py3-none-any.whl

synapse_sdk/plugins/categories/base.py

@@ -11,7 +11,8 @@ from synapse_sdk.clients.ray import RayClient
 from synapse_sdk.plugins.enums import RunMethod
 from synapse_sdk.plugins.exceptions import ActionError
 from synapse_sdk.plugins.models import PluginRelease, Run
-from synapse_sdk.plugins.upload import archive_and_upload, build_and_upload, download_and_upload
+from synapse_sdk.plugins.upload import archive_and_upload, build_and_upload
+from synapse_sdk.shared import init_sentry, needs_sentry_init
 from synapse_sdk.utils.module_loading import import_string
 from synapse_sdk.utils.pydantic.errors import pydantic_to_drf_error
 
@@ -108,15 +109,37 @@ class Action:
     def plugin_url(self):
         if self.debug:
             plugin_path = self.envs.get('SYNAPSE_DEBUG_PLUGIN_PATH') or '.'
-            if plugin_path.startswith('https://'):  # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
-                plugin_url = plugin_path
-            elif plugin_path.startswith('http://'):
-                plugin_url = download_and_upload(plugin_path, self.plugin_storage_url)
-            else:
+
+            # For HTTP/HTTPS URLs in debug mode, convert to Ray GCS (Global Control Store) URL
+            if plugin_path.startswith(('http://', 'https://')):
+                try:
+                    from synapse_sdk.plugins.utils import convert_http_to_ray_gcs
+
+                    plugin_url = convert_http_to_ray_gcs(plugin_path)
+                except (ImportError, RuntimeError):
+                    plugin_url = plugin_path
+
+            elif self.envs.get('SYNAPSE_PLUGIN_STORAGE'):
                 plugin_url = archive_and_upload(plugin_path, self.plugin_storage_url)
+            else:
+                plugin_url = plugin_path
+
             self.envs['SYNAPSE_DEBUG_PLUGIN_PATH'] = plugin_url
             return plugin_url
-        return self.plugin_release.get_url(self.plugin_storage_url)
+
+        # Production path: get URL from storage provider
+        url = self.plugin_release.get_url(self.plugin_storage_url)
+
+        # Convert HTTP URLs to Ray GCS URLs if needed
+        if url.startswith(('http://', 'https://')):
+            try:
+                from synapse_sdk.plugins.utils import convert_http_to_ray_gcs
+
+                url = convert_http_to_ray_gcs(url)
+            except (ImportError, RuntimeError):
+                pass
+
+        return url
 
     @property
     def debug_modules(self):
@@ -138,8 +161,26 @@ class Action:
 
     @property
     def package_manager_options(self):
-        options = {'pip': {}, 'uv': {'uv_pip_install_options': []}}
-        return options[self.plugin_package_manager]
+        # Get user-defined options from plugin config
+        user_options = self.plugin_config.get('package_manager_options', [])
+
+        if self.plugin_package_manager == 'uv':
+            defaults = ['--no-cache']
+            # Add defaults if not already present
+            options_list = defaults.copy()
+            for option in user_options:
+                if option not in options_list:
+                    options_list.append(option)
+            return {'uv_pip_install_options': options_list}
+        else:
+            # For pip, use pip_install_options with --upgrade flag to ensure
+            # packages from requirements.txt (like synapse-sdk) override pre-installed versions
+            defaults = ['--upgrade']
+            options_list = defaults.copy()
+            for option in user_options:
+                if option not in options_list:
+                    options_list.append(option)
+            return {'pip_install_options': options_list}
 
     def get_run(self):
         context = {
@@ -149,6 +190,7 @@ class Action:
             'params': self.params,
             'envs': self.envs,
             'debug': self.debug,
+            'action_name': self.name,
         }
         return self.run_class(self.job_id, context)
 
@@ -167,6 +209,10 @@ class Action:
         for key, value in self.package_manager_options.items():
             runtime_env[self.plugin_package_manager][key] = value
 
+        # Sentry init if SENTRY_DSN is set
+        if needs_sentry_init():
+            runtime_env['worker_process_setup_hook'] = 'synapse_sdk.shared.worker_process_setup_hook'
+
         # 맨 마지막에 진행되어야 함
         runtime_env['env_vars'] = self.envs
 
@@ -243,6 +289,8 @@ class Action:
 
         * Executes the entrypoint with Ray job. Ray job manages the entrypoint execution and stores the results.
         """
+        self.ray_init()
+
         main_options = []
         options = ['run', '--direct']
         arguments = [self.name, f'{json.dumps(json.dumps(self.params))}']
@@ -297,5 +345,7 @@ class Action:
     def ray_init(self):
         import ray
 
+        init_sentry()
+
         if not ray.is_initialized():
             ray.init(address=self.envs['RAY_ADDRESS'], ignore_reinit_error=True)

synapse_sdk/plugins/categories/export/actions/__init__.py

@@ -0,0 +1,3 @@
+from .export import ExportAction
+
+__all__ = ['ExportAction']

synapse_sdk/plugins/categories/export/actions/export/__init__.py

@@ -0,0 +1,28 @@
+from .action import ExportAction
+from .enums import ExportStatus, LogCode
+from .exceptions import ExportError, ExportTargetError, ExportValidationError
+from .models import ExportParams
+from .run import ExportRun
+from .utils import (
+    AssignmentExportTargetHandler,
+    ExportTargetHandler,
+    GroundTruthExportTargetHandler,
+    TargetHandlerFactory,
+    TaskExportTargetHandler,
+)
+
+__all__ = [
+    'ExportAction',
+    'ExportStatus',
+    'LogCode',
+    'ExportError',
+    'ExportTargetError',
+    'ExportValidationError',
+    'ExportParams',
+    'ExportRun',
+    'ExportTargetHandler',
+    'AssignmentExportTargetHandler',
+    'GroundTruthExportTargetHandler',
+    'TaskExportTargetHandler',
+    'TargetHandlerFactory',
+]

synapse_sdk/plugins/categories/export/actions/export/action.py

@@ -0,0 +1,165 @@
+from itertools import tee
+from typing import Any, Dict
+
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.utils.storage import get_pathlib
+
+from .enums import LogCode
+from .models import ExportParams
+from .run import ExportRun
+from .utils import TargetHandlerFactory
+
+
+@register_action
+class ExportAction(Action):
+    """Main export action for processing and exporting data from various targets.
+
+    Handles export operations including target validation, data retrieval,
+    and file generation. Supports export from assignment, ground_truth, and task
+    targets with comprehensive progress tracking and error handling.
+
+    Features:
+    - Multiple target source support (assignment, ground_truth, task)
+    - Filter validation and data retrieval
+    - Original file and data file export options
+    - Progress tracking with detailed metrics
+    - Comprehensive error logging
+    - Project configuration handling
+
+    Class Attributes:
+        name (str): Action identifier ('export')
+        category (PluginCategory): EXPORT category
+        method (RunMethod): JOB execution method
+        run_class (type): ExportRun for specialized logging
+        params_model (type): ExportParams for parameter validation
+        progress_categories (dict): Progress tracking configuration
+        metrics_categories (dict): Metrics collection configuration
+
+    Example:
+        >>> action = ExportAction(
+        ...     params={
+        ...         'name': 'Assignment Export',
+        ...         'storage': 1,
+        ...         'path': '/exports/assignments',
+        ...         'target': 'assignment',
+        ...         'filter': {'project': 123}
+        ...     },
+        ...     plugin_config=config
+        ... )
+        >>> result = action.start()
+    """
+
+    name = 'export'
+    category = PluginCategory.EXPORT
+    method = RunMethod.JOB
+    params_model = ExportParams
+    run_class = ExportRun
+    progress_categories = {
+        'dataset_conversion': {
+            'proportion': 100,
+        }
+    }
+    metrics_categories = {
+        'data_file': {
+            'stand_by': 0,
+            'failed': 0,
+            'success': 0,
+        },
+        'original_file': {
+            'stand_by': 0,
+            'failed': 0,
+            'success': 0,
+        },
+    }
+
+    def get_filtered_results(self, filters, handler):
+        """Get filtered target results.
+
+        Retrieves data from the specified target using the provided filters
+        through the appropriate target handler.
+
+        Args:
+            filters (dict): Filter criteria to apply
+            handler (ExportTargetHandler): Target-specific handler
+
+        Returns:
+            tuple: (results, count) where results is the data and count is total
+
+        Raises:
+            PydanticCustomError: If data retrieval fails
+        """
+        try:
+            result_list = handler.get_results(self.client, filters)
+            results = result_list[0]
+            count = result_list[1]
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get dataset.'))
+        return results, count
+
+    def start(self) -> Dict[str, Any]:
+        """Start the export process.
+
+        Main entry point for export operations. Handles parameter preparation,
+        target handler selection, data retrieval, and export execution.
+
+        Returns:
+            Dict[str, Any]: Export results from the entrypoint
+
+        Raises:
+            Various exceptions based on validation and processing failures
+        """
+        self.run.log_message_with_code(LogCode.EXPORT_STARTED)
+
+        # Get expand setting from config, default to True (expand data)
+        filters = {**self.params['filter']}
+        data_expand = self.config.get('data_expand', True)
+        if data_expand:
+            filters['expand'] = 'data'
+
+        target = self.params['target']
+        handler = TargetHandlerFactory.get_handler(target)
+
+        self.params['results'], self.params['count'] = self.get_filtered_results(filters, handler)
+
+        if self.params['count'] == 0:
+            self.run.log_message_with_code(LogCode.NO_RESULTS_FOUND)
+        else:
+            self.run.log_message_with_code(LogCode.RESULTS_RETRIEVED, self.params['count'])
+
+        # For the 'ground_truth' target, retrieve project information from the first result and add configuration
+        if target == 'ground_truth':
+            try:
+                # Split generator into two using tee()
+                peek_iter, main_iter = tee(self.params['results'])
+                first_result = next(peek_iter)  # Peek first value only
+                project_pk = first_result['project']
+                project_info = self.client.get_project(project_pk)
+                self.params['project_id'] = project_pk
+                self.params['configuration'] = project_info.get('configuration', {})
+                self.params['results'] = main_iter  # Keep original generator intact
+            except (StopIteration, KeyError):
+                self.params['configuration'] = {}
+        # For the 'assignment' and 'task' targets, retrieve the project from the filter as before
+        elif target in ['assignment', 'task'] and 'project' in self.params['filter']:
+            project_pk = self.params['filter']['project']
+            project_info = self.client.get_project(project_pk)
+            self.params['configuration'] = project_info.get('configuration', {})
+
+        export_items = handler.get_export_item(self.params['results'])
+        storage = self.client.get_storage(self.params['storage'])
+        pathlib_cwd = get_pathlib(storage, self.params['path'])
+        exporter = self.entrypoint(self.run, export_items, pathlib_cwd, **self.params)
+
+        try:
+            result = exporter.export()
+            self.run.log_message_with_code(LogCode.EXPORT_COMPLETED)
+            return result
+        except Exception as e:
+            self.run.log_message_with_code(LogCode.EXPORT_FAILED, str(e))
+            raise

synapse_sdk/plugins/categories/export/actions/export/enums.py

@@ -0,0 +1,113 @@
+from enum import Enum
+
+from synapse_sdk.shared.enums import Context
+
+
+class ExportStatus(str, Enum):
+    """Export processing status enumeration.
+
+    Defines the possible states for export operations, data files, and export items
+    throughout the export process.
+
+    Attributes:
+        SUCCESS: Export completed successfully
+        FAILED: Export failed with errors
+        STAND_BY: Export waiting to be processed
+    """
+
+    SUCCESS = 'success'
+    FAILED = 'failed'
+    STAND_BY = 'stand_by'
+
+
+class LogCode(str, Enum):
+    """Type-safe logging codes for export operations.
+
+    Enumeration of all possible log events during export processing. Each code
+    corresponds to a specific event or error state with predefined message
+    templates and log levels.
+
+    The codes are organized by category:
+    - Validation codes (VALIDATION_FAILED, STORAGE_VALIDATION_FAILED, etc.)
+    - Export processing codes (EXPORT_STARTED, EXPORT_COMPLETED, etc.)
+    - File processing codes (ORIGINAL_FILE_EXPORTED, DATA_FILE_EXPORTED, etc.)
+    - Error handling codes (TARGET_HANDLER_ERROR, EXPORT_FAILED, etc.)
+
+    Each code maps to a configuration in LOG_MESSAGES with message template
+    and appropriate log level.
+    """
+
+    STORAGE_VALIDATION_FAILED = 'STORAGE_VALIDATION_FAILED'
+    FILTER_VALIDATION_FAILED = 'FILTER_VALIDATION_FAILED'
+    TARGET_VALIDATION_FAILED = 'TARGET_VALIDATION_FAILED'
+    VALIDATION_FAILED = 'VALIDATION_FAILED'
+    EXPORT_STARTED = 'EXPORT_STARTED'
+    EXPORT_COMPLETED = 'EXPORT_COMPLETED'
+    EXPORT_FAILED = 'EXPORT_FAILED'
+    NO_RESULTS_FOUND = 'NO_RESULTS_FOUND'
+    RESULTS_RETRIEVED = 'RESULTS_RETRIEVED'
+    ORIGINAL_FILE_EXPORTED = 'ORIGINAL_FILE_EXPORTED'
+    DATA_FILE_EXPORTED = 'DATA_FILE_EXPORTED'
+    FILE_EXPORT_FAILED = 'FILE_EXPORT_FAILED'
+    TARGET_HANDLER_ERROR = 'TARGET_HANDLER_ERROR'
+    NULL_DATA_DETECTED = 'NULL_DATA_DETECTED'
+
+
+LOG_MESSAGES = {
+    LogCode.STORAGE_VALIDATION_FAILED: {
+        'message': 'Storage validation failed.',
+        'level': Context.DANGER,
+    },
+    LogCode.FILTER_VALIDATION_FAILED: {
+        'message': 'Filter validation failed.',
+        'level': Context.DANGER,
+    },
+    LogCode.TARGET_VALIDATION_FAILED: {
+        'message': 'Target validation failed.',
+        'level': Context.DANGER,
+    },
+    LogCode.VALIDATION_FAILED: {
+        'message': 'Validation failed.',
+        'level': Context.DANGER,
+    },
+    LogCode.EXPORT_STARTED: {
+        'message': 'Export process started.',
+        'level': None,
+    },
+    LogCode.EXPORT_COMPLETED: {
+        'message': 'Export process completed.',
+        'level': None,
+    },
+    LogCode.EXPORT_FAILED: {
+        'message': 'Export process failed: {}',
+        'level': Context.DANGER,
+    },
+    LogCode.NO_RESULTS_FOUND: {
+        'message': 'No results found for export.',
+        'level': Context.WARNING,
+    },
+    LogCode.RESULTS_RETRIEVED: {
+        'message': 'Retrieved {} results for export',
+        'level': None,
+    },
+    LogCode.ORIGINAL_FILE_EXPORTED: {
+        'message': 'Original file exported successfully.',
+        'level': None,
+    },
+    LogCode.DATA_FILE_EXPORTED: {
+        'message': 'Data file exported successfully.',
+        'level': None,
+    },
+    LogCode.FILE_EXPORT_FAILED: {
+        'message': 'Failed to export file: {}',
+        'level': Context.DANGER,
+    },
+    LogCode.TARGET_HANDLER_ERROR: {
+        'message': 'Target handler error: {}',
+        'level': Context.DANGER,
+    },
+    LogCode.NULL_DATA_DETECTED: {
+        'message': 'Data is null for export item',
+        'level': Context.WARNING,
+    },
+}

synapse_sdk/plugins/categories/export/actions/export/exceptions.py

@@ -0,0 +1,53 @@
+class ExportError(Exception):
+    """Base exception for export-related errors.
+
+    This exception is raised when an export operation encounters errors
+    that prevent successful completion. It serves as the base class for
+    more specific export-related exceptions.
+
+    Used during export processing to handle various error conditions
+    such as validation failures, data access errors, or processing issues.
+
+    Example:
+        >>> if not validate_export_data(data):
+        ...     raise ExportError("Export data validation failed")
+    """
+
+    pass
+
+
+class ExportValidationError(ExportError):
+    """Exception raised when export parameter validation fails.
+
+    This exception is raised when export parameters or configuration
+    fail validation checks, preventing the export operation from starting.
+
+    Used during parameter validation to distinguish validation errors
+    from other types of export failures.
+
+    Example:
+        >>> if not storage_exists(storage_id):
+        ...     raise ExportValidationError(f"Storage {storage_id} does not exist")
+    """
+
+    pass
+
+
+class ExportTargetError(ExportError):
+    """Exception raised when export target handling encounters errors.
+
+    This exception is raised when target-specific operations (assignment,
+    ground_truth, task) fail due to data access issues, filter problems,
+    or target-specific validation failures.
+
+    Used during target data retrieval and processing to handle target-specific
+    errors separately from general export errors.
+
+    Example:
+        >>> try:
+        ...     results = client.list_assignments(params=filters)
+        ... except ClientError as e:
+        ...     raise ExportTargetError(f"Failed to retrieve assignments: {e}")
+    """
+
+    pass

synapse_sdk/plugins/categories/export/actions/export/models.py

@@ -0,0 +1,74 @@
+from typing import Annotated, Literal
+
+from pydantic import AfterValidator, BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.utils.pydantic.validators import non_blank
+
+from .utils import TargetHandlerFactory
+
+
+class ExportParams(BaseModel):
+    """Export action parameter validation model.
+
+    Defines and validates all parameters required for export operations.
+    Uses Pydantic for type validation and custom validators to ensure
+    storage and filter resources exist before processing.
+
+    Attributes:
+        name (str): Human-readable name for the export operation
+        description (str | None): Optional description of the export
+        storage (int): Storage ID where exported data will be saved
+        save_original_file (bool): Whether to save the original file
+        path (str): File system path where exported data will be saved
+        target (str): The target source to export data from (assignment, ground_truth, task)
+        filter (dict): Filter criteria to apply when retrieving data
+        extra_params (dict | None): Additional parameters for export customization.
+            Example: {"include_metadata": True, "compression": "gzip"}
+
+    Validation:
+        - name: Must be non-blank after validation
+        - storage: Must exist and be accessible via client API
+        - target: Must be one of the supported target types
+        - filter: Must be valid for the specified target type
+
+    Example:
+        >>> params = ExportParams(
+        ...     name="Assignment Export",
+        ...     storage=1,
+        ...     path="/exports/assignments",
+        ...     target="assignment",
+        ...     filter={"project": 123}
+        ... )
+    """
+
+    name: Annotated[str, AfterValidator(non_blank)]
+    description: str | None = None
+    storage: int
+    save_original_file: bool = True
+    path: str
+    target: Literal['assignment', 'ground_truth', 'task']
+    filter: dict
+    extra_params: dict | None = None
+
+    @field_validator('storage')
+    @staticmethod
+    def check_storage_exists(value, info):
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_storage(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get storage from Synapse backend.'))
+        return value
+
+    @field_validator('filter')
+    @staticmethod
+    def check_filter_by_target(value, info):
+        action = info.context['action']
+        client = action.client
+        target = action.params['target']
+        handler = TargetHandlerFactory.get_handler(target)
+        return handler.validate_filter(value, client)