synapse-sdk 1.0.0a13__py3-none-any.whl → 2025.11.7__py3-none-any.whl

synapse_sdk/plugins/__init__.py

@@ -1,13 +0,0 @@
-import os
-
-from dotenv import load_dotenv
-
-from synapse_sdk.plugins.cli import cli
-
-
-def init():
-    load_dotenv(os.path.join(os.getcwd(), '.env'))
-    cli(obj={}, auto_envvar_prefix='SYNAPSE_PLUGIN')
-
-
-__all__ = ['init']

synapse_sdk/plugins/categories/base.py

@@ -11,12 +11,38 @@ from synapse_sdk.clients.ray import RayClient
 from synapse_sdk.plugins.enums import RunMethod
 from synapse_sdk.plugins.exceptions import ActionError
 from synapse_sdk.plugins.models import PluginRelease, Run
-from synapse_sdk.plugins.upload import archive_and_upload, build_and_upload, download_and_upload
+from synapse_sdk.plugins.upload import archive_and_upload, build_and_upload
+from synapse_sdk.shared import init_sentry, needs_sentry_init
 from synapse_sdk.utils.module_loading import import_string
 from synapse_sdk.utils.pydantic.errors import pydantic_to_drf_error
 
 
 class Action:
+    """Base class for all plugin actions.
+
+    Attrs:
+        name (str): The name of the action.
+        category (PluginCategory): The category of the action.
+        method (RunMethod): The method to run of the action.
+        run_class (Run): The class to run the action.
+        params_model (BaseModel): The model to validate the params.
+        progress_categories (Dict[str] | None): The categories to update the progress.
+        metrics_categories (Dict[str] | None): The categories to update the metrics.
+        params (Dict): The params to run the action.
+        plugin_config (Dict): The plugin config.
+        plugin_release (PluginRelease): The plugin release.
+        config (Dict): The action config.
+        requirements (List[str]): The requirements to install.
+        job_id (str): The job id.
+        direct (bool): The flag to run the action directly.
+        debug (bool): The flag to run the action in debug mode.
+        envs (Dict): The runtime envs.
+        run (Run): The run instance.
+
+    Raises:
+        ActionError: If the action fails.
+    """
+
     # class 변수
     name = None
     category = None
@@ -24,19 +50,22 @@ class Action:
     run_class = Run
     params_model = None
     progress_categories = None
+    metrics_categories = None
 
     # init 변수
     params = None
     plugin_config = None
     plugin_release = None
     config = None
+    requirements = None
     job_id = None
     direct = None
     debug = None
     envs = None
     run = None
 
-    default_envs = [
+    # TODO: Refactor to use Synapse Access Token instead of SYNAPSE_PLUGIN_RUN_USER_TOKEN and SYNAPSE_PLUGIN_RUN_TENANT
+    REQUIRED_ENVS = [
         'RAY_ADDRESS',
         'RAY_DASHBOARD_URL',
         'RAY_SERVE_ADDRESS',
@@ -48,11 +77,12 @@ class Action:
         'SYNAPSE_PLUGIN_RUN_TENANT',
     ]
 
-    def __init__(self, params, plugin_config, envs=None, job_id=None, direct=False, debug=False):
+    def __init__(self, params, plugin_config, requirements=None, envs=None, job_id=None, direct=False, debug=False):
         self.params = params
         self.plugin_config = plugin_config
         self.plugin_release = PluginRelease(config=plugin_config)
         self.config = self.plugin_release.get_action_config(self.name)
+        self.requirements = requirements
         self.job_id = job_id
         self.direct = direct
         self.debug = debug
@@ -79,49 +109,109 @@ class Action:
     def plugin_url(self):
         if self.debug:
             plugin_path = self.envs.get('SYNAPSE_DEBUG_PLUGIN_PATH') or '.'
-            if plugin_path.startswith('https://'):  # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
-                plugin_url = plugin_path
-            elif plugin_path.startswith('http://'):
-                plugin_url = download_and_upload(plugin_path, self.plugin_storage_url)
-            else:
+
+            # For HTTP/HTTPS URLs in debug mode, convert to Ray GCS (Global Control Store) URL
+            if plugin_path.startswith(('http://', 'https://')):
+                try:
+                    from synapse_sdk.plugins.utils import convert_http_to_ray_gcs
+
+                    plugin_url = convert_http_to_ray_gcs(plugin_path)
+                except (ImportError, RuntimeError):
+                    plugin_url = plugin_path
+
+            elif self.envs.get('SYNAPSE_PLUGIN_STORAGE'):
                 plugin_url = archive_and_upload(plugin_path, self.plugin_storage_url)
+            else:
+                plugin_url = plugin_path
+
             self.envs['SYNAPSE_DEBUG_PLUGIN_PATH'] = plugin_url
             return plugin_url
-        return self.plugin_release.get_url(self.plugin_storage_url)
+
+        # Production path: get URL from storage provider
+        url = self.plugin_release.get_url(self.plugin_storage_url)
+
+        # Convert HTTP URLs to Ray GCS URLs if needed
+        if url.startswith(('http://', 'https://')):
+            try:
+                from synapse_sdk.plugins.utils import convert_http_to_ray_gcs
+
+                url = convert_http_to_ray_gcs(url)
+            except (ImportError, RuntimeError):
+                pass
+
+        return url
 
     @property
     def debug_modules(self):
         debug_modules = []
-        for module_path in self.envs.get('SYNAPSE_DEBUG_MODULES', '').split(','):
-            if module_path.startswith('https://'):  # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
-                module_url = module_path
-            else:
-                module_url = build_and_upload(module_path, self.plugin_storage_url)
-            debug_modules.append(module_url)
-        self.envs['SYNAPSE_DEBUG_MODULES'] = ','.join(debug_modules)
+        if self.envs.get('SYNAPSE_DEBUG_MODULES'):
+            for module_path in self.envs['SYNAPSE_DEBUG_MODULES'].split(','):
+                # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
+                if module_path.startswith('https://'):
+                    module_url = module_path
+                else:
+                    module_url = build_and_upload(module_path, self.plugin_storage_url)
+                debug_modules.append(module_url)
+            self.envs['SYNAPSE_DEBUG_MODULES'] = ','.join(debug_modules)
         return debug_modules
 
+    @property
+    def plugin_package_manager(self):
+        return self.plugin_config.get('package_manager', 'pip')
+
+    @property
+    def package_manager_options(self):
+        # Get user-defined options from plugin config
+        user_options = self.plugin_config.get('package_manager_options', [])
+
+        if self.plugin_package_manager == 'uv':
+            defaults = ['--no-cache']
+            # Add defaults if not already present
+            options_list = defaults.copy()
+            for option in user_options:
+                if option not in options_list:
+                    options_list.append(option)
+            return {'uv_pip_install_options': options_list}
+        else:
+            # For pip, use pip_install_options with --upgrade flag to ensure
+            # packages from requirements.txt (like synapse-sdk) override pre-installed versions
+            defaults = ['--upgrade']
+            options_list = defaults.copy()
+            for option in user_options:
+                if option not in options_list:
+                    options_list.append(option)
+            return {'pip_install_options': options_list}
+
     def get_run(self):
         context = {
             'plugin_release': self.plugin_release,
             'progress_categories': self.progress_categories,
+            'metrics_categories': self.metrics_categories,
             'params': self.params,
             'envs': self.envs,
             'debug': self.debug,
+            'action_name': self.name,
         }
         return self.run_class(self.job_id, context)
 
     def get_default_envs(self):
-        return {env: os.environ[env] for env in self.default_envs if env in os.environ}
+        return {env: os.environ[env] for env in self.REQUIRED_ENVS if env in os.environ}
 
     def get_runtime_env(self):
-        runtime_env = {
-            'pip': ['-r ${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/requirements.txt'],
-            'working_dir': self.plugin_url,
-        }
+        runtime_env = {self.plugin_package_manager: {'packages': []}, 'working_dir': self.plugin_url}
+
+        if self.requirements:
+            runtime_env[self.plugin_package_manager]['packages'] += self.requirements
 
         if self.debug:
-            runtime_env['pip'] += self.debug_modules
+            runtime_env[self.plugin_package_manager]['packages'] += self.debug_modules
+
+        for key, value in self.package_manager_options.items():
+            runtime_env[self.plugin_package_manager][key] = value
+
+        # Sentry init if SENTRY_DSN is set
+        if needs_sentry_init():
+            runtime_env['worker_process_setup_hook'] = 'synapse_sdk.shared.worker_process_setup_hook'
 
         # 맨 마지막에 진행되어야 함
         runtime_env['env_vars'] = self.envs
@@ -145,16 +235,25 @@ class Action:
                 return self.start_by_restapi()
             else:
                 result = self.start()
-                self.post_action_by_job(result)
+                if self.job_id:
+                    self.post_action_by_job(result)
                 return result
         return getattr(self, f'start_by_{self.method.value}')()
 
     def start(self):
+        """Start the action.
+
+        TODO: Specify the return type of start method for overrided methods.
+        """
         if self.method == RunMethod.JOB:
             return self.entrypoint(self.run, **self.params)
         return self.entrypoint(**self.params)
 
     def start_by_task(self):
+        """Ray Task based execution.
+
+        * A task method that simply executes the entrypoint without job management functionality.
+        """
         import ray
         from ray.exceptions import RayTaskError
 
@@ -180,13 +279,18 @@ class Action:
 
         kwargs['direct'] = True
         try:
-            if not ray.is_initialized():
-                ray.init(address=self.envs['RAY_ADDRESS'], ignore_reinit_error=True)
+            self.ray_init()
             return ray.get(run_task.remote(self.category.value, self.name, *args, **kwargs))
         except RayTaskError as e:
             raise ActionError(e.cause)
 
     def start_by_job(self):
+        """Ray Job based execution.
+
+        * Executes the entrypoint with Ray job. Ray job manages the entrypoint execution and stores the results.
+        """
+        self.ray_init()
+
         main_options = []
         options = ['run', '--direct']
         arguments = [self.name, f'{json.dumps(json.dumps(self.params))}']
@@ -202,11 +306,15 @@ class Action:
         client = self.get_job_client()
         return client.submit_job(
             submission_id=self.job_id,
-            entrypoint=f'python main.py {cmd}',
+            entrypoint=f'synapse plugin {cmd}',
             runtime_env=self.get_runtime_env(),
         )
 
     def start_by_restapi(self):
+        """Ray Serve based execution.
+
+        * This method executes a Fastapi endpoint defined within the Plugin.
+        """
         path = self.params.pop('path', '')
         method = self.params.pop('method')
 
@@ -225,12 +333,19 @@ class Action:
             raise ActionError('Unable to connect to serve application')
 
     def post_action_by_job(self, result):
-        if self.client:
-            job_client = self.get_job_client()
-            logs = job_client.get_job_logs(self.job_id).split('\n')
-            self.client.update_job(self.job_id, data={'result': result or {}, 'console_logs': logs})
+        job_client = self.get_job_client()
+        logs = job_client.get_job_logs(self.job_id).split('\n')
+        self.client.update_job(self.job_id, data={'result': result or {}, 'console_logs': logs})
 
     def get_job_client(self):
         from ray.dashboard.modules.job.sdk import JobSubmissionClient
 
         return JobSubmissionClient(address=self.envs.get('RAY_DASHBOARD_URL'))
+
+    def ray_init(self):
+        import ray
+
+        init_sentry()
+
+        if not ray.is_initialized():
+            ray.init(address=self.envs['RAY_ADDRESS'], ignore_reinit_error=True)

synapse_sdk/plugins/categories/data_validation/actions/validation.py

@@ -1,10 +1,82 @@
+from enum import Enum
+from typing import Any, Dict, List
+
+from pydantic import BaseModel
+
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
 from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 
 
+class ValidationDataStatus(str, Enum):
+    """Validation data status enumeration.
+
+    Represents the possible status values for validation operations.
+
+    Attributes:
+        SUCCESS: Validation completed successfully with no errors.
+        FAILED: Validation failed with one or more errors.
+    """
+
+    SUCCESS = 'success'
+    FAILED = 'failed'
+
+
+class ValidationResult(BaseModel):
+    """Validation result model.
+
+    Args:
+        status: The validation status.
+        errors: List of validation errors.
+    """
+
+    status: ValidationDataStatus
+    errors: List[str]
+
+
+class CriticalError(Exception):
+    """Critical error exception for validation processing.
+
+    Raised when a critical error occurs during validation that prevents
+    the validation process from continuing normally.
+
+    Args:
+        message: Custom error message. Defaults to a standard critical error message.
+
+    Attributes:
+        message: The error message associated with this exception.
+    """
+
+    def __init__(self, message: str = 'Critical error occured while processing validation'):
+        self.message = message
+        super().__init__(self.message)
+
+
+class ValidationParams(BaseModel):
+    """Validation action parameters.
+
+    Args:
+        data (dict): The validation data.
+    """
+
+    data: Dict[str, Any]
+
+
 @register_action
 class ValidationAction(Action):
+    """Validation action for data validation processing.
+
+    This action handles the process of validating data with assignment IDs.
+    It supports validation methods and provides structured logging.
+
+    Attrs:
+        name (str): Action name, set to 'validation'.
+        category (PluginCategory): Plugin category, set to DATA_VALIDATION.
+        method (RunMethod): Execution method, set to TASK.
+        params_model (Type[ValidationParams]): Parameter validation model.
+    """
+
     name = 'validation'
     category = PluginCategory.DATA_VALIDATION
     method = RunMethod.TASK
+    params_model = ValidationParams

synapse_sdk/plugins/categories/data_validation/templates/plugin/validation.py

@@ -1,5 +1,33 @@
-def validate(data, assignment_id, **kwargs):
-    errors = []
-    if data.get('errors'):
-        errors += data['errors']
-    return errors
+from typing import Any, Dict, List, Optional
+
+from synapse_sdk.plugins.categories.data_validation.actions.validation import ValidationDataStatus, ValidationResult
+
+
+def validate(data: Optional[Dict[str, Any]] = None, **kwargs: Any) -> Dict[str, Any]:
+    """Validate data with assignment data.
+
+    * Custom validation logic can be added here.
+    * Error messages can be added to the errors list if errors exist in data.
+    * The validation result will be returned as a dict with ValidationResult structure.
+
+    Args:
+        data: The data to validate.
+        **kwargs: Additional arguments.
+
+    Returns:
+        Dict[str, Any]: The validation result as a dictionary with ValidationResult structure.
+    """
+    errors: List[str] = []
+
+    # Add custom validation logic here
+
+    # Add error messages into errors list if errors exist in data
+
+    # Determine status based on errors
+    status = ValidationDataStatus.FAILED if errors else ValidationDataStatus.SUCCESS
+
+    # DO NOT MODIFY BELOW THIS LINE - Validation result should be returned as a dumped ValidationResult.
+    validation_result = ValidationResult(status=status, errors=errors)
+    result_dict = validation_result.model_dump()
+    result_dict['status'] = status.value
+    return result_dict

synapse_sdk/plugins/categories/export/actions/__init__.py

@@ -0,0 +1,3 @@
+from .export import ExportAction
+
+__all__ = ['ExportAction']

synapse_sdk/plugins/categories/export/actions/export/__init__.py

@@ -0,0 +1,28 @@
+from .action import ExportAction
+from .enums import ExportStatus, LogCode
+from .exceptions import ExportError, ExportTargetError, ExportValidationError
+from .models import ExportParams
+from .run import ExportRun
+from .utils import (
+    AssignmentExportTargetHandler,
+    ExportTargetHandler,
+    GroundTruthExportTargetHandler,
+    TargetHandlerFactory,
+    TaskExportTargetHandler,
+)
+
+__all__ = [
+    'ExportAction',
+    'ExportStatus',
+    'LogCode',
+    'ExportError',
+    'ExportTargetError',
+    'ExportValidationError',
+    'ExportParams',
+    'ExportRun',
+    'ExportTargetHandler',
+    'AssignmentExportTargetHandler',
+    'GroundTruthExportTargetHandler',
+    'TaskExportTargetHandler',
+    'TargetHandlerFactory',
+]

synapse_sdk/plugins/categories/export/actions/export/action.py

@@ -0,0 +1,165 @@
+from itertools import tee
+from typing import Any, Dict
+
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.utils.storage import get_pathlib
+
+from .enums import LogCode
+from .models import ExportParams
+from .run import ExportRun
+from .utils import TargetHandlerFactory
+
+
+@register_action
+class ExportAction(Action):
+    """Main export action for processing and exporting data from various targets.
+
+    Handles export operations including target validation, data retrieval,
+    and file generation. Supports export from assignment, ground_truth, and task
+    targets with comprehensive progress tracking and error handling.
+
+    Features:
+    - Multiple target source support (assignment, ground_truth, task)
+    - Filter validation and data retrieval
+    - Original file and data file export options
+    - Progress tracking with detailed metrics
+    - Comprehensive error logging
+    - Project configuration handling
+
+    Class Attributes:
+        name (str): Action identifier ('export')
+        category (PluginCategory): EXPORT category
+        method (RunMethod): JOB execution method
+        run_class (type): ExportRun for specialized logging
+        params_model (type): ExportParams for parameter validation
+        progress_categories (dict): Progress tracking configuration
+        metrics_categories (dict): Metrics collection configuration
+
+    Example:
+        >>> action = ExportAction(
+        ...     params={
+        ...         'name': 'Assignment Export',
+        ...         'storage': 1,
+        ...         'path': '/exports/assignments',
+        ...         'target': 'assignment',
+        ...         'filter': {'project': 123}
+        ...     },
+        ...     plugin_config=config
+        ... )
+        >>> result = action.start()
+    """
+
+    name = 'export'
+    category = PluginCategory.EXPORT
+    method = RunMethod.JOB
+    params_model = ExportParams
+    run_class = ExportRun
+    progress_categories = {
+        'dataset_conversion': {
+            'proportion': 100,
+        }
+    }
+    metrics_categories = {
+        'data_file': {
+            'stand_by': 0,
+            'failed': 0,
+            'success': 0,
+        },
+        'original_file': {
+            'stand_by': 0,
+            'failed': 0,
+            'success': 0,
+        },
+    }
+
+    def get_filtered_results(self, filters, handler):
+        """Get filtered target results.
+
+        Retrieves data from the specified target using the provided filters
+        through the appropriate target handler.
+
+        Args:
+            filters (dict): Filter criteria to apply
+            handler (ExportTargetHandler): Target-specific handler
+
+        Returns:
+            tuple: (results, count) where results is the data and count is total
+
+        Raises:
+            PydanticCustomError: If data retrieval fails
+        """
+        try:
+            result_list = handler.get_results(self.client, filters)
+            results = result_list[0]
+            count = result_list[1]
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get dataset.'))
+        return results, count
+
+    def start(self) -> Dict[str, Any]:
+        """Start the export process.
+
+        Main entry point for export operations. Handles parameter preparation,
+        target handler selection, data retrieval, and export execution.
+
+        Returns:
+            Dict[str, Any]: Export results from the entrypoint
+
+        Raises:
+            Various exceptions based on validation and processing failures
+        """
+        self.run.log_message_with_code(LogCode.EXPORT_STARTED)
+
+        # Get expand setting from config, default to True (expand data)
+        filters = {**self.params['filter']}
+        data_expand = self.config.get('data_expand', True)
+        if data_expand:
+            filters['expand'] = 'data'
+
+        target = self.params['target']
+        handler = TargetHandlerFactory.get_handler(target)
+
+        self.params['results'], self.params['count'] = self.get_filtered_results(filters, handler)
+
+        if self.params['count'] == 0:
+            self.run.log_message_with_code(LogCode.NO_RESULTS_FOUND)
+        else:
+            self.run.log_message_with_code(LogCode.RESULTS_RETRIEVED, self.params['count'])
+
+        # For the 'ground_truth' target, retrieve project information from the first result and add configuration
+        if target == 'ground_truth':
+            try:
+                # Split generator into two using tee()
+                peek_iter, main_iter = tee(self.params['results'])
+                first_result = next(peek_iter)  # Peek first value only
+                project_pk = first_result['project']
+                project_info = self.client.get_project(project_pk)
+                self.params['project_id'] = project_pk
+                self.params['configuration'] = project_info.get('configuration', {})
+                self.params['results'] = main_iter  # Keep original generator intact
+            except (StopIteration, KeyError):
+                self.params['configuration'] = {}
+        # For the 'assignment' and 'task' targets, retrieve the project from the filter as before
+        elif target in ['assignment', 'task'] and 'project' in self.params['filter']:
+            project_pk = self.params['filter']['project']
+            project_info = self.client.get_project(project_pk)
+            self.params['configuration'] = project_info.get('configuration', {})
+
+        export_items = handler.get_export_item(self.params['results'])
+        storage = self.client.get_storage(self.params['storage'])
+        pathlib_cwd = get_pathlib(storage, self.params['path'])
+        exporter = self.entrypoint(self.run, export_items, pathlib_cwd, **self.params)
+
+        try:
+            result = exporter.export()
+            self.run.log_message_with_code(LogCode.EXPORT_COMPLETED)
+            return result
+        except Exception as e:
+            self.run.log_message_with_code(LogCode.EXPORT_FAILED, str(e))
+            raise