synapse-sdk 2025.9.1__py3-none-any.whl → 2025.9.4__py3-none-any.whl

synapse_sdk/devtools/docs/sidebars.ts

@@ -53,7 +53,14 @@ const sidebars: SidebarsConfig = {
           type: 'category',
           label: 'Clients',
           items: [
+            'api/clients/index',
             'api/clients/backend',
+            'api/clients/annotation-mixin',
+            'api/clients/core-mixin',
+            'api/clients/data-collection-mixin',
+            'api/clients/hitl-mixin',
+            'api/clients/integration-mixin',
+            'api/clients/ml-mixin',
             'api/clients/agent',
             'api/clients/ray',
             'api/clients/base',

synapse_sdk/plugins/README.md

@@ -329,7 +329,7 @@ from .enums import LogCode, LOG_MESSAGES, UploadStatus
 from .exceptions import ExcelParsingError, ExcelSecurityError
 from .models import UploadParams
 from .run import UploadRun
-from .utils import ExcelMetadataUtils, ExcelSecurityConfig, PathAwareJSONEncoder
+from .utils import ExcelSecurityConfig, PathAwareJSONEncoder
 
 __all__ = [
     'UploadAction',
@@ -342,7 +342,6 @@ __all__ = [
     'ExcelParsingError',
     'PathAwareJSONEncoder',
     'ExcelSecurityConfig',
-    'ExcelMetadataUtils',
 ]
 ```
 

synapse_sdk/plugins/categories/base.py

@@ -12,6 +12,7 @@ from synapse_sdk.plugins.enums import RunMethod
 from synapse_sdk.plugins.exceptions import ActionError
 from synapse_sdk.plugins.models import PluginRelease, Run
 from synapse_sdk.plugins.upload import archive_and_upload, build_and_upload, download_and_upload
+from synapse_sdk.shared import init_sentry, needs_sentry_init
 from synapse_sdk.utils.module_loading import import_string
 from synapse_sdk.utils.pydantic.errors import pydantic_to_drf_error
 
@@ -181,6 +182,10 @@ class Action:
         for key, value in self.package_manager_options.items():
             runtime_env[self.plugin_package_manager][key] = value
 
+        # Sentry init if SENTRY_DSN is set
+        if needs_sentry_init():
+            runtime_env['worker_process_setup_hook'] = init_sentry
+
         # 맨 마지막에 진행되어야 함
         runtime_env['env_vars'] = self.envs
 
@@ -311,5 +316,7 @@ class Action:
     def ray_init(self):
         import ray
 
+        init_sentry()
+
         if not ray.is_initialized():
             ray.init(address=self.envs['RAY_ADDRESS'], ignore_reinit_error=True)

synapse_sdk/plugins/categories/export/actions/__init__.py

@@ -0,0 +1,3 @@
+from .export import ExportAction
+
+__all__ = ['ExportAction']

synapse_sdk/plugins/categories/export/actions/export/__init__.py

@@ -0,0 +1,28 @@
+from .action import ExportAction
+from .enums import ExportStatus, LogCode
+from .exceptions import ExportError, ExportTargetError, ExportValidationError
+from .models import ExportParams
+from .run import ExportRun
+from .utils import (
+    AssignmentExportTargetHandler,
+    ExportTargetHandler,
+    GroundTruthExportTargetHandler,
+    TargetHandlerFactory,
+    TaskExportTargetHandler,
+)
+
+__all__ = [
+    'ExportAction',
+    'ExportStatus',
+    'LogCode',
+    'ExportError',
+    'ExportTargetError',
+    'ExportValidationError',
+    'ExportParams',
+    'ExportRun',
+    'ExportTargetHandler',
+    'AssignmentExportTargetHandler',
+    'GroundTruthExportTargetHandler',
+    'TaskExportTargetHandler',
+    'TargetHandlerFactory',
+]

synapse_sdk/plugins/categories/export/actions/export/action.py

@@ -0,0 +1,160 @@
+from itertools import tee
+from typing import Any, Dict
+
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.utils.storage import get_pathlib
+
+from .enums import LogCode
+from .models import ExportParams
+from .run import ExportRun
+from .utils import TargetHandlerFactory
+
+
+@register_action
+class ExportAction(Action):
+    """Main export action for processing and exporting data from various targets.
+
+    Handles export operations including target validation, data retrieval,
+    and file generation. Supports export from assignment, ground_truth, and task
+    targets with comprehensive progress tracking and error handling.
+
+    Features:
+    - Multiple target source support (assignment, ground_truth, task)
+    - Filter validation and data retrieval
+    - Original file and data file export options
+    - Progress tracking with detailed metrics
+    - Comprehensive error logging
+    - Project configuration handling
+
+    Class Attributes:
+        name (str): Action identifier ('export')
+        category (PluginCategory): EXPORT category
+        method (RunMethod): JOB execution method
+        run_class (type): ExportRun for specialized logging
+        params_model (type): ExportParams for parameter validation
+        progress_categories (dict): Progress tracking configuration
+        metrics_categories (dict): Metrics collection configuration
+
+    Example:
+        >>> action = ExportAction(
+        ...     params={
+        ...         'name': 'Assignment Export',
+        ...         'storage': 1,
+        ...         'path': '/exports/assignments',
+        ...         'target': 'assignment',
+        ...         'filter': {'project': 123}
+        ...     },
+        ...     plugin_config=config
+        ... )
+        >>> result = action.start()
+    """
+
+    name = 'export'
+    category = PluginCategory.EXPORT
+    method = RunMethod.JOB
+    params_model = ExportParams
+    run_class = ExportRun
+    progress_categories = {
+        'dataset_conversion': {
+            'proportion': 100,
+        }
+    }
+    metrics_categories = {
+        'data_file': {
+            'stand_by': 0,
+            'failed': 0,
+            'success': 0,
+        },
+        'original_file': {
+            'stand_by': 0,
+            'failed': 0,
+            'success': 0,
+        },
+    }
+
+    def get_filtered_results(self, filters, handler):
+        """Get filtered target results.
+
+        Retrieves data from the specified target using the provided filters
+        through the appropriate target handler.
+
+        Args:
+            filters (dict): Filter criteria to apply
+            handler (ExportTargetHandler): Target-specific handler
+
+        Returns:
+            tuple: (results, count) where results is the data and count is total
+
+        Raises:
+            PydanticCustomError: If data retrieval fails
+        """
+        try:
+            result_list = handler.get_results(self.client, filters)
+            results = result_list[0]
+            count = result_list[1]
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset.'))
+        return results, count
+
+    def start(self) -> Dict[str, Any]:
+        """Start the export process.
+
+        Main entry point for export operations. Handles parameter preparation,
+        target handler selection, data retrieval, and export execution.
+
+        Returns:
+            Dict[str, Any]: Export results from the entrypoint
+
+        Raises:
+            Various exceptions based on validation and processing failures
+        """
+        self.run.log_message_with_code(LogCode.EXPORT_STARTED)
+
+        filters = {'expand': 'data', **self.params['filter']}
+        target = self.params['target']
+        handler = TargetHandlerFactory.get_handler(target)
+
+        self.params['results'], self.params['count'] = self.get_filtered_results(filters, handler)
+
+        if self.params['count'] == 0:
+            self.run.log_message_with_code(LogCode.NO_RESULTS_FOUND)
+        else:
+            self.run.log_message_with_code(LogCode.RESULTS_RETRIEVED, self.params['count'])
+
+        # For the 'ground_truth' target, retrieve project information from the first result and add configuration
+        if target == 'ground_truth':
+            try:
+                # Split generator into two using tee()
+                peek_iter, main_iter = tee(self.params['results'])
+                first_result = next(peek_iter)  # Peek first value only
+                project_pk = first_result['project']
+                project_info = self.client.get_project(project_pk)
+                self.params['project_id'] = project_pk
+                self.params['configuration'] = project_info.get('configuration', {})
+                self.params['results'] = main_iter  # Keep original generator intact
+            except (StopIteration, KeyError):
+                self.params['configuration'] = {}
+        # For the 'assignment' and 'task' targets, retrieve the project from the filter as before
+        elif target in ['assignment', 'task'] and 'project' in self.params['filter']:
+            project_pk = self.params['filter']['project']
+            project_info = self.client.get_project(project_pk)
+            self.params['configuration'] = project_info.get('configuration', {})
+
+        export_items = handler.get_export_item(self.params['results'])
+        storage = self.client.get_storage(self.params['storage'])
+        pathlib_cwd = get_pathlib(storage, self.params['path'])
+        exporter = self.entrypoint(self.run, export_items, pathlib_cwd, **self.params)
+
+        try:
+            result = exporter.export()
+            self.run.log_message_with_code(LogCode.EXPORT_COMPLETED)
+            return result
+        except Exception as e:
+            self.run.log_message_with_code(LogCode.EXPORT_FAILED, str(e))
+            raise

synapse_sdk/plugins/categories/export/actions/export/enums.py

@@ -0,0 +1,113 @@
+from enum import Enum
+
+from synapse_sdk.shared.enums import Context
+
+
+class ExportStatus(str, Enum):
+    """Export processing status enumeration.
+
+    Defines the possible states for export operations, data files, and export items
+    throughout the export process.
+
+    Attributes:
+        SUCCESS: Export completed successfully
+        FAILED: Export failed with errors
+        STAND_BY: Export waiting to be processed
+    """
+
+    SUCCESS = 'success'
+    FAILED = 'failed'
+    STAND_BY = 'stand_by'
+
+
+class LogCode(str, Enum):
+    """Type-safe logging codes for export operations.
+
+    Enumeration of all possible log events during export processing. Each code
+    corresponds to a specific event or error state with predefined message
+    templates and log levels.
+
+    The codes are organized by category:
+    - Validation codes (VALIDATION_FAILED, STORAGE_VALIDATION_FAILED, etc.)
+    - Export processing codes (EXPORT_STARTED, EXPORT_COMPLETED, etc.)
+    - File processing codes (ORIGINAL_FILE_EXPORTED, DATA_FILE_EXPORTED, etc.)
+    - Error handling codes (TARGET_HANDLER_ERROR, EXPORT_FAILED, etc.)
+
+    Each code maps to a configuration in LOG_MESSAGES with message template
+    and appropriate log level.
+    """
+
+    STORAGE_VALIDATION_FAILED = 'STORAGE_VALIDATION_FAILED'
+    FILTER_VALIDATION_FAILED = 'FILTER_VALIDATION_FAILED'
+    TARGET_VALIDATION_FAILED = 'TARGET_VALIDATION_FAILED'
+    VALIDATION_FAILED = 'VALIDATION_FAILED'
+    EXPORT_STARTED = 'EXPORT_STARTED'
+    EXPORT_COMPLETED = 'EXPORT_COMPLETED'
+    EXPORT_FAILED = 'EXPORT_FAILED'
+    NO_RESULTS_FOUND = 'NO_RESULTS_FOUND'
+    RESULTS_RETRIEVED = 'RESULTS_RETRIEVED'
+    ORIGINAL_FILE_EXPORTED = 'ORIGINAL_FILE_EXPORTED'
+    DATA_FILE_EXPORTED = 'DATA_FILE_EXPORTED'
+    FILE_EXPORT_FAILED = 'FILE_EXPORT_FAILED'
+    TARGET_HANDLER_ERROR = 'TARGET_HANDLER_ERROR'
+    NULL_DATA_DETECTED = 'NULL_DATA_DETECTED'
+
+
+LOG_MESSAGES = {
+    LogCode.STORAGE_VALIDATION_FAILED: {
+        'message': 'Storage validation failed.',
+        'level': Context.DANGER,
+    },
+    LogCode.FILTER_VALIDATION_FAILED: {
+        'message': 'Filter validation failed.',
+        'level': Context.DANGER,
+    },
+    LogCode.TARGET_VALIDATION_FAILED: {
+        'message': 'Target validation failed.',
+        'level': Context.DANGER,
+    },
+    LogCode.VALIDATION_FAILED: {
+        'message': 'Validation failed.',
+        'level': Context.DANGER,
+    },
+    LogCode.EXPORT_STARTED: {
+        'message': 'Export process started.',
+        'level': None,
+    },
+    LogCode.EXPORT_COMPLETED: {
+        'message': 'Export process completed.',
+        'level': None,
+    },
+    LogCode.EXPORT_FAILED: {
+        'message': 'Export process failed: {}',
+        'level': Context.DANGER,
+    },
+    LogCode.NO_RESULTS_FOUND: {
+        'message': 'No results found for export.',
+        'level': Context.WARNING,
+    },
+    LogCode.RESULTS_RETRIEVED: {
+        'message': 'Retrieved {} results for export',
+        'level': None,
+    },
+    LogCode.ORIGINAL_FILE_EXPORTED: {
+        'message': 'Original file exported successfully.',
+        'level': None,
+    },
+    LogCode.DATA_FILE_EXPORTED: {
+        'message': 'Data file exported successfully.',
+        'level': None,
+    },
+    LogCode.FILE_EXPORT_FAILED: {
+        'message': 'Failed to export file: {}',
+        'level': Context.DANGER,
+    },
+    LogCode.TARGET_HANDLER_ERROR: {
+        'message': 'Target handler error: {}',
+        'level': Context.DANGER,
+    },
+    LogCode.NULL_DATA_DETECTED: {
+        'message': 'Data is null for export item',
+        'level': Context.WARNING,
+    },
+}

synapse_sdk/plugins/categories/export/actions/export/exceptions.py

@@ -0,0 +1,53 @@
+class ExportError(Exception):
+    """Base exception for export-related errors.
+
+    This exception is raised when an export operation encounters errors
+    that prevent successful completion. It serves as the base class for
+    more specific export-related exceptions.
+
+    Used during export processing to handle various error conditions
+    such as validation failures, data access errors, or processing issues.
+
+    Example:
+        >>> if not validate_export_data(data):
+        ...     raise ExportError("Export data validation failed")
+    """
+
+    pass
+
+
+class ExportValidationError(ExportError):
+    """Exception raised when export parameter validation fails.
+
+    This exception is raised when export parameters or configuration
+    fail validation checks, preventing the export operation from starting.
+
+    Used during parameter validation to distinguish validation errors
+    from other types of export failures.
+
+    Example:
+        >>> if not storage_exists(storage_id):
+        ...     raise ExportValidationError(f"Storage {storage_id} does not exist")
+    """
+
+    pass
+
+
+class ExportTargetError(ExportError):
+    """Exception raised when export target handling encounters errors.
+
+    This exception is raised when target-specific operations (assignment,
+    ground_truth, task) fail due to data access issues, filter problems,
+    or target-specific validation failures.
+
+    Used during target data retrieval and processing to handle target-specific
+    errors separately from general export errors.
+
+    Example:
+        >>> try:
+        ...     results = client.list_assignments(params=filters)
+        ... except ClientError as e:
+        ...     raise ExportTargetError(f"Failed to retrieve assignments: {e}")
+    """
+
+    pass

synapse_sdk/plugins/categories/export/actions/export/models.py

@@ -0,0 +1,74 @@
+from typing import Annotated, Literal
+
+from pydantic import AfterValidator, BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.utils.pydantic.validators import non_blank
+
+from .utils import TargetHandlerFactory
+
+
+class ExportParams(BaseModel):
+    """Export action parameter validation model.
+
+    Defines and validates all parameters required for export operations.
+    Uses Pydantic for type validation and custom validators to ensure
+    storage and filter resources exist before processing.
+
+    Attributes:
+        name (str): Human-readable name for the export operation
+        description (str | None): Optional description of the export
+        storage (int): Storage ID where exported data will be saved
+        save_original_file (bool): Whether to save the original file
+        path (str): File system path where exported data will be saved
+        target (str): The target source to export data from (assignment, ground_truth, task)
+        filter (dict): Filter criteria to apply when retrieving data
+        extra_params (dict | None): Additional parameters for export customization.
+            Example: {"include_metadata": True, "compression": "gzip"}
+
+    Validation:
+        - name: Must be non-blank after validation
+        - storage: Must exist and be accessible via client API
+        - target: Must be one of the supported target types
+        - filter: Must be valid for the specified target type
+
+    Example:
+        >>> params = ExportParams(
+        ...     name="Assignment Export",
+        ...     storage=1,
+        ...     path="/exports/assignments",
+        ...     target="assignment",
+        ...     filter={"project": 123}
+        ... )
+    """
+
+    name: Annotated[str, AfterValidator(non_blank)]
+    description: str | None = None
+    storage: int
+    save_original_file: bool = True
+    path: str
+    target: Literal['assignment', 'ground_truth', 'task']
+    filter: dict
+    extra_params: dict | None = None
+
+    @field_validator('storage')
+    @staticmethod
+    def check_storage_exists(value, info):
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_storage(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get storage from Synapse backend.'))
+        return value
+
+    @field_validator('filter')
+    @staticmethod
+    def check_filter_by_target(value, info):
+        action = info.context['action']
+        client = action.client
+        target = action.params['target']
+        handler = TargetHandlerFactory.get_handler(target)
+        return handler.validate_filter(value, client)

synapse_sdk/plugins/categories/export/actions/export/run.py

@@ -0,0 +1,195 @@
+import json
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel
+
+from synapse_sdk.plugins.models import Run
+from synapse_sdk.shared.enums import Context
+
+from .enums import LOG_MESSAGES, ExportStatus, LogCode
+
+
+class ExportRun(Run):
+    """Export-specific run management class.
+
+    Extends the base Run class with export-specific logging capabilities
+    and event tracking. Provides type-safe logging using LogCode enums
+    and specialized methods for tracking export progress.
+
+    Manages logging for export events, data files, and export targets
+    throughout the export lifecycle. Each log entry includes status,
+    timestamps, and relevant metadata.
+
+    Attributes:
+        Inherits all attributes from base Run class plus export-specific
+        logging methods and nested model classes for structured logging.
+
+    Example:
+        >>> run = ExportRun(job_id, context)
+        >>> run.log_message_with_code(LogCode.EXPORT_STARTED)
+        >>> run.log_export_event(LogCode.RESULTS_RETRIEVED, target_id, count)
+    """
+
+    class ExportEventLog(BaseModel):
+        """Model for export event log entries.
+
+        Records significant events during export processing with
+        target identification and status information.
+
+        Attributes:
+            target_id (int): The ID of the export target
+            info (str | None): Optional additional information
+            status (Context): Event status/severity level
+            created (str): Timestamp when event occurred
+        """
+
+        target_id: int
+        info: str | None = None
+        status: Context
+        created: str
+
+    class DataFileLog(BaseModel):
+        """Model for data file export log entries.
+
+        Tracks the export status of individual data files during
+        export operations.
+
+        Attributes:
+            target_id (int): The ID of the target being exported
+            data_file_info (str | None): JSON information about the data file
+            status (ExportStatus): Export status (SUCCESS/FAILED/STAND_BY)
+            error (str | None): Error message if export failed
+            created (str): Timestamp when log entry was created
+        """
+
+        target_id: int
+        data_file_info: str | None
+        status: ExportStatus
+        error: str | None = None
+        created: str
+
+    class MetricsRecord(BaseModel):
+        """Model for export metrics tracking.
+
+        Records count-based metrics for monitoring export
+        progress and success rates.
+
+        Attributes:
+            stand_by (int): Number of items waiting to be processed
+            failed (int): Number of items that failed processing
+            success (int): Number of items successfully processed
+        """
+
+        stand_by: int
+        failed: int
+        success: int
+
+    def log_message_with_code(self, code: LogCode, *args, level: Optional[Context] = None):
+        """Log message using predefined code with type safety.
+
+        Args:
+            code (LogCode): The log message code
+            *args: Arguments to format the message
+            level (Context | None): Optional context level override
+        """
+        if code not in LOG_MESSAGES:
+            self.log_message(f'Unknown log code: {code}')
+            return
+
+        log_config = LOG_MESSAGES[code]
+        message = log_config['message'].format(*args) if args else log_config['message']
+        log_level = level or log_config['level'] or Context.INFO
+
+        # Always call log_message for basic logging
+        if log_level:
+            self.log_message(message, context=log_level.value)
+        else:
+            self.log_message(message)
+
+    def log_file(
+        self, log_type: str, target_id: int, data_file_info: dict, status: ExportStatus, error: str | None = None
+    ):
+        """Log export file information.
+
+        Args:
+            log_type (str): The type of log ('export_data_file' or 'export_original_file').
+            target_id (int): The ID of the data file.
+            data_file_info (dict): The JSON info of the data file.
+            status (ExportStatus): The status of the data file.
+            error (str | None): The error message, if any.
+        """
+        now = datetime.now().isoformat()
+        self.log(
+            log_type,
+            self.DataFileLog(
+                target_id=target_id,
+                data_file_info=json.dumps(data_file_info),
+                status=status.value,
+                error=error,
+                created=now,
+            ).model_dump(),
+        )
+
+    def log_export_event(self, code: LogCode, target_id: int, *args, level: Context | None = None):
+        """Log export event using predefined code.
+
+        Args:
+            code (str): The log message code.
+            target_id (int): The ID of the export target.
+            *args: Arguments to format the message.
+            level (Context | None): Optional context level override.
+        """
+        # Call log_message_with_code to handle the basic logging
+        self.log_message_with_code(code, *args, level=level)
+
+        # Also log the event for export-specific tracking
+        if code not in LOG_MESSAGES:
+            now = datetime.now().isoformat()
+            self.log(
+                'export_event',
+                self.ExportEventLog(
+                    target_id=target_id, info=f'Unknown log code: {code}', status=Context.DANGER, created=now
+                ).model_dump(),
+            )
+            return
+
+        log_config = LOG_MESSAGES[code]
+        message = log_config['message'].format(*args) if args else log_config['message']
+        log_level = level or log_config['level'] or Context.INFO
+
+        now = datetime.now().isoformat()
+        self.log(
+            'export_event',
+            self.ExportEventLog(info=message, status=log_level, target_id=target_id, created=now).model_dump(),
+        )
+
+    def log_metrics(self, record: MetricsRecord, category: str):
+        """Log export metrics.
+
+        Args:
+            record (MetricsRecord): The metrics record to log.
+            category (str): The category of the metrics.
+        """
+        record = self.MetricsRecord.model_validate(record)
+        self.set_metrics(value=record.model_dump(), category=category)
+
+    def export_log_json_file(
+        self,
+        target_id: int,
+        data_file_info: dict,
+        status: ExportStatus = ExportStatus.STAND_BY,
+        error: str | None = None,
+    ):
+        """Log export json data file."""
+        self.log_file('export_data_file', target_id, data_file_info, status, error)
+
+    def export_log_original_file(
+        self,
+        target_id: int,
+        data_file_info: dict,
+        status: ExportStatus = ExportStatus.STAND_BY,
+        error: str | None = None,
+    ):
+        """Log export origin data file."""
+        self.log_file('export_original_file', target_id, data_file_info, status, error)