synapse-sdk 1.0.0a23__py3-none-any.whl → 2025.12.3__py3-none-any.whl

synapse_sdk/plugins/categories/export/actions/export/exceptions.py

@@ -0,0 +1,53 @@
+class ExportError(Exception):
+    """Base exception for export-related errors.
+
+    This exception is raised when an export operation encounters errors
+    that prevent successful completion. It serves as the base class for
+    more specific export-related exceptions.
+
+    Used during export processing to handle various error conditions
+    such as validation failures, data access errors, or processing issues.
+
+    Example:
+        >>> if not validate_export_data(data):
+        ...     raise ExportError("Export data validation failed")
+    """
+
+    pass
+
+
+class ExportValidationError(ExportError):
+    """Exception raised when export parameter validation fails.
+
+    This exception is raised when export parameters or configuration
+    fail validation checks, preventing the export operation from starting.
+
+    Used during parameter validation to distinguish validation errors
+    from other types of export failures.
+
+    Example:
+        >>> if not storage_exists(storage_id):
+        ...     raise ExportValidationError(f"Storage {storage_id} does not exist")
+    """
+
+    pass
+
+
+class ExportTargetError(ExportError):
+    """Exception raised when export target handling encounters errors.
+
+    This exception is raised when target-specific operations (assignment,
+    ground_truth, task) fail due to data access issues, filter problems,
+    or target-specific validation failures.
+
+    Used during target data retrieval and processing to handle target-specific
+    errors separately from general export errors.
+
+    Example:
+        >>> try:
+        ...     results = client.list_assignments(params=filters)
+        ... except ClientError as e:
+        ...     raise ExportTargetError(f"Failed to retrieve assignments: {e}")
+    """
+
+    pass

synapse_sdk/plugins/categories/export/actions/export/models.py

@@ -0,0 +1,74 @@
+from typing import Annotated, Literal
+
+from pydantic import AfterValidator, BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.utils.pydantic.validators import non_blank
+
+from .utils import TargetHandlerFactory
+
+
+class ExportParams(BaseModel):
+    """Export action parameter validation model.
+
+    Defines and validates all parameters required for export operations.
+    Uses Pydantic for type validation and custom validators to ensure
+    storage and filter resources exist before processing.
+
+    Attributes:
+        name (str): Human-readable name for the export operation
+        description (str | None): Optional description of the export
+        storage (int): Storage ID where exported data will be saved
+        save_original_file (bool): Whether to save the original file
+        path (str): File system path where exported data will be saved
+        target (str): The target source to export data from (assignment, ground_truth, task)
+        filter (dict): Filter criteria to apply when retrieving data
+        extra_params (dict | None): Additional parameters for export customization.
+            Example: {"include_metadata": True, "compression": "gzip"}
+
+    Validation:
+        - name: Must be non-blank after validation
+        - storage: Must exist and be accessible via client API
+        - target: Must be one of the supported target types
+        - filter: Must be valid for the specified target type
+
+    Example:
+        >>> params = ExportParams(
+        ...     name="Assignment Export",
+        ...     storage=1,
+        ...     path="/exports/assignments",
+        ...     target="assignment",
+        ...     filter={"project": 123}
+        ... )
+    """
+
+    name: Annotated[str, AfterValidator(non_blank)]
+    description: str | None = None
+    storage: int
+    save_original_file: bool = True
+    path: str
+    target: Literal['assignment', 'ground_truth', 'task']
+    filter: dict
+    extra_params: dict | None = None
+
+    @field_validator('storage')
+    @staticmethod
+    def check_storage_exists(value, info):
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_storage(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get storage from Synapse backend.'))
+        return value
+
+    @field_validator('filter')
+    @staticmethod
+    def check_filter_by_target(value, info):
+        action = info.context['action']
+        client = action.client
+        target = action.params['target']
+        handler = TargetHandlerFactory.get_handler(target)
+        return handler.validate_filter(value, client)

synapse_sdk/plugins/categories/export/actions/export/run.py

@@ -0,0 +1,195 @@
+import json
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel
+
+from synapse_sdk.plugins.models import Run
+from synapse_sdk.shared.enums import Context
+
+from .enums import LOG_MESSAGES, ExportStatus, LogCode
+
+
+class ExportRun(Run):
+    """Export-specific run management class.
+
+    Extends the base Run class with export-specific logging capabilities
+    and event tracking. Provides type-safe logging using LogCode enums
+    and specialized methods for tracking export progress.
+
+    Manages logging for export events, data files, and export targets
+    throughout the export lifecycle. Each log entry includes status,
+    timestamps, and relevant metadata.
+
+    Attributes:
+        Inherits all attributes from base Run class plus export-specific
+        logging methods and nested model classes for structured logging.
+
+    Example:
+        >>> run = ExportRun(job_id, context)
+        >>> run.log_message_with_code(LogCode.EXPORT_STARTED)
+        >>> run.log_export_event(LogCode.RESULTS_RETRIEVED, target_id, count)
+    """
+
+    class ExportEventLog(BaseModel):
+        """Model for export event log entries.
+
+        Records significant events during export processing with
+        target identification and status information.
+
+        Attributes:
+            target_id (int): The ID of the export target
+            info (str | None): Optional additional information
+            status (Context): Event status/severity level
+            created (str): Timestamp when event occurred
+        """
+
+        target_id: int
+        info: str | None = None
+        status: Context
+        created: str
+
+    class DataFileLog(BaseModel):
+        """Model for data file export log entries.
+
+        Tracks the export status of individual data files during
+        export operations.
+
+        Attributes:
+            target_id (int): The ID of the target being exported
+            data_file_info (str | None): JSON information about the data file
+            status (ExportStatus): Export status (SUCCESS/FAILED/STAND_BY)
+            error (str | None): Error message if export failed
+            created (str): Timestamp when log entry was created
+        """
+
+        target_id: int
+        data_file_info: str | None
+        status: ExportStatus
+        error: str | None = None
+        created: str
+
+    class MetricsRecord(BaseModel):
+        """Model for export metrics tracking.
+
+        Records count-based metrics for monitoring export
+        progress and success rates.
+
+        Attributes:
+            stand_by (int): Number of items waiting to be processed
+            failed (int): Number of items that failed processing
+            success (int): Number of items successfully processed
+        """
+
+        stand_by: int
+        failed: int
+        success: int
+
+    def log_message_with_code(self, code: LogCode, *args, level: Optional[Context] = None):
+        """Log message using predefined code with type safety.
+
+        Args:
+            code (LogCode): The log message code
+            *args: Arguments to format the message
+            level (Context | None): Optional context level override
+        """
+        if code not in LOG_MESSAGES:
+            self.log_message(f'Unknown log code: {code}')
+            return
+
+        log_config = LOG_MESSAGES[code]
+        message = log_config['message'].format(*args) if args else log_config['message']
+        log_level = level or log_config['level'] or Context.INFO
+
+        # Always call log_message for basic logging
+        if log_level:
+            self.log_message(message, context=log_level.value)
+        else:
+            self.log_message(message)
+
+    def log_file(
+        self, log_type: str, target_id: int, data_file_info: dict, status: ExportStatus, error: str | None = None
+    ):
+        """Log export file information.
+
+        Args:
+            log_type (str): The type of log ('export_data_file' or 'export_original_file').
+            target_id (int): The ID of the data file.
+            data_file_info (dict): The JSON info of the data file.
+            status (ExportStatus): The status of the data file.
+            error (str | None): The error message, if any.
+        """
+        now = datetime.now().isoformat()
+        self.log(
+            log_type,
+            self.DataFileLog(
+                target_id=target_id,
+                data_file_info=json.dumps(data_file_info),
+                status=status.value,
+                error=error,
+                created=now,
+            ).model_dump(),
+        )
+
+    def log_export_event(self, code: LogCode, target_id: int, *args, level: Context | None = None):
+        """Log export event using predefined code.
+
+        Args:
+            code (str): The log message code.
+            target_id (int): The ID of the export target.
+            *args: Arguments to format the message.
+            level (Context | None): Optional context level override.
+        """
+        # Call log_message_with_code to handle the basic logging
+        self.log_message_with_code(code, *args, level=level)
+
+        # Also log the event for export-specific tracking
+        if code not in LOG_MESSAGES:
+            now = datetime.now().isoformat()
+            self.log(
+                'export_event',
+                self.ExportEventLog(
+                    target_id=target_id, info=f'Unknown log code: {code}', status=Context.DANGER, created=now
+                ).model_dump(),
+            )
+            return
+
+        log_config = LOG_MESSAGES[code]
+        message = log_config['message'].format(*args) if args else log_config['message']
+        log_level = level or log_config['level'] or Context.INFO
+
+        now = datetime.now().isoformat()
+        self.log(
+            'export_event',
+            self.ExportEventLog(info=message, status=log_level, target_id=target_id, created=now).model_dump(),
+        )
+
+    def log_metrics(self, record: MetricsRecord, category: str):
+        """Log export metrics.
+
+        Args:
+            record (MetricsRecord): The metrics record to log.
+            category (str): The category of the metrics.
+        """
+        record = self.MetricsRecord.model_validate(record)
+        self.set_metrics(value=record.model_dump(), category=category)
+
+    def export_log_json_file(
+        self,
+        target_id: int,
+        data_file_info: dict,
+        status: ExportStatus = ExportStatus.STAND_BY,
+        error: str | None = None,
+    ):
+        """Log export json data file."""
+        self.log_file('export_data_file', target_id, data_file_info, status, error)
+
+    def export_log_original_file(
+        self,
+        target_id: int,
+        data_file_info: dict,
+        status: ExportStatus = ExportStatus.STAND_BY,
+        error: str | None = None,
+    ):
+        """Log export origin data file."""
+        self.log_file('export_original_file', target_id, data_file_info, status, error)

synapse_sdk/plugins/categories/export/actions/export/utils.py

@@ -0,0 +1,187 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.i18n import gettext as _
+
+
+class ExportTargetHandler(ABC):
+    """
+    Abstract base class for handling export targets.
+
+    This class defines the blueprint for export target handlers, requiring the implementation
+    of methods to validate filters, retrieve results, and process collections of results.
+    """
+
+    @abstractmethod
+    def validate_filter(self, value: dict, client: Any):
+        """
+        Validate filter query params to request original data from api.
+
+        Args:
+            value (dict): The filter criteria to validate.
+            client (Any): The client used to validate the filter.
+
+        Raises:
+            PydanticCustomError: If the filter criteria are invalid.
+
+        Returns:
+            dict: The validated filter criteria.
+        """
+        pass
+
+    @abstractmethod
+    def get_results(self, client: Any, filters: dict):
+        """
+        Retrieve original data from target sources.
+
+        Args:
+            client (Any): The client used to retrieve the results.
+            filters (dict): The filter criteria to apply.
+
+        Returns:
+            tuple: A tuple containing the results and the total count of results.
+        """
+        pass
+
+    @abstractmethod
+    def get_export_item(self, results):
+        """
+        Providing elements to build export data.
+
+        Args:
+            results (list): The results to process.
+
+        Yields:
+            generator: A generator that yields processed data items.
+        """
+        pass
+
+
+class AssignmentExportTargetHandler(ExportTargetHandler):
+    """Handler for assignment target exports.
+
+    Implements ExportTargetHandler interface for assignment-specific
+    export operations including validation, data retrieval, and processing.
+    """
+
+    def validate_filter(self, value: dict, client: Any):
+        if 'project' not in value:
+            raise PydanticCustomError('missing_field', _('Project is required for Assignment.'))
+        try:
+            client.list_assignments(params=value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get Assignment.'))
+        return value
+
+    def get_results(self, client: Any, filters: dict):
+        return client.list_assignments(params=filters, list_all=True)
+
+    def get_export_item(self, results):
+        for result in results:
+            yield {
+                'data': result['data'],
+                'files': result['file'],
+                'id': result['id'],
+            }
+
+
+class GroundTruthExportTargetHandler(ExportTargetHandler):
+    """Handler for ground truth target exports.
+
+    Implements ExportTargetHandler interface for ground truth dataset
+    export operations including validation, data retrieval, and processing.
+    """
+
+    def validate_filter(self, value: dict, client: Any):
+        if 'ground_truth_dataset_version' not in value:
+            raise PydanticCustomError('missing_field', _('Ground Truth dataset version is required.'))
+        try:
+            client.get_ground_truth_version(value['ground_truth_dataset_version'])
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset version.'))
+        return value
+
+    def get_results(self, client: Any, filters: dict):
+        filters['ground_truth_dataset_versions'] = filters.pop('ground_truth_dataset_version')
+        return client.list_ground_truth_events(params=filters, list_all=True)
+
+    def get_export_item(self, results):
+        for result in results:
+            files_key = next(iter(result['data_unit']['files']))
+            yield {
+                'data': result['data'],
+                'files': result['data_unit']['files'][files_key],
+                'id': result['id'],
+            }
+
+
+class TaskExportTargetHandler(ExportTargetHandler):
+    """Handler for task target exports.
+
+    Implements ExportTargetHandler interface for task-specific
+    export operations including validation, data retrieval, and processing.
+    """
+
+    def validate_filter(self, value: dict, client: Any):
+        if 'project' not in value:
+            raise PydanticCustomError('missing_field', _('Project is required for Task.'))
+        try:
+            client.list_tasks(params=value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get Task.'))
+        return value
+
+    def get_results(self, client: Any, filters: dict):
+        filters['expand'] = ['data_unit', 'assignment', 'workshop']
+        return client.list_tasks(params=filters, list_all=True)
+
+    def get_export_item(self, results):
+        for result in results:
+            files_key = next(iter(result['data_unit']['files']))
+            yield {
+                'data': result['data'],
+                'files': result['data_unit']['files'][files_key],
+                'id': result['id'],
+            }
+
+
+class TargetHandlerFactory:
+    """Factory class for creating export target handlers.
+
+    Provides a centralized way to create appropriate target handlers
+    based on the target type. Supports assignment, ground_truth, and task targets.
+
+    Example:
+        >>> handler = TargetHandlerFactory.get_handler('assignment')
+        >>> isinstance(handler, AssignmentExportTargetHandler)
+        True
+    """
+
+    @staticmethod
+    def get_handler(target: str) -> ExportTargetHandler:
+        """Get the appropriate target handler for the given target type.
+
+        Args:
+            target (str): The target type ('assignment', 'ground_truth', 'task')
+
+        Returns:
+            ExportTargetHandler: The appropriate handler instance
+
+        Raises:
+            ValueError: If the target type is not supported
+
+        Example:
+            >>> handler = TargetHandlerFactory.get_handler('assignment')
+            >>> handler.validate_filter({'project': 123}, client)
+        """
+        if target == 'assignment':
+            return AssignmentExportTargetHandler()
+        elif target == 'ground_truth':
+            return GroundTruthExportTargetHandler()
+        elif target == 'task':
+            return TaskExportTargetHandler()
+        else:
+            raise ValueError(f'Unknown target: {target}')

synapse_sdk/plugins/categories/export/templates/config.yaml

@@ -0,0 +1,21 @@
+actions:
+  export:
+    entrypoint: plugin.export.Exporter
+    # Filter to only export action specific annotation types (e.g., only export image annotations project)
+    # Supported types: image, video, audio, text, pcd, prompt
+    annotation_types:
+      - image
+      - video
+      - audio
+      - text
+      - pcd
+      - prompt
+
+# Filter to only export specific data types (e.g., only export image data)
+# Supported types: image, video, audio, text, pcd
+data_types:
+  - image
+  - video
+  - audio
+  - text
+  - pcd