synapse-sdk 1.0.0a35__py3-none-any.whl → 2025.11.7__py3-none-any.whl

synapse_sdk/plugins/categories/export/actions/export/models.py

@@ -0,0 +1,74 @@
+from typing import Annotated, Literal
+
+from pydantic import AfterValidator, BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.utils.pydantic.validators import non_blank
+
+from .utils import TargetHandlerFactory
+
+
+class ExportParams(BaseModel):
+    """Export action parameter validation model.
+
+    Defines and validates all parameters required for export operations.
+    Uses Pydantic for type validation and custom validators to ensure
+    storage and filter resources exist before processing.
+
+    Attributes:
+        name (str): Human-readable name for the export operation
+        description (str | None): Optional description of the export
+        storage (int): Storage ID where exported data will be saved
+        save_original_file (bool): Whether to save the original file
+        path (str): File system path where exported data will be saved
+        target (str): The target source to export data from (assignment, ground_truth, task)
+        filter (dict): Filter criteria to apply when retrieving data
+        extra_params (dict | None): Additional parameters for export customization.
+            Example: {"include_metadata": True, "compression": "gzip"}
+
+    Validation:
+        - name: Must be non-blank after validation
+        - storage: Must exist and be accessible via client API
+        - target: Must be one of the supported target types
+        - filter: Must be valid for the specified target type
+
+    Example:
+        >>> params = ExportParams(
+        ...     name="Assignment Export",
+        ...     storage=1,
+        ...     path="/exports/assignments",
+        ...     target="assignment",
+        ...     filter={"project": 123}
+        ... )
+    """
+
+    name: Annotated[str, AfterValidator(non_blank)]
+    description: str | None = None
+    storage: int
+    save_original_file: bool = True
+    path: str
+    target: Literal['assignment', 'ground_truth', 'task']
+    filter: dict
+    extra_params: dict | None = None
+
+    @field_validator('storage')
+    @staticmethod
+    def check_storage_exists(value, info):
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_storage(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get storage from Synapse backend.'))
+        return value
+
+    @field_validator('filter')
+    @staticmethod
+    def check_filter_by_target(value, info):
+        action = info.context['action']
+        client = action.client
+        target = action.params['target']
+        handler = TargetHandlerFactory.get_handler(target)
+        return handler.validate_filter(value, client)

synapse_sdk/plugins/categories/export/actions/export/run.py

@@ -0,0 +1,195 @@
+import json
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel
+
+from synapse_sdk.plugins.models import Run
+from synapse_sdk.shared.enums import Context
+
+from .enums import LOG_MESSAGES, ExportStatus, LogCode
+
+
+class ExportRun(Run):
+    """Export-specific run management class.
+
+    Extends the base Run class with export-specific logging capabilities
+    and event tracking. Provides type-safe logging using LogCode enums
+    and specialized methods for tracking export progress.
+
+    Manages logging for export events, data files, and export targets
+    throughout the export lifecycle. Each log entry includes status,
+    timestamps, and relevant metadata.
+
+    Attributes:
+        Inherits all attributes from base Run class plus export-specific
+        logging methods and nested model classes for structured logging.
+
+    Example:
+        >>> run = ExportRun(job_id, context)
+        >>> run.log_message_with_code(LogCode.EXPORT_STARTED)
+        >>> run.log_export_event(LogCode.RESULTS_RETRIEVED, target_id, count)
+    """
+
+    class ExportEventLog(BaseModel):
+        """Model for export event log entries.
+
+        Records significant events during export processing with
+        target identification and status information.
+
+        Attributes:
+            target_id (int): The ID of the export target
+            info (str | None): Optional additional information
+            status (Context): Event status/severity level
+            created (str): Timestamp when event occurred
+        """
+
+        target_id: int
+        info: str | None = None
+        status: Context
+        created: str
+
+    class DataFileLog(BaseModel):
+        """Model for data file export log entries.
+
+        Tracks the export status of individual data files during
+        export operations.
+
+        Attributes:
+            target_id (int): The ID of the target being exported
+            data_file_info (str | None): JSON information about the data file
+            status (ExportStatus): Export status (SUCCESS/FAILED/STAND_BY)
+            error (str | None): Error message if export failed
+            created (str): Timestamp when log entry was created
+        """
+
+        target_id: int
+        data_file_info: str | None
+        status: ExportStatus
+        error: str | None = None
+        created: str
+
+    class MetricsRecord(BaseModel):
+        """Model for export metrics tracking.
+
+        Records count-based metrics for monitoring export
+        progress and success rates.
+
+        Attributes:
+            stand_by (int): Number of items waiting to be processed
+            failed (int): Number of items that failed processing
+            success (int): Number of items successfully processed
+        """
+
+        stand_by: int
+        failed: int
+        success: int
+
+    def log_message_with_code(self, code: LogCode, *args, level: Optional[Context] = None):
+        """Log message using predefined code with type safety.
+
+        Args:
+            code (LogCode): The log message code
+            *args: Arguments to format the message
+            level (Context | None): Optional context level override
+        """
+        if code not in LOG_MESSAGES:
+            self.log_message(f'Unknown log code: {code}')
+            return
+
+        log_config = LOG_MESSAGES[code]
+        message = log_config['message'].format(*args) if args else log_config['message']
+        log_level = level or log_config['level'] or Context.INFO
+
+        # Always call log_message for basic logging
+        if log_level:
+            self.log_message(message, context=log_level.value)
+        else:
+            self.log_message(message)
+
+    def log_file(
+        self, log_type: str, target_id: int, data_file_info: dict, status: ExportStatus, error: str | None = None
+    ):
+        """Log export file information.
+
+        Args:
+            log_type (str): The type of log ('export_data_file' or 'export_original_file').
+            target_id (int): The ID of the data file.
+            data_file_info (dict): The JSON info of the data file.
+            status (ExportStatus): The status of the data file.
+            error (str | None): The error message, if any.
+        """
+        now = datetime.now().isoformat()
+        self.log(
+            log_type,
+            self.DataFileLog(
+                target_id=target_id,
+                data_file_info=json.dumps(data_file_info),
+                status=status.value,
+                error=error,
+                created=now,
+            ).model_dump(),
+        )
+
+    def log_export_event(self, code: LogCode, target_id: int, *args, level: Context | None = None):
+        """Log export event using predefined code.
+
+        Args:
+            code (str): The log message code.
+            target_id (int): The ID of the export target.
+            *args: Arguments to format the message.
+            level (Context | None): Optional context level override.
+        """
+        # Call log_message_with_code to handle the basic logging
+        self.log_message_with_code(code, *args, level=level)
+
+        # Also log the event for export-specific tracking
+        if code not in LOG_MESSAGES:
+            now = datetime.now().isoformat()
+            self.log(
+                'export_event',
+                self.ExportEventLog(
+                    target_id=target_id, info=f'Unknown log code: {code}', status=Context.DANGER, created=now
+                ).model_dump(),
+            )
+            return
+
+        log_config = LOG_MESSAGES[code]
+        message = log_config['message'].format(*args) if args else log_config['message']
+        log_level = level or log_config['level'] or Context.INFO
+
+        now = datetime.now().isoformat()
+        self.log(
+            'export_event',
+            self.ExportEventLog(info=message, status=log_level, target_id=target_id, created=now).model_dump(),
+        )
+
+    def log_metrics(self, record: MetricsRecord, category: str):
+        """Log export metrics.
+
+        Args:
+            record (MetricsRecord): The metrics record to log.
+            category (str): The category of the metrics.
+        """
+        record = self.MetricsRecord.model_validate(record)
+        self.set_metrics(value=record.model_dump(), category=category)
+
+    def export_log_json_file(
+        self,
+        target_id: int,
+        data_file_info: dict,
+        status: ExportStatus = ExportStatus.STAND_BY,
+        error: str | None = None,
+    ):
+        """Log export json data file."""
+        self.log_file('export_data_file', target_id, data_file_info, status, error)
+
+    def export_log_original_file(
+        self,
+        target_id: int,
+        data_file_info: dict,
+        status: ExportStatus = ExportStatus.STAND_BY,
+        error: str | None = None,
+    ):
+        """Log export origin data file."""
+        self.log_file('export_original_file', target_id, data_file_info, status, error)

synapse_sdk/plugins/categories/export/actions/{export.py → export/utils.py}

@@ -1,15 +1,10 @@
 from abc import ABC, abstractmethod
-from typing import Any, Literal
+from typing import Any
 
-from pydantic import BaseModel, field_validator
 from pydantic_core import PydanticCustomError
 
 from synapse_sdk.clients.exceptions import ClientError
 from synapse_sdk.i18n import gettext as _
-from synapse_sdk.plugins.categories.base import Action
-from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import PluginCategory, RunMethod
-from synapse_sdk.utils.storage import get_pathlib
 
 
 class ExportTargetHandler(ABC):
@@ -66,6 +61,12 @@ class ExportTargetHandler(ABC):
 
 
 class AssignmentExportTargetHandler(ExportTargetHandler):
+    """Handler for assignment target exports.
+
+    Implements ExportTargetHandler interface for assignment-specific
+    export operations including validation, data retrieval, and processing.
+    """
+
     def validate_filter(self, value: dict, client: Any):
         if 'project' not in value:
             raise PydanticCustomError('missing_field', _('Project is required for Assignment.'))
@@ -88,6 +89,12 @@ class AssignmentExportTargetHandler(ExportTargetHandler):
 
 
 class GroundTruthExportTargetHandler(ExportTargetHandler):
+    """Handler for ground truth target exports.
+
+    Implements ExportTargetHandler interface for ground truth dataset
+    export operations including validation, data retrieval, and processing.
+    """
+
     def validate_filter(self, value: dict, client: Any):
         if 'ground_truth_dataset_version' not in value:
             raise PydanticCustomError('missing_field', _('Ground Truth dataset version is required.'))
@@ -107,11 +114,17 @@ class GroundTruthExportTargetHandler(ExportTargetHandler):
             yield {
                 'data': result['data'],
                 'files': result['data_unit']['files'][files_key],
-                'id': result['ground_truth'],
+                'id': result['id'],
             }
 
 
 class TaskExportTargetHandler(ExportTargetHandler):
+    """Handler for task target exports.
+
+    Implements ExportTargetHandler interface for task-specific
+    export operations including validation, data retrieval, and processing.
+    """
+
     def validate_filter(self, value: dict, client: Any):
         if 'project' not in value:
             raise PydanticCustomError('missing_field', _('Project is required for Task.'))
@@ -122,7 +135,7 @@ class TaskExportTargetHandler(ExportTargetHandler):
         return value
 
     def get_results(self, client: Any, filters: dict):
-        filters['expand'] = 'data_unit'
+        filters['expand'] = ['data_unit', 'assignment', 'workshop']
         return client.list_tasks(params=filters, list_all=True)
 
     def get_export_item(self, results):
@@ -136,8 +149,34 @@ class TaskExportTargetHandler(ExportTargetHandler):
 
 
 class TargetHandlerFactory:
+    """Factory class for creating export target handlers.
+
+    Provides a centralized way to create appropriate target handlers
+    based on the target type. Supports assignment, ground_truth, and task targets.
+
+    Example:
+        >>> handler = TargetHandlerFactory.get_handler('assignment')
+        >>> isinstance(handler, AssignmentExportTargetHandler)
+        True
+    """
+
     @staticmethod
     def get_handler(target: str) -> ExportTargetHandler:
+        """Get the appropriate target handler for the given target type.
+
+        Args:
+            target (str): The target type ('assignment', 'ground_truth', 'task')
+
+        Returns:
+            ExportTargetHandler: The appropriate handler instance
+
+        Raises:
+            ValueError: If the target type is not supported
+
+        Example:
+            >>> handler = TargetHandlerFactory.get_handler('assignment')
+            >>> handler.validate_filter({'project': 123}, client)
+        """
         if target == 'assignment':
             return AssignmentExportTargetHandler()
         elif target == 'ground_truth':
@@ -146,77 +185,3 @@ class TargetHandlerFactory:
             return TaskExportTargetHandler()
         else:
             raise ValueError(f'Unknown target: {target}')
-
-
-class ExportParams(BaseModel):
-    """
-    Parameters for the export action.
-
-    Attributes:
-        storage (int): The storage ID to save the exported data.
-        save_original_file (bool): Whether to save the original file.
-        path (str): The path to save the exported data.
-        target (str): The target source to export data from. (ex. ground_truth, assignment, task)
-        filter (dict): The filter criteria to apply.
-    """
-
-    storage: int
-    save_original_file: bool = True
-    path: str
-    target: Literal['assignment', 'ground_truth', 'task']
-    filter: dict
-
-    @field_validator('storage')
-    @staticmethod
-    def check_storage_exists(value, info):
-        action = info.context['action']
-        client = action.client
-        try:
-            client.get_storage(value)
-        except ClientError:
-            raise PydanticCustomError('client_error', _('Unable to get storage from Synapse backend.'))
-        return value
-
-    @field_validator('filter')
-    @staticmethod
-    def check_filter_by_target(value, info):
-        action = info.context['action']
-        client = action.client
-        target = action.params['target']
-        handler = TargetHandlerFactory.get_handler(target)
-        return handler.validate_filter(value, client)
-
-
-@register_action
-class ExportAction(Action):
-    name = 'export'
-    category = PluginCategory.EXPORT
-    method = RunMethod.JOB
-    params_model = ExportParams
-    progress_categories = {
-        'dataset_conversion': {
-            'proportion': 100,
-        }
-    }
-
-    def get_filtered_results(self, filters, handler):
-        """Get filtered target results."""
-        try:
-            result_list = handler.get_results(self.client, filters)
-            results = result_list[0]
-            count = result_list[1]
-        except ClientError:
-            raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset.'))
-        return results, count
-
-    def start(self):
-        filters = {'expand': 'data', **self.params['filter']}
-        target = self.params['target']
-        handler = TargetHandlerFactory.get_handler(target)
-
-        self.params['results'], self.params['count'] = self.get_filtered_results(filters, handler)
-        export_items = handler.get_export_item(self.params['results'])
-
-        storage = self.client.get_storage(self.params['storage'])
-        pathlib_cwd = get_pathlib(storage, self.params['path'])
-        return self.entrypoint(self.run, export_items, pathlib_cwd, **self.params)

synapse_sdk/plugins/categories/export/templates/config.yaml

@@ -1,3 +1,21 @@
 actions:
   export:
-    entrypoint: plugin.export.export
+    entrypoint: plugin.export.Exporter
+    # Filter to only export action specific annotation types (e.g., only export image annotations project)
+    # Supported types: image, video, audio, text, pcd, prompt
+    annotation_types:
+      - image
+      - video
+      - audio
+      - text
+      - pcd
+      - prompt
+
+# Filter to only export specific data types (e.g., only export image data)
+# Supported types: image, video, audio, text, pcd
+data_types:
+  - image
+  - video
+  - audio
+  - text
+  - pcd