synapse-sdk 1.0.0b5__py3-none-any.whl → 2025.12.3__py3-none-any.whl

synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/base.py

@@ -0,0 +1,170 @@
+"""Abstract base classes for ToTask strategies."""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Tuple
+
+from synapse_sdk.clients.backend import BackendClient
+
+from ..enums import AnnotationMethod
+from ..models import MetricsRecord
+
+
+class ToTaskContext:
+    """Shared context for ToTask action execution."""
+
+    def __init__(
+        self,
+        params: Dict[str, Any],
+        client: BackendClient,
+        logger: Any,
+        entrypoint: Any = None,
+        config: Optional[Dict[str, Any]] = None,
+        plugin_config: Optional[Dict[str, Any]] = None,
+        job_id: Optional[str] = None,
+        progress_categories: Optional[Dict[str, Any]] = None,
+        metrics_categories: Optional[Dict[str, Any]] = None,
+        project: Optional[Dict[str, Any]] = None,
+        data_collection: Optional[Dict[str, Any]] = None,
+        task_ids: Optional[List[int]] = None,
+        metrics: Optional[MetricsRecord] = None,
+        annotation_method: Optional[AnnotationMethod] = None,
+    ):
+        self.params = params
+        self.client = client
+        self.logger = logger
+        self.entrypoint = entrypoint
+        self.config = config
+        self.plugin_config = plugin_config
+        self.job_id = job_id
+        self.progress_categories = progress_categories
+        self.metrics_categories = metrics_categories
+        self.project = project
+        self.data_collection = data_collection
+        self.task_ids = task_ids or []
+        self.metrics = metrics or MetricsRecord(stand_by=0, failed=0, success=0)
+        self.annotation_method = annotation_method
+        self.temp_files: List[str] = []
+        self.rollback_actions: List[callable] = []
+
+    def add_temp_file(self, file_path: str):
+        """Track temporary files for cleanup."""
+        self.temp_files.append(file_path)
+
+    def add_rollback_action(self, action: callable):
+        """Add rollback action for error recovery."""
+        self.rollback_actions.append(action)
+
+    def update_metrics(self, success_count: int, failed_count: int, total_count: int):
+        """Update execution metrics."""
+        self.metrics = MetricsRecord(
+            stand_by=total_count - success_count - failed_count, failed=failed_count, success=success_count
+        )
+
+
+class ValidationStrategy(ABC):
+    """Abstract base class for validation strategies."""
+
+    @abstractmethod
+    def validate(self, context: ToTaskContext) -> Dict[str, Any]:
+        """Validate specific aspects of the ToTask execution.
+
+        Args:
+            context: Shared context for the action execution
+
+        Returns:
+            Dict with 'success' boolean and optional 'error' message
+        """
+        pass
+
+
+class AnnotationStrategy(ABC):
+    """Abstract base class for annotation strategies."""
+
+    @abstractmethod
+    def process_task(self, context: ToTaskContext, task_id: int, task_data: Dict[str, Any], **kwargs) -> Dict[str, Any]:
+        """Process a single task for annotation.
+
+        Args:
+            context: Shared context for the action execution
+            task_id: The task ID to process
+            task_data: The task data dictionary
+            **kwargs: Additional method-specific parameters
+
+        Returns:
+            Dict with 'success' boolean and optional 'error' message
+        """
+        pass
+
+
+class PreProcessorStrategy(ABC):
+    """Abstract base class for pre-processor management strategies."""
+
+    @abstractmethod
+    def get_preprocessor_info(self, context: ToTaskContext, preprocessor_id: int) -> Dict[str, Any]:
+        """Get pre-processor information.
+
+        Args:
+            context: Shared context for the action execution
+            preprocessor_id: The pre-processor ID
+
+        Returns:
+            Dict with pre-processor info or error
+        """
+        pass
+
+    @abstractmethod
+    def ensure_preprocessor_running(self, context: ToTaskContext, preprocessor_code: str) -> Dict[str, Any]:
+        """Ensure pre-processor is running.
+
+        Args:
+            context: Shared context for the action execution
+            preprocessor_code: The pre-processor code
+
+        Returns:
+            Dict indicating success or failure
+        """
+        pass
+
+
+class DataExtractionStrategy(ABC):
+    """Abstract base class for data extraction strategies."""
+
+    @abstractmethod
+    def extract_data(self, context: ToTaskContext, task_data: Dict[str, Any]) -> Tuple[Optional[str], Optional[str]]:
+        """Extract required data from task.
+
+        Args:
+            context: Shared context for the action execution
+            task_data: The task data dictionary
+
+        Returns:
+            Tuple of extracted data values
+        """
+        pass
+
+
+class MetricsStrategy(ABC):
+    """Abstract base class for metrics strategies."""
+
+    @abstractmethod
+    def update_progress(self, context: ToTaskContext, current: int, total: int):
+        """Update progress tracking.
+
+        Args:
+            context: Shared context for the action execution
+            current: Current progress count
+            total: Total items to process
+        """
+        pass
+
+    @abstractmethod
+    def record_task_result(self, context: ToTaskContext, task_id: int, success: bool, error: Optional[str] = None):
+        """Record the result of processing a single task.
+
+        Args:
+            context: Shared context for the action execution
+            task_id: The task ID that was processed
+            success: Whether the task processing was successful
+            error: Error message if unsuccessful
+        """
+        pass

synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/extraction.py

@@ -0,0 +1,83 @@
+"""Data extraction strategies for ToTask action."""
+
+from typing import Any, Dict, Optional, Tuple
+
+from .base import DataExtractionStrategy, ToTaskContext
+
+
+class FileUrlExtractionStrategy(DataExtractionStrategy):
+    """Strategy for extracting file URLs from task data."""
+
+    def extract_data(self, context: ToTaskContext, task_data: Dict[str, Any]) -> str | None:
+        """Extract primary file URL from task data.
+
+        Args:
+            context: Shared context for the action execution
+            task_data: The task data dictionary
+
+        Returns:
+            str: The primary file URL or None if not found
+        """
+        try:
+            # This implementation follows the original _extract_primary_file_url logic
+            data_unit = task_data.get('data_unit')
+            if not data_unit:
+                return None
+
+            data_unit_files = data_unit.get('files', {})
+            if not data_unit_files:
+                return None
+
+            # Find primary file URL
+            for key in data_unit_files:
+                if data_unit_files[key]['is_primary']:
+                    return data_unit_files[key]['url']
+
+            return None
+
+        except Exception as e:
+            context.logger.log_message_with_code(
+                context.logger.LogCode.DATA_EXTRACTION_FAILED
+                if hasattr(context.logger, 'LogCode')
+                else 'DATA_EXTRACTION_FAILED',
+                str(e),
+            )
+            return None
+
+
+class InferenceDataExtractionStrategy(DataExtractionStrategy):
+    """Strategy for extracting inference data from task data."""
+
+    def extract_data(self, context: ToTaskContext, task_data: Dict[str, Any]) -> Tuple[Optional[str], Optional[str]]:
+        """Extract data needed for inference processing.
+
+        Args:
+            context: Shared context for the action execution
+            task_data: The task data dictionary
+
+        Returns:
+            Tuple of (primary_file_url, inference_metadata)
+        """
+        try:
+            # Reuse the file URL extraction logic
+            file_strategy = FileUrlExtractionStrategy()
+            primary_url = file_strategy.extract_data(context, task_data)
+
+            # Extract additional inference-specific metadata
+            data_unit = task_data.get('data_unit', {})
+            inference_metadata = {
+                'data_unit_id': data_unit.get('id'),
+                'task_id': task_data.get('id'),
+                'additional_params': context.params.get('inference_params', {}),
+            }
+
+            return primary_url, str(inference_metadata) if inference_metadata else None
+
+        except Exception as e:
+            context.logger.log_message_with_code(
+                context.logger.LogCode.DATA_EXTRACTION_FAILED
+                if hasattr(context.logger, 'LogCode')
+                else 'DATA_EXTRACTION_FAILED',
+                str(e),
+            )
+            return None, None

synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/metrics.py

@@ -0,0 +1,92 @@
+"""Metrics and progress tracking strategies for ToTask action."""
+
+from typing import Optional
+
+from ..enums import AnnotateTaskDataStatus, LogCode
+from .base import MetricsStrategy, ToTaskContext
+
+
+class ProgressTrackingStrategy(MetricsStrategy):
+    """Strategy for tracking progress and metrics during task processing."""
+
+    def update_progress(self, context: ToTaskContext, current: int, total: int):
+        """Update progress tracking.
+
+        Args:
+            context: Shared context for the action execution
+            current: Current progress count
+            total: Total items to process
+        """
+        try:
+            context.logger.set_progress(current, total, category='annotate_task_data')
+        except Exception as e:
+            context.logger.log_message_with_code(LogCode.PROGRESS_UPDATE_FAILED, str(e))
+
+    def record_task_result(self, context: ToTaskContext, task_id: int, success: bool, error: Optional[str] = None):
+        """Record the result of processing a single task.
+
+        Args:
+            context: Shared context for the action execution
+            task_id: The task ID that was processed
+            success: Whether the task processing was successful
+            error: Error message if unsuccessful
+        """
+        try:
+            if success:
+                status = AnnotateTaskDataStatus.SUCCESS
+                log_data = {'task_id': task_id}
+            else:
+                status = AnnotateTaskDataStatus.FAILED
+                log_data = {'task_id': task_id, 'error': error or 'Unknown error'}
+
+            context.logger.log_annotate_task_data(log_data, status)
+
+        except Exception as e:
+            context.logger.log_message_with_code(LogCode.METRICS_RECORDING_FAILED, str(e))
+
+    def update_metrics(self, context: ToTaskContext, total_tasks: int, success_count: int, failed_count: int):
+        """Update execution metrics.
+
+        Args:
+            context: Shared context for the action execution
+            total_tasks: Total number of tasks
+            success_count: Number of successful tasks
+            failed_count: Number of failed tasks
+        """
+        try:
+            stand_by_count = total_tasks - success_count - failed_count
+            context.update_metrics(success_count, failed_count, total_tasks)
+
+            # Update metrics in the logger
+            metrics_data = {
+                'stand_by': stand_by_count,
+                'failed': failed_count,
+                'success': success_count,
+            }
+            context.logger.log_metrics(metrics_data, 'annotate_task_data')
+
+        except Exception as e:
+            context.logger.log_message_with_code(LogCode.METRICS_UPDATE_FAILED, str(e))
+
+    def finalize_metrics(self, context: ToTaskContext):
+        """Finalize metrics at the end of processing.
+
+        Args:
+            context: Shared context for the action execution
+        """
+        try:
+            metrics = context.metrics
+            total_tasks = len(context.task_ids)
+
+            context.logger.log_message_with_code(LogCode.ANNOTATION_COMPLETED, metrics.success, metrics.failed)
+
+            # Final progress update - only set completion if some tasks succeeded
+            if metrics.success > 0:
+                # Success: Set completion progress
+                self.update_progress(context, total_tasks, total_tasks)
+            else:
+                # Failure: Mark as failed (all tasks failed)
+                context.logger.set_progress_failed(category='annotate_task_data')
+
+        except Exception as e:
+            context.logger.log_message_with_code(LogCode.METRICS_FINALIZATION_FAILED, str(e))

synapse_sdk/plugins/categories/pre_annotation/actions/to_task/strategies/preprocessor.py

@@ -0,0 +1,243 @@
+"""Pre-processor management strategies for ToTask action."""
+
+import time
+from typing import Any, Dict, List
+
+from .base import PreProcessorStrategy, ToTaskContext
+
+
+class PreProcessorManagementStrategy(PreProcessorStrategy):
+    """Strategy for managing pre-processor lifecycle."""
+
+    def _get_preprocessor_config(self, context: ToTaskContext, preprocessor_id: int) -> Dict[str, Any]:
+        """Retrieve pre-processor configuration from the backend.
+
+        Args:
+            context: Shared context for the action execution
+            preprocessor_id: The pre-processor ID
+
+        Returns:
+            Dict with 'success', 'config', 'version', and optional 'error'
+        """
+        try:
+            client = context.client
+            pre_processor_response = client.get_plugin_release(preprocessor_id)
+
+            if isinstance(pre_processor_response, str):
+                return {'success': False, 'error': 'Invalid pre-processor response received'}
+
+            if not isinstance(pre_processor_response, dict):
+                return {'success': False, 'error': 'Unexpected pre-processor response format'}
+
+            config = pre_processor_response.get('config', {})
+            version = pre_processor_response.get('version')
+
+            return {'success': True, 'config': config, 'version': version}
+
+        except Exception as e:
+            return {'success': False, 'error': f'Failed to get pre-processor config: {str(e)}'}
+
+    def get_preprocessor_info(self, context: ToTaskContext, preprocessor_id: int) -> Dict[str, Any]:
+        """Get pre-processor information from the backend.
+
+        Args:
+            context: Shared context for the action execution
+            preprocessor_id: The pre-processor ID
+
+        Returns:
+            Dict with pre-processor info or error
+        """
+        config_result = self._get_preprocessor_config(context, preprocessor_id)
+        if not config_result['success']:
+            return config_result
+
+        config = config_result['config']
+        code = config.get('code')
+        version = config_result['version']
+
+        if not code or not version:
+            return {'success': False, 'error': 'Invalid pre-processor configuration'}
+
+        return {'success': True, 'code': code, 'version': version}
+
+    def _get_running_serve_apps(self, context: ToTaskContext, preprocessor_code: str) -> Dict[str, Any]:
+        """Get list of running serve applications for a preprocessor.
+
+        Args:
+            context: Shared context for the action execution
+            preprocessor_code: The pre-processor code
+
+        Returns:
+            Dict with 'success', 'running_apps' (list), and optional 'error'
+        """
+        try:
+            client = context.client
+            list_serve_applications_params = {
+                'plugin_code': preprocessor_code,
+                'job__agent': context.params.get('agent') if context.params else None,
+            }
+
+            serve_applications_response = client.list_serve_applications(params=list_serve_applications_params)
+            if isinstance(serve_applications_response, str):
+                return {'success': False, 'running_apps': [], 'error': 'Invalid serve applications response'}
+
+            if not isinstance(serve_applications_response, dict):
+                return {'success': False, 'running_apps': [], 'error': 'Unexpected serve applications response format'}
+
+            results = serve_applications_response.get('results', [])
+            running_apps: List[Dict[str, Any]] = [
+                app for app in results if isinstance(app, dict) and app.get('status') == 'RUNNING'
+            ]
+
+            return {'success': True, 'running_apps': running_apps}
+
+        except Exception as e:
+            return {'success': False, 'running_apps': [], 'error': f'Failed to get running serve apps: {str(e)}'}
+
+    def ensure_preprocessor_running(self, context: ToTaskContext, preprocessor_code: str) -> Dict[str, Any]:
+        """Ensure the pre-processor is running, restart if necessary.
+
+        Args:
+            context: Shared context for the action execution
+            preprocessor_code: The pre-processor code
+
+        Returns:
+            Dict indicating success or failure
+        """
+        try:
+            # Check if pre-processor is already running
+            result = self._get_running_serve_apps(context, preprocessor_code)
+            if not result['success']:
+                return {'success': False, 'error': result.get('error', 'Failed to check running apps')}
+
+            if result['running_apps']:
+                return {'success': True}
+
+            # If not running, restart the pre-processor
+            restart_result = self._restart_preprocessor(context, preprocessor_code)
+            if not restart_result['success']:
+                return restart_result
+
+            return {'success': True}
+
+        except Exception as e:
+            return {'success': False, 'error': f'Failed to ensure pre-processor running: {str(e)}'}
+
+    def _restart_preprocessor(self, context: ToTaskContext, preprocessor_code: str) -> Dict[str, Any]:
+        """Restart the pre-processor and wait for it to be running.
+
+        Starts the pre-processor deployment and polls for up to 3 minutes
+        to verify it is running, logging progress messages during the wait.
+
+        Args:
+            context: Shared context for the action execution
+            preprocessor_code: The pre-processor code
+
+        Returns:
+            Dict indicating success or failure
+        """
+        MAX_WAIT_SECONDS = 180  # 3 minutes
+        POLL_INTERVAL_SECONDS = 10
+
+        try:
+            # Start deployment
+            start_result = self._start_preprocessor_deployment(context, preprocessor_code)
+            if not start_result['success']:
+                return start_result
+
+            context.logger.log_message('Pre-processor deployment started, waiting for it to be ready...')
+
+            # Poll for running status with logging
+            return self._wait_for_preprocessor_ready(
+                context, preprocessor_code, MAX_WAIT_SECONDS, POLL_INTERVAL_SECONDS
+            )
+
+        except Exception as e:
+            return {'success': False, 'error': f'Failed to restart pre-processor: {str(e)}'}
+
+    def _start_preprocessor_deployment(self, context: ToTaskContext, preprocessor_code: str) -> Dict[str, Any]:
+        """Start the pre-processor deployment.
+
+        Args:
+            context: Shared context for the action execution
+            preprocessor_code: The pre-processor code
+
+        Returns:
+            Dict indicating success or failure with optional job_id
+        """
+        try:
+            # Retrieve Pre-Processor Configuration
+            pre_processor_id = context.params.get('pre_processor')
+            if not pre_processor_id:
+                return {'success': False, 'error': 'No pre-processor ID provided'}
+
+            config_result = self._get_preprocessor_config(context, pre_processor_id)
+            if not config_result['success']:
+                return config_result
+
+            config = config_result['config']
+            inference_config = config.get('actions', {}).get('inference', {})
+            required_resources = inference_config.get('required_resources', {})
+
+            # Build deployment payload
+            serve_application_deployment_payload = {
+                'agent': context.params.get('agent') if context.params else None,
+                'action': 'deployment',
+                'params': {
+                    'num_cpus': required_resources.get('required_cpu_count', 1),
+                    'num_gpus': required_resources.get('required_gpu_count', 0.1),
+                },
+                'debug': True,
+            }
+
+            deployment_result = context.client.run_plugin(
+                preprocessor_code,
+                serve_application_deployment_payload,
+            )
+
+            deployment_job_id = deployment_result.get('job_id')
+            if not deployment_job_id:
+                return {'success': False, 'error': 'No deployment job ID returned'}
+
+            return {'success': True, 'job_id': deployment_job_id}
+
+        except Exception as e:
+            return {'success': False, 'error': f'Failed to start deployment: {str(e)}'}
+
+    def _wait_for_preprocessor_ready(
+        self,
+        context: ToTaskContext,
+        preprocessor_code: str,
+        max_wait_seconds: int,
+        poll_interval_seconds: int,
+    ) -> Dict[str, Any]:
+        """Wait for the pre-processor to be running with polling and logging.
+
+        Args:
+            context: Shared context for the action execution
+            preprocessor_code: The pre-processor code
+            max_wait_seconds: Maximum time to wait in seconds
+            poll_interval_seconds: Interval between polling attempts in seconds
+
+        Returns:
+            Dict indicating success or failure
+        """
+        max_attempts = max_wait_seconds // poll_interval_seconds
+
+        for attempt in range(max_attempts):
+            elapsed_seconds = attempt * poll_interval_seconds
+            context.logger.log_message(
+                f'Waiting for pre-processor to start... ({elapsed_seconds}s / {max_wait_seconds}s)'
+            )
+
+            time.sleep(poll_interval_seconds)
+
+            result = self._get_running_serve_apps(context, preprocessor_code)
+            if not result['success']:
+                continue  # Keep trying on transient errors
+
+            if result['running_apps']:
+                context.logger.log_message('Pre-processor started successfully')
+                return {'success': True}
+
+        return {'success': False, 'error': f'Pre-processor failed to start within {max_wait_seconds} seconds'}