synapse-sdk 1.0.0a74__py3-none-any.whl → 1.0.0a76__py3-none-any.whl

synapse_sdk/clients/backend/annotation.py

@@ -15,10 +15,6 @@ class AnnotationClientMixin(BaseClient):
         path = f'tasks/{pk}/annotate_task_data/'
         return self._put(path, data=data)
 
-    def patch_task(self, pk, data):
-        path = f'tasks/{pk}/'
-        return self._patch(path, data=data)
-
     def get_task_tag(self, pk):
         path = f'task_tags/{pk}/'
         return self._get(path)

synapse_sdk/devtools/docs/sidebars.ts

@@ -17,6 +17,15 @@ const sidebars: SidebarsConfig = {
     'introduction',
     'installation', 
     'quickstart',
+    {
+      type: 'category',
+      label: 'Features',
+      items: [
+        'features/features',
+        'features/plugins/plugins',
+        'features/converters/converters',
+      ],
+    },
     {
       type: 'category',
       label: 'API Reference',
@@ -28,7 +37,6 @@ const sidebars: SidebarsConfig = {
     'configuration',
     'troubleshooting',
     'faq',
-    'changelog',
     'contributing',
   ],
 };

synapse_sdk/plugins/categories/pre_annotation/actions/to_task.py

@@ -13,6 +13,7 @@ from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
 from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 from synapse_sdk.plugins.models import Run
+from synapse_sdk.shared.enums import Context
 from synapse_sdk.utils.pydantic.validators import non_blank
 
 
@@ -26,6 +27,14 @@ class AnnotateTaskDataStatus(str, Enum):
     FAILED = 'failed'
 
 
+class CriticalError(Exception):
+    """Critical error."""
+
+    def __init__(self, message: str = 'Critical error occured while processing task'):
+        self.message = message
+        super().__init__(self.message)
+
+
 class ToTaskRun(Run):
     class AnnotateTaskDataLog(BaseModel):
         """Log model for annotate task data."""
@@ -278,6 +287,12 @@ class ToTaskAction(Action):
                 self._update_metrics(total_tasks, success_count, failed_count)
                 self.run.set_progress(current_progress, total_tasks, category='annotate_task_data')
 
+            except CriticalError:
+                self.run.log_message(
+                    'Critical error occured while processing task. Stopping the job.', context=Context.DANGER.value
+                )
+                return
+
             except Exception as e:
                 self.run.log_message(f'Failed to process task {task_id}: {str(e)}')
                 self.run.log_annotate_task_data({'task_id': task_id, 'error': str(e)}, AnnotateTaskDataStatus.FAILED)
@@ -427,48 +442,43 @@ class ToTaskAction(Action):
         assert isinstance(self.run, ToTaskRun)
 
         try:
-            # Get pre-processor information
+            # Validate pre-processor ID
             pre_processor_id = self.params.get('pre_processor')
-
             if not pre_processor_id:
                 error_msg = 'Pre-processor ID is required for inference annotation method'
                 self.run.log_message(f'{error_msg} for task {task_id}')
                 self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
                 return {'success': False, 'error': error_msg}
 
-            # Call inference pre processor if specified.
-            pre_processor = client.get_plugin_release(pre_processor_id)
-            pre_processor_code = pre_processor['config']['code']
-            pre_processor_version = pre_processor['version']
-
-            # Extract task data for inference
-            data_unit = task.get('data_unit', {})
-            files = data_unit.get('files', {})
-
-            # Find primary image URL from files
-            primary_file_url = ''
-            for file_info in files.values():
-                if file_info.get('is_primary') and file_info.get('url'):
-                    primary_file_url = file_info['url']
-                    break
+            # Get pre-processor information
+            pre_processor_info = self._get_pre_processor_info(client, pre_processor_id)
+            if not pre_processor_info['success']:
+                return pre_processor_info
+
+            pre_processor_code = pre_processor_info['code']
+            pre_processor_version = pre_processor_info['version']
+
+            # Ensure pre-processor is running
+            pre_processor_status = self._ensure_pre_processor_running(client, pre_processor_code)
+            if not pre_processor_status['success']:
+                return pre_processor_status
+
+            # Extract primary file URL from task data
+            primary_file_url = self._extract_primary_file_url(task)
+            if not primary_file_url:
+                error_msg = 'Primary image URL not found in task data'
+                self.run.log_message(f'{error_msg} for task {task_id}')
+                self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
+                return {'success': False, 'error': error_msg}
 
-            pre_processor_params = self.params.get('pre_processor_params', {})
-            pre_processor_params['image_path'] = primary_file_url
-            inference_payload = {
-                'agent': 1,
-                'action': 'inference',
-                'version': pre_processor_version,
-                'params': {
-                    'model': self.params.get('model'),
-                    'method': 'post',
-                    'json': pre_processor_params,
-                },
-            }
-            inference_data = client.run_plugin(pre_processor_code, inference_payload)
+            # Run inference
+            inference_result = self._run_inference(client, pre_processor_code, pre_processor_version, primary_file_url)
+            if not inference_result['success']:
+                return inference_result
 
-            # Convert data to task object
+            # Convert and submit inference data
             annotation_to_task = self.entrypoint(self.run)
-            converted_result = annotation_to_task.convert_data_from_inference(inference_data)
+            converted_result = annotation_to_task.convert_data_from_inference(inference_result['data'])
 
             # Submit inference annotation data
             client.annotate_task_data(task_id, data={'action': 'submit', 'data': converted_result})
@@ -485,6 +495,172 @@ class ToTaskAction(Action):
             self.run.log_annotate_task_data({'task_id': task_id, 'error': error_msg}, AnnotateTaskDataStatus.FAILED)
             return {'success': False, 'error': error_msg}
 
+    def _get_pre_processor_info(self, client: BackendClient, pre_processor_id: int) -> Dict[str, Any]:
+        """Get pre-processor information from the backend.
+
+        Args:
+            client (BackendClient): The backend client instance.
+            pre_processor_id (int): The pre-processor ID.
+
+        Returns:
+            Dict[str, Any]: Result dictionary with pre-processor info or error.
+        """
+        try:
+            pre_processor_response = client.get_plugin_release(pre_processor_id)
+            if isinstance(pre_processor_response, str):
+                return {'success': False, 'error': 'Invalid pre-processor response received'}
+
+            pre_processor: Dict[str, Any] = pre_processor_response
+            config = pre_processor.get('config', {})
+            code = config.get('code')
+            version = pre_processor.get('version')
+
+            if not code or not version:
+                return {'success': False, 'error': 'Invalid pre-processor configuration'}
+
+            return {'success': True, 'code': code, 'version': version}
+        except Exception as e:
+            return {'success': False, 'error': f'Failed to get pre-processor info: {str(e)}'}
+
+    def _ensure_pre_processor_running(self, client: BackendClient, pre_processor_code: str) -> Dict[str, Any]:
+        """Ensure the pre-processor is running, restart if necessary.
+
+        Args:
+            client (BackendClient): The backend client instance.
+            pre_processor_code (str): The pre-processor code.
+
+        Returns:
+            Dict[str, Any]: Result dictionary indicating success or failure.
+        """
+        try:
+            # Check if pre-processor is running
+            serve_applications_response = client.list_serve_applications(params={'plugin_code': pre_processor_code})
+            if isinstance(serve_applications_response, str):
+                return {'success': False, 'error': 'Invalid serve applications response'}
+
+            # Handle the response properly - it should be a dict with 'results' key
+            if not isinstance(serve_applications_response, dict):
+                return {'success': False, 'error': 'Unexpected serve applications response format'}
+
+            serve_applications: Dict[str, Any] = serve_applications_response
+            results = serve_applications.get('results', [])
+            running_serve_apps = [app for app in results if isinstance(app, dict) and app.get('status') == 'RUNNING']
+
+            # If not running, restart the pre-processor
+            if not running_serve_apps:
+                restart_result = self._restart_pre_processor(client, pre_processor_code)
+                if not restart_result['success']:
+                    return restart_result
+
+                # Verify restart was successful
+                serve_applications_response = client.list_serve_applications(params={'plugin_code': pre_processor_code})
+                if isinstance(serve_applications_response, str):
+                    return {'success': False, 'error': 'Failed to verify pre-processor restart'}
+
+                if not isinstance(serve_applications_response, dict):
+                    return {'success': False, 'error': 'Unexpected serve applications response format after restart'}
+
+                serve_applications: Dict[str, Any] = serve_applications_response
+                results = serve_applications.get('results', [])
+                running_serve_apps = [
+                    app for app in results if isinstance(app, dict) and app.get('status') == 'RUNNING'
+                ]
+
+                if not running_serve_apps:
+                    return {'success': False, 'error': 'Failed to restart pre-processor'}
+
+            return {'success': True}
+        except Exception as e:
+            return {'success': False, 'error': f'Failed to ensure pre-processor running: {str(e)}'}
+
+    def _restart_pre_processor(self, client: BackendClient, pre_processor_code: str) -> Dict[str, Any]:
+        """Restart the pre-processor.
+
+        Args:
+            client (BackendClient): The backend client instance.
+            pre_processor_code (str): The pre-processor code.
+
+        Returns:
+            Dict[str, Any]: Result dictionary indicating success or failure.
+        """
+        try:
+            if not self.config:
+                return {'success': False, 'error': 'Configuration not available'}
+
+            inference_options = self.config.get('inference_options', {})
+            serve_application_deployment_payload = {
+                'agent': self.params.get('agent') if self.params else None,
+                'action': 'deployment',
+                'params': {
+                    'num_cpus': inference_options.get('required_cpu_count', 2),
+                    'num_gpus': inference_options.get('required_gpu_count', 1),
+                },
+                'debug': True,
+            }
+
+            deployment_result = client.run_plugin(pre_processor_code, serve_application_deployment_payload)
+            if not deployment_result:
+                return {'success': False, 'error': 'Failed to restart pre-processor'}
+
+            return {'success': True}
+        except Exception as e:
+            return {'success': False, 'error': f'Failed to restart pre-processor: {str(e)}'}
+
+    def _extract_primary_file_url(self, task: Dict[str, Any]) -> Optional[str]:
+        """Extract the primary file URL from task data.
+
+        Args:
+            task (Dict[str, Any]): The task data.
+
+        Returns:
+            Optional[str]: The primary file URL if found, None otherwise.
+        """
+        data_unit = task.get('data_unit', {})
+        files = data_unit.get('files', {})
+
+        for file_info in files.values():
+            if isinstance(file_info, dict) and file_info.get('is_primary') and file_info.get('url'):
+                return file_info['url']
+
+        return None
+
+    def _run_inference(
+        self, client: BackendClient, pre_processor_code: str, pre_processor_version: str, primary_file_url: str
+    ) -> Dict[str, Any]:
+        """Run inference using the pre-processor.
+
+        Args:
+            client (BackendClient): The backend client instance.
+            pre_processor_code (str): The pre-processor code.
+            pre_processor_version (str): The pre-processor version.
+            primary_file_url (str): The primary image URL.
+
+        Returns:
+            Dict[str, Any]: Result dictionary with inference data or error.
+        """
+        try:
+            if not self.params:
+                return {'success': False, 'error': 'Parameters not available'}
+
+            pre_processor_params = self.params.get('pre_processor_params', {})
+            pre_processor_params['image_path'] = primary_file_url
+
+            inference_payload = {
+                'agent': 1,
+                'action': 'inference',
+                'version': pre_processor_version,
+                'params': {
+                    'model': self.params['model'],
+                    'method': 'post',
+                    'json': pre_processor_params,
+                },
+            }
+
+            inference_data = client.run_plugin(pre_processor_code, inference_payload)
+            return {'success': True, 'data': inference_data}
+        except Exception as e:
+            return {'success': False, 'error': f'Failed to run inference: {str(e)}'}
+
     def _update_metrics(self, total_tasks: int, success_count: int, failed_count: int):
         """Update metrics for task annotation progress.
 

synapse_sdk/plugins/categories/pre_annotation/templates/config.yaml

@@ -3,6 +3,9 @@ actions:
     entrypoint: plugin.pre_annotation.pre_annotate
   to_task:
     entrypoint: plugin.to_task.AnnotationToTask
+    inference_options:
+      required_cpu_count: 1
+      required_gpu_count: 0.1
     ui_schema:
       - $formkit: "radio"
         name: "schema_to_convert"

synapse_sdk/plugins/utils/__init__.py

@@ -0,0 +1,43 @@
+# New utilities
+from .actions import (
+    get_action,
+    get_action_class,
+    get_available_actions,
+    is_action_available,
+)
+from .config import (
+    get_action_config,
+    get_plugin_actions,
+    get_plugin_metadata,
+    read_plugin_config,
+    validate_plugin_config,
+)
+
+# Import legacy functions for backward compatibility
+from .legacy import read_requirements, run_plugin
+from .registry import (
+    get_category_display_name,
+    get_plugin_categories,
+    is_valid_category,
+)
+
+__all__ = [
+    # Config utilities
+    'get_plugin_actions',
+    'get_action_config',
+    'read_plugin_config',
+    'validate_plugin_config',
+    'get_plugin_metadata',
+    # Action utilities
+    'get_action',
+    'get_action_class',
+    'get_available_actions',
+    'is_action_available',
+    # Registry utilities
+    'get_plugin_categories',
+    'is_valid_category',
+    'get_category_display_name',
+    # Legacy utilities for backward compatibility
+    'read_requirements',
+    'run_plugin',
+]

synapse_sdk/plugins/utils/actions.py

@@ -0,0 +1,119 @@
+"""Plugin action utilities."""
+
+import json
+from typing import Any, Dict, Union
+
+from synapse_sdk.plugins.categories.registry import _REGISTERED_ACTIONS, register_actions
+from synapse_sdk.utils.file import get_dict_from_file
+
+from .config import read_plugin_config
+
+
+def get_action(action: str, params_data: Union[str, Dict[str, Any]], *args, **kwargs):
+    """Get a plugin action instance with validated parameters.
+
+    Args:
+        action: Name of the action to get.
+        params_data: Parameters as string (JSON/file path) or dictionary.
+        *args: Additional positional arguments.
+        **kwargs: Additional keyword arguments including 'config'.
+
+    Returns:
+        Configured action instance ready for execution.
+
+    Raises:
+        ActionError: If parameters are invalid or action not found.
+    """
+    if isinstance(params_data, str):
+        try:
+            params = json.loads(params_data)
+        except json.JSONDecodeError:
+            params = get_dict_from_file(params_data)
+    else:
+        params = params_data
+
+    config_data = kwargs.pop('config', False)
+    if config_data:
+        if isinstance(config_data, str):
+            config = read_plugin_config(plugin_path=config_data)
+        else:
+            config = config_data
+    else:
+        config = read_plugin_config()
+
+    category = config['category']
+    return get_action_class(category, action)(params, config, *args, **kwargs)
+
+
+def get_action_class(category: str, action: str):
+    """Get action class by category and action name.
+
+    Args:
+        category: Plugin category (e.g., 'neural_net', 'export').
+        action: Action name (e.g., 'train', 'inference').
+
+    Returns:
+        Action class ready for instantiation.
+
+    Raises:
+        KeyError: If category or action not found in registry.
+    """
+    register_actions()
+    try:
+        return _REGISTERED_ACTIONS[category][action]
+    except KeyError as e:
+        if category not in _REGISTERED_ACTIONS:
+            available_categories = list(_REGISTERED_ACTIONS.keys())
+            raise KeyError(f"Category '{category}' not found. Available categories: {available_categories}") from e
+        else:
+            available_actions = list(_REGISTERED_ACTIONS[category].keys())
+            raise KeyError(
+                f"Action '{action}' not found in category '{category}'. Available actions: {available_actions}"
+            ) from e
+
+
+def get_available_actions(category: str) -> list:
+    """Get list of available actions for a plugin category.
+
+    Args:
+        category: Plugin category to get actions for.
+
+    Returns:
+        List of available action names.
+
+    Raises:
+        KeyError: If category not found in registry.
+
+    Examples:
+        >>> get_available_actions('neural_net')
+        ['train', 'inference', 'test', 'deployment', 'gradio', 'tune']
+    """
+    register_actions()
+    if category not in _REGISTERED_ACTIONS:
+        available_categories = list(_REGISTERED_ACTIONS.keys())
+        raise KeyError(f"Category '{category}' not found. Available categories: {available_categories}")
+
+    return list(_REGISTERED_ACTIONS[category].keys())
+
+
+def is_action_available(category: str, action: str) -> bool:
+    """Check if an action is available in a given category.
+
+    Args:
+        category: Plugin category to check.
+        action: Action name to check.
+
+    Returns:
+        True if action is available, False otherwise.
+
+    Examples:
+        >>> is_action_available('neural_net', 'train')
+        True
+        >>> is_action_available('neural_net', 'nonexistent')
+        False
+    """
+    try:
+        available_actions = get_available_actions(category)
+        return action in available_actions
+    except KeyError:
+        return False