synapse-sdk 2025.9.5__py3-none-any.whl → 2025.10.6__py3-none-any.whl

synapse_sdk/plugins/categories/base.py

@@ -164,6 +164,7 @@ class Action:
             'params': self.params,
             'envs': self.envs,
             'debug': self.debug,
+            'action_name': self.name,
         }
         return self.run_class(self.job_id, context)
 

synapse_sdk/plugins/categories/export/actions/export/action.py

@@ -95,11 +95,11 @@ class ExportAction(Action):
             PydanticCustomError: If data retrieval fails
         """
         try:
-            result_list = handler.get_results(self.client, filters)
+            result_list = handler.get_results(self.client, filters, run=self.run)
             results = result_list[0]
             count = result_list[1]
         except ClientError:
-            raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset.'))
+            raise PydanticCustomError('client_error', _('Unable to get dataset.'))
         return results, count
 
     def start(self) -> Dict[str, Any]:
@@ -116,7 +116,12 @@ class ExportAction(Action):
         """
         self.run.log_message_with_code(LogCode.EXPORT_STARTED)
 
-        filters = {'expand': 'data', **self.params['filter']}
+        # Get expand setting from config, default to True (expand data)
+        filters = {**self.params['filter']}
+        data_expand = self.config.get('data_expand', True)
+        if data_expand:
+            filters['expand'] = 'data'
+
         target = self.params['target']
         handler = TargetHandlerFactory.get_handler(target)
 

synapse_sdk/plugins/categories/export/actions/export/utils.py

@@ -1,10 +1,12 @@
 from abc import ABC, abstractmethod
-from typing import Any
+from typing import Any, Optional
+import time
 
 from pydantic_core import PydanticCustomError
 
 from synapse_sdk.clients.exceptions import ClientError
 from synapse_sdk.i18n import gettext as _
+from synapse_sdk.shared.enums import Context
 
 
 class ExportTargetHandler(ABC):
@@ -15,6 +17,103 @@ class ExportTargetHandler(ABC):
     of methods to validate filters, retrieve results, and process collections of results.
     """
 
+    # TODO: This is a temporary workaround and needs improvement in the future
+    def _get_results_chunked(self, list_method, filters, chunk_size=100, max_retries=3, retry_delay=1, run=None):
+        """
+        Retrieve results in chunks to avoid memory and response size limits.
+
+        Args:
+            list_method: The client method to call (e.g., client.list_assignments)
+            filters (dict): The filter criteria to apply
+            chunk_size (int): Number of items to fetch per chunk
+            max_retries (int): Maximum number of retries for failed requests
+            retry_delay (int): Delay in seconds between retries
+
+        Returns:
+            tuple: A tuple containing the results generator and the total count
+        """
+        filters = filters.copy()
+        filters['page_size'] = chunk_size
+
+        page = 1
+        results = []
+        total_count = 0
+
+        try:
+            while True:
+                filters['page'] = page
+
+                # Retry logic for handling temporary server issues
+                for attempt in range(max_retries + 1):
+                    try:
+                        response = list_method(params=filters, list_all=False)
+                        break
+                    except ClientError as e:
+                        error_msg = str(e)
+
+                        # Use log_dev_event for better debugging and monitoring
+                        if run:
+                            run.log_dev_event(
+                                'Chunked data retrieval error',
+                                {
+                                    'page': page,
+                                    'attempt': attempt + 1,
+                                    'error_message': error_msg,
+                                    'chunk_size': chunk_size,
+                                },
+                                level=Context.WARNING,
+                            )
+
+                        # Check for JSON decode errors specifically
+                        if 'Expecting value' in error_msg or 'JSONDecodeError' in error_msg:
+                            if run:
+                                run.log_dev_event(
+                                    'JSON parsing error - skipping page',
+                                    {'page': page, 'error_type': 'JSON_DECODE_ERROR', 'error_details': error_msg},
+                                    level=Context.DANGER,
+                                )
+                            # Skip this page and continue with next
+                            page += 1
+                            break
+                        elif attempt < max_retries and ('503' in error_msg or 'connection' in error_msg.lower()):
+                            retry_delay_seconds = retry_delay * (2**attempt)
+                            if run:
+                                run.log_dev_event(
+                                    'Server issue - retrying with backoff',
+                                    {
+                                        'page': page,
+                                        'retry_attempt': attempt + 1,
+                                        'max_retries': max_retries,
+                                        'retry_delay_seconds': retry_delay_seconds,
+                                        'error_type': 'SERVER_ISSUE',
+                                    },
+                                    level=Context.INFO,
+                                )
+                            time.sleep(retry_delay_seconds)  # Exponential backoff
+                            continue
+                        else:
+                            raise
+
+                if page == 1:
+                    total_count = response['count']
+
+                current_results = response.get('results', [])
+                results.extend(current_results)
+
+                # Check if we've got all results or if there are no more results
+                if len(current_results) < chunk_size or not response.get('next'):
+                    break
+
+                page += 1
+
+                # Small delay between pages to avoid overwhelming the server
+                time.sleep(0.1)
+
+            return results, total_count
+        except Exception:
+            # Re-raise the exception to be handled by the calling method
+            raise
+
     @abstractmethod
     def validate_filter(self, value: dict, client: Any):
         """
@@ -33,13 +132,14 @@ class ExportTargetHandler(ABC):
         pass
 
     @abstractmethod
-    def get_results(self, client: Any, filters: dict):
+    def get_results(self, client: Any, filters: dict, run=None):
         """
         Retrieve original data from target sources.
 
         Args:
             client (Any): The client used to retrieve the results.
             filters (dict): The filter criteria to apply.
+            run: Optional ExportRun instance for logging.
 
         Returns:
             tuple: A tuple containing the results and the total count of results.
@@ -76,8 +176,8 @@ class AssignmentExportTargetHandler(ExportTargetHandler):
             raise PydanticCustomError('client_error', _('Unable to get Assignment.'))
         return value
 
-    def get_results(self, client: Any, filters: dict):
-        return client.list_assignments(params=filters, list_all=True)
+    def get_results(self, client: Any, filters: dict, run=None):
+        return self._get_results_chunked(client.list_assignments, filters, run=run)
 
     def get_export_item(self, results):
         for result in results:
@@ -104,9 +204,9 @@ class GroundTruthExportTargetHandler(ExportTargetHandler):
             raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset version.'))
         return value
 
-    def get_results(self, client: Any, filters: dict):
+    def get_results(self, client: Any, filters: dict, run=None):
         filters['ground_truth_dataset_versions'] = filters.pop('ground_truth_dataset_version')
-        return client.list_ground_truth_events(params=filters, list_all=True)
+        return self._get_results_chunked(client.list_ground_truth_events, filters, run=run)
 
     def get_export_item(self, results):
         for result in results:
@@ -134,9 +234,9 @@ class TaskExportTargetHandler(ExportTargetHandler):
             raise PydanticCustomError('client_error', _('Unable to get Task.'))
         return value
 
-    def get_results(self, client: Any, filters: dict):
+    def get_results(self, client: Any, filters: dict, run=None):
         filters['expand'] = ['data_unit', 'assignment', 'workshop']
-        return client.list_tasks(params=filters, list_all=True)
+        return self._get_results_chunked(client.list_tasks, filters, run=run)
 
     def get_export_item(self, results):
         for result in results:

synapse_sdk/plugins/categories/export/templates/config.yaml

@@ -1,3 +1,21 @@
 actions:
   export:
     entrypoint: plugin.export.Exporter
+    # Filter to only export action specific annotation types (e.g., only export image annotations project)
+    # Supported types: image, video, audio, text, pcd, prompt
+    annotation_types:
+      - image
+      - video
+      - audio
+      - text
+      - pcd
+      - prompt
+
+# Filter to only export specific data types (e.g., only export image data)
+# Supported types: image, video, audio, text, pcd
+data_types:
+  - image
+  - video
+  - audio
+  - text
+  - pcd

synapse_sdk/plugins/categories/export/templates/plugin/export.py

@@ -61,3 +61,100 @@ class Exporter(BaseExporter):
     def after_convert(self, data):
         """Post-processes the data after conversion."""
         return data
+
+    def sample_dev_log(self):
+        """Sample development logging examples for plugin developers.
+
+        This method demonstrates various ways to use log_dev_event() for debugging,
+        monitoring, and tracking plugin execution. The event_type is automatically
+        generated as 'export_dev_log' for export actions and cannot be modified.
+
+        Use Cases:
+        1. Process Tracking: Log when important processes start/complete
+        2. Error Handling: Capture detailed error information with appropriate severity
+        3. Performance Monitoring: Record timing and resource usage
+        4. Data Validation: Log validation results and data quality metrics
+        5. Debug Information: Track variable states and execution flow
+
+        Examples show different scenarios where development logging is beneficial:
+        - Basic process logging with structured data
+        - Error logging with exception details and danger level
+        - Performance tracking with timing information
+        - Validation logging with success/failure status
+        """
+        # Example 1: Basic Process Tracking
+        # Use when: Starting important processes that you want to monitor
+        # Benefits: Helps track execution flow and identify bottlenecks
+        self.run.log_dev_event(
+            'Starting data conversion process',
+            {'data_type': 'img', 'data_size': 'unknown', 'conversion_method': 'custom_format'},
+        )
+
+        # Example 2: Error Handling with Detailed Information
+        # Use when: Catching exceptions that you want to analyze later
+        # Benefits: Provides structured error data for debugging and monitoring
+        from synapse_sdk.shared.enums import Context
+
+        try:
+            # Simulated operation that might fail
+            pass
+        except Exception as e:
+            self.run.log_dev_event(
+                f'Data conversion failed: {str(e)}',
+                {
+                    'error_type': type(e).__name__,
+                    'error_details': str(e),
+                    'operation': 'data_conversion',
+                    'recovery_attempted': False,
+                },
+                level=Context.DANGER,
+            )
+
+        # Example 3: Performance Monitoring
+        # Use when: Tracking processing time for optimization
+        # Benefits: Identifies performance bottlenecks and optimization opportunities
+        import time
+
+        start_time = time.time()
+        # Simulated processing work
+        time.sleep(0.001)
+        processing_time = time.time() - start_time
+
+        self.run.log_dev_event(
+            'File processing completed',
+            {
+                'processing_time_ms': round(processing_time * 1000, 2),
+                'files_processed': 1,
+                'performance_rating': 'excellent' if processing_time < 0.1 else 'normal',
+            },
+        )
+
+        # Example 4: Data Validation Logging
+        # Use when: Validating data quality or structure
+        # Benefits: Helps identify data issues and track validation metrics
+        validation_passed = True  # Simulated validation result
+        self.run.log_dev_event(
+            'Data validation completed',
+            {
+                'validation_passed': validation_passed,
+                'validation_rules': ['format_check', 'required_fields', 'data_types'],
+                'data_quality_score': 95.5,
+            },
+            level=Context.SUCCESS if validation_passed else Context.WARNING,
+        )
+
+        # Example 5: Debug Information with Variable States
+        # Use when: Debugging complex logic or tracking variable changes
+        # Benefits: Provides insight into execution state at specific points
+        current_batch_size = 100
+        memory_usage = 45.2  # Simulated memory usage in MB
+
+        self.run.log_dev_event(
+            'Processing checkpoint reached',
+            {
+                'current_batch_size': current_batch_size,
+                'memory_usage_mb': memory_usage,
+                'checkpoint_location': 'after_data_preprocessing',
+                'next_operation': 'file_saving',
+            },
+        )