synapse-sdk 1.0.0a23__py3-none-any.whl → 2025.12.3__py3-none-any.whl

synapse_sdk/plugins/categories/export/templates/plugin/__init__.py

@@ -0,0 +1,390 @@
+import json
+from pathlib import Path
+from typing import Generator
+
+import requests
+
+from synapse_sdk.plugins.categories.export.actions.export.enums import ExportStatus
+
+
+class BaseExporter:
+    """Base class for export plugins with common functionality.
+
+    This class handles common tasks like progress tracking, logging, and metrics
+    that are shared across all export plugins. Plugin developers should inherit
+    from this class and implement the required methods for their specific logic.
+
+    Core Methods:
+        export(): Main export method - handles the complete export workflow
+        process_data_conversion(): Handle data conversion pipeline
+        process_file_saving(): Handle file saving operations (can be overridden)
+        setup_output_directories(): Setup output directories (can be overridden)
+
+    Required Methods (should be implemented by subclasses):
+        convert_data(): Transform data during export
+
+    Optional Methods (can be overridden by subclasses):
+        save_original_file(): Save original files from export items
+        save_as_json(): Save data as JSON files
+        before_convert(): Pre-process data before conversion
+        after_convert(): Post-process data after conversion
+        process_file_saving(): Custom file saving logic
+
+    Helper Methods:
+        _process_original_file_saving(): Handle original file saving with metrics
+        _process_json_file_saving(): Handle JSON file saving with metrics
+
+    Auto-provided Utilities:
+        Progress tracking via self.run.set_progress()
+        Logging via self.run.log_message() and other run methods
+        Error handling and metrics collection via self.run methods
+    """
+
+    def __init__(self, run, export_items: Generator, path_root: Path, **params):
+        """Initialize the base export class.
+
+        Args:
+            run: Plugin run object with logging capabilities.
+            export_items (generator): Export items generator
+            path_root: pathlib object, the path to export
+            **params: Additional parameters
+        """
+        self.run = run
+        self.export_items = export_items
+        self.path_root = path_root
+        self.params = params
+
+    def _create_unique_export_path(self, base_name: str) -> Path:
+        """Create a unique export path to avoid conflicts."""
+        export_path = self.path_root / base_name
+        unique_export_path = export_path
+        counter = 1
+        while unique_export_path.exists():
+            unique_export_path = export_path.with_name(f'{export_path.name}({counter})')
+            counter += 1
+        unique_export_path.mkdir(parents=True)
+        return unique_export_path
+
+    def _save_error_list(self, export_path: Path, errors_json_file_list: list, errors_original_file_list: list):
+        """Save error list files if there are any errors."""
+        if len(errors_json_file_list) > 0 or len(errors_original_file_list) > 0:
+            export_error_file = {'json_file_name': errors_json_file_list, 'origin_file_name': errors_original_file_list}
+            with (export_path / 'error_file_list.json').open('w', encoding='utf-8') as f:
+                json.dump(export_error_file, f, indent=4, ensure_ascii=False)
+
+    def get_original_file_name(self, files):
+        """Retrieve the original file path from the given file information.
+
+        Args:
+            files (dict): A dictionary containing file information
+
+        Returns:
+            file_name (str): The original file name extracted from the file information.
+        """
+        return files['file_name_original']
+
+    def save_original_file(self, result, base_path, error_file_list):
+        """Saves the original file.
+
+        Args:
+            result (dict): API response data containing file information.
+            base_path (Path): The directory where the file will be saved.
+            error_file_list (list): A list to store error files.
+        """
+        file_url = result['files']['url']
+        file_name = self.get_original_file_name(result['files'])
+        response = requests.get(file_url)
+        file_info = {'file_name': file_name}
+        error_msg = ''
+        try:
+            with (base_path / file_name).open('wb') as file:
+                file.write(response.content)
+            status = ExportStatus.SUCCESS
+        except Exception as e:
+            error_msg = str(e)
+            error_file_list.append([file_name, error_msg])
+            status = ExportStatus.FAILED
+
+        self.run.export_log_original_file(result['id'], file_info, status, error_msg)
+        return status
+
+    def save_as_json(self, result, base_path, error_file_list):
+        """Saves the data as a JSON file.
+
+        Args:
+            result (dict): API response data containing file information.
+            base_path (Path): The directory where the file will be saved.
+            error_file_list (list): A list to store error files.
+        """
+        file_name = Path(self.get_original_file_name(result['files'])).stem
+        json_data = result['data']
+        file_info = {'file_name': f'{file_name}.json'}
+
+        if json_data is None:
+            error_msg = 'data is Null'
+            error_file_list.append([f'{file_name}.json', error_msg])
+            status = ExportStatus.FAILED
+            self.run.log_export_event('NULL_DATA_DETECTED', result['id'])
+            self.run.export_log_json_file(result['id'], file_info, status, error_msg)
+            return status
+
+        error_msg = ''
+        try:
+            with (base_path / f'{file_name}.json').open('w', encoding='utf-8') as f:
+                json.dump(json_data, f, indent=4, ensure_ascii=False)
+            status = ExportStatus.SUCCESS
+        except Exception as e:
+            error_msg = str(e)
+            error_file_list.append([f'{file_name}.json', str(e)])
+            status = ExportStatus.FAILED
+
+        self.run.export_log_json_file(result['id'], file_info, status, error_msg)
+        return status
+
+    # Abstract methods that should be implemented by subclasses
+    def convert_data(self, data):
+        """Converts the data. Should be implemented by subclasses."""
+        return data
+
+    def before_convert(self, data):
+        """Preprocesses the data before conversion. Should be implemented by subclasses."""
+        return data
+
+    def after_convert(self, data):
+        """Post-processes the data after conversion. Should be implemented by subclasses."""
+        return data
+
+    def _process_original_file_saving(
+        self, final_data, origin_files_output_path, errors_original_file_list, original_file_metrics_record, no
+    ):
+        """Process original file saving with metrics tracking.
+
+        Args:
+            final_data: Converted data to save
+            origin_files_output_path: Path to save original files
+            errors_original_file_list: List to collect errors
+            original_file_metrics_record: Metrics record for tracking
+            no: Current item number for logging
+
+        Returns:
+            bool: True if processing should continue, False if should skip to next item
+        """
+        if no == 1:
+            self.run.log_message('Saving original file.')
+        original_status = self.save_original_file(final_data, origin_files_output_path, errors_original_file_list)
+
+        original_file_metrics_record.stand_by -= 1
+        if original_status == ExportStatus.FAILED:
+            original_file_metrics_record.failed += 1
+            return False  # Skip to next item
+        else:
+            original_file_metrics_record.success += 1
+            return True  # Continue processing
+
+    def _process_json_file_saving(
+        self, final_data, json_output_path, errors_json_file_list, data_file_metrics_record, no
+    ):
+        """Process JSON file saving with metrics tracking.
+
+        Args:
+            final_data: Converted data to save
+            json_output_path: Path to save JSON files
+            errors_json_file_list: List to collect errors
+            data_file_metrics_record: Metrics record for tracking
+            no: Current item number for logging
+
+        Returns:
+            bool: True if processing should continue, False if should skip to next item
+        """
+        if no == 1:
+            self.run.log_message('Saving json file.')
+        data_status = self.save_as_json(final_data, json_output_path, errors_json_file_list)
+
+        data_file_metrics_record.stand_by -= 1
+        if data_status == ExportStatus.FAILED:
+            data_file_metrics_record.failed += 1
+            return False  # Skip to next item
+        else:
+            data_file_metrics_record.success += 1
+            return True  # Continue processing
+
+    def setup_output_directories(self, unique_export_path, save_original_file_flag):
+        """Setup output directories for export.
+
+        This method can be overridden by subclasses to customize directory structure.
+        The default implementation creates 'json' and 'origin_files' directories.
+
+        Args:
+            unique_export_path: Base path for export
+            save_original_file_flag: Whether original files will be saved
+
+        Returns:
+            dict: Dictionary containing paths for different file types
+                 Example: {'json_output_path': Path, 'origin_files_output_path': Path}
+        """
+        # Path to save JSON files
+        json_output_path = unique_export_path / 'json'
+        json_output_path.mkdir(parents=True, exist_ok=True)
+
+        output_paths = {'json_output_path': json_output_path}
+
+        # Path to save original files
+        if save_original_file_flag:
+            origin_files_output_path = unique_export_path / 'origin_files'
+            origin_files_output_path.mkdir(parents=True, exist_ok=True)
+            output_paths['origin_files_output_path'] = origin_files_output_path
+
+        return output_paths
+
+    def process_data_conversion(self, export_item):
+        """Process data conversion pipeline for a single export item.
+
+        This method handles the complete data conversion process:
+        before_convert -> convert_data -> after_convert
+
+        Args:
+            export_item: Single export item to process
+
+        Returns:
+            Final processed data ready for saving
+        """
+        preprocessed_data = self.before_convert(export_item)
+        converted_data = self.convert_data(preprocessed_data)
+        final_data = self.after_convert(converted_data)
+        return final_data
+
+    def process_file_saving(
+        self,
+        final_data,
+        unique_export_path,
+        save_original_file_flag,
+        errors_json_file_list,
+        errors_original_file_list,
+        original_file_metrics_record,
+        data_file_metrics_record,
+        no,
+    ):
+        """Process file saving operations for a single export item.
+
+        This method can be overridden by subclasses to implement custom file saving logic.
+        The default implementation saves original files and JSON files based on configuration.
+
+        Args:
+            final_data: Converted data ready for saving
+            unique_export_path: Base path for export
+            save_original_file_flag: Whether to save original files
+            errors_json_file_list: List to collect JSON file errors
+            errors_original_file_list: List to collect original file errors
+            original_file_metrics_record: Metrics record for original files
+            data_file_metrics_record: Metrics record for JSON files
+            no: Current item number for logging
+
+        Returns:
+            bool: True if processing should continue, False if should skip to next item
+        """
+        # Get paths from setup (directories already created)
+        json_output_path = unique_export_path / 'json'
+        origin_files_output_path = unique_export_path / 'origin_files' if save_original_file_flag else None
+
+        if save_original_file_flag:
+            should_continue = self._process_original_file_saving(
+                final_data, origin_files_output_path, errors_original_file_list, original_file_metrics_record, no
+            )
+            if not should_continue:
+                return False
+
+        self.run.log_metrics(record=original_file_metrics_record, category='original_file')
+
+        # Extract data as JSON files
+        should_continue = self._process_json_file_saving(
+            final_data, json_output_path, errors_json_file_list, data_file_metrics_record, no
+        )
+        if not should_continue:
+            return False
+
+        self.run.log_metrics(record=data_file_metrics_record, category='data_file')
+
+        return True
+
+    def additional_file_saving(self, unique_export_path):
+        """Save additional files after processing all export items.
+
+        This method is called after the main export loop completes and is intended
+        for saving files that need to be created based on the collective data from
+        all processed export items (e.g., metadata files, configuration files,
+        summary files, etc.).
+
+        Args:
+            unique_export_path (str): The unique export directory path where
+                additional files should be saved.
+        """
+        pass
+
+    def export(self, export_items=None, results=None, **_kwargs) -> dict:
+        """Main export method that can be overridden by subclasses for custom logic.
+
+        This default implementation provides standard file saving functionality.
+        Subclasses can override this method to implement custom export logic
+        while still using the helper methods for specific operations.
+
+
+        Subclasses can override process_file_saving() method to implement custom file saving logic.
+
+        Args:
+            export_items: Optional export items to process. If not provided, uses self.export_items.
+            results: Optional results data to process alongside export_items.
+            **kwargs: Additional parameters for export customization.
+
+        Returns:
+            dict: Export result containing export path and status information.
+        """
+        # Use provided export_items or fall back to instance variable
+        items_to_process = export_items if export_items is not None else self.export_items
+
+        unique_export_path = self._create_unique_export_path(self.params['name'])
+
+        self.run.log_message('Starting export process.')
+
+        save_original_file_flag = self.params.get('save_original_file')
+        errors_json_file_list = []
+        errors_original_file_list = []
+
+        # Setup output directories (can be customized by subclasses)
+        self.setup_output_directories(unique_export_path, save_original_file_flag)
+
+        total = self.params['count']
+
+        original_file_metrics_record = self.run.MetricsRecord(stand_by=total, success=0, failed=0)
+        data_file_metrics_record = self.run.MetricsRecord(stand_by=total, success=0, failed=0)
+
+        # progress init
+        self.run.set_progress(0, total, category='dataset_conversion')
+
+        for no, export_item in enumerate(items_to_process, start=1):
+            self.run.set_progress(min(no, total), total, category='dataset_conversion')
+            if no == 1:
+                self.run.log_message('Converting dataset.')
+
+            final_data = self.process_data_conversion(export_item)
+
+            # Process file saving (can be overridden by subclasses)
+            should_continue = self.process_file_saving(
+                final_data,
+                unique_export_path,
+                save_original_file_flag,
+                errors_json_file_list,
+                errors_original_file_list,
+                original_file_metrics_record,
+                data_file_metrics_record,
+                no,
+            )
+            if not should_continue:
+                continue
+
+        self.additional_file_saving(unique_export_path)
+        self.run.end_log()
+
+        # Save error list files
+        self._save_error_list(unique_export_path, errors_json_file_list, errors_original_file_list)
+
+        return {'export_path': str(self.path_root)}

synapse_sdk/plugins/categories/export/templates/plugin/export.py

@@ -0,0 +1,160 @@
+from pathlib import Path
+from typing import Generator
+
+from . import BaseExporter
+
+
+class Exporter(BaseExporter):
+    """Plugin export action interface for organizing files.
+
+    This class provides a minimal interface for plugin developers to implement
+    their own export logic.
+    """
+
+    def __init__(self, run, export_items: Generator, path_root: Path, **params):
+        """Initialize the plugin export action class.
+        Args:
+            run: Plugin run object with logging capabilities.
+            export_items (generator):
+                - data (dict): dm_schema_data information.
+                - files (dict): File information. Includes file URL, original file path, metadata, etc.
+                - id (int): target ID (ex. assignment id, task id, ground_truth_event id)
+            path_root: pathlib object, the path to export
+            **params: Additional parameters
+                - name (str): The name of the action.
+                - description (str | None): The description of the action.
+                - storage (int): The storage ID to save the exported data.
+                - save_original_file (bool): Whether to save the original file.
+                - path (str): The path to save the exported data.
+                - target (str): The target source to export data from. (ex. ground_truth, assignment, task)
+                - filter (dict): The filter criteria to apply.
+                - extra_params (dict | None): Additional parameters for export customization.
+                    Example: {"include_metadata": True, "compression": "gzip"}
+                - count (int): Total number of results.
+                - results (list): List of results fetched through the list API.
+                - project_id (int): Project ID.
+                - configuration (dict): Project configuration.
+        """
+        super().__init__(run, export_items, path_root, **params)
+
+    def export(self, export_items=None, results=None, **kwargs) -> dict:
+        """Executes the export task using the base class implementation.
+
+        Args:
+            export_items: Optional export items to process. If not provided, uses self.export_items.
+            results: Optional results data to process alongside export_items.
+            **kwargs: Additional parameters for export customization.
+
+        Returns:
+            dict: Result
+        """
+        return super().export(export_items, results, **kwargs)
+
+    def convert_data(self, data):
+        """Converts the data."""
+        return data
+
+    def before_convert(self, data):
+        """Preprocesses the data before conversion."""
+        return data
+
+    def after_convert(self, data):
+        """Post-processes the data after conversion."""
+        return data
+
+    def sample_dev_log(self):
+        """Sample development logging examples for plugin developers.
+
+        This method demonstrates various ways to use log_dev_event() for debugging,
+        monitoring, and tracking plugin execution. The event_type is automatically
+        generated as 'export_dev_log' for export actions and cannot be modified.
+
+        Use Cases:
+        1. Process Tracking: Log when important processes start/complete
+        2. Error Handling: Capture detailed error information with appropriate severity
+        3. Performance Monitoring: Record timing and resource usage
+        4. Data Validation: Log validation results and data quality metrics
+        5. Debug Information: Track variable states and execution flow
+
+        Examples show different scenarios where development logging is beneficial:
+        - Basic process logging with structured data
+        - Error logging with exception details and danger level
+        - Performance tracking with timing information
+        - Validation logging with success/failure status
+        """
+        # Example 1: Basic Process Tracking
+        # Use when: Starting important processes that you want to monitor
+        # Benefits: Helps track execution flow and identify bottlenecks
+        self.run.log_dev_event(
+            'Starting data conversion process',
+            {'data_type': 'img', 'data_size': 'unknown', 'conversion_method': 'custom_format'},
+        )
+
+        # Example 2: Error Handling with Detailed Information
+        # Use when: Catching exceptions that you want to analyze later
+        # Benefits: Provides structured error data for debugging and monitoring
+        from synapse_sdk.shared.enums import Context
+
+        try:
+            # Simulated operation that might fail
+            pass
+        except Exception as e:
+            self.run.log_dev_event(
+                f'Data conversion failed: {str(e)}',
+                {
+                    'error_type': type(e).__name__,
+                    'error_details': str(e),
+                    'operation': 'data_conversion',
+                    'recovery_attempted': False,
+                },
+                level=Context.DANGER,
+            )
+
+        # Example 3: Performance Monitoring
+        # Use when: Tracking processing time for optimization
+        # Benefits: Identifies performance bottlenecks and optimization opportunities
+        import time
+
+        start_time = time.time()
+        # Simulated processing work
+        time.sleep(0.001)
+        processing_time = time.time() - start_time
+
+        self.run.log_dev_event(
+            'File processing completed',
+            {
+                'processing_time_ms': round(processing_time * 1000, 2),
+                'files_processed': 1,
+                'performance_rating': 'excellent' if processing_time < 0.1 else 'normal',
+            },
+        )
+
+        # Example 4: Data Validation Logging
+        # Use when: Validating data quality or structure
+        # Benefits: Helps identify data issues and track validation metrics
+        validation_passed = True  # Simulated validation result
+        self.run.log_dev_event(
+            'Data validation completed',
+            {
+                'validation_passed': validation_passed,
+                'validation_rules': ['format_check', 'required_fields', 'data_types'],
+                'data_quality_score': 95.5,
+            },
+            level=Context.SUCCESS if validation_passed else Context.WARNING,
+        )
+
+        # Example 5: Debug Information with Variable States
+        # Use when: Debugging complex logic or tracking variable changes
+        # Benefits: Provides insight into execution state at specific points
+        current_batch_size = 100
+        memory_usage = 45.2  # Simulated memory usage in MB
+
+        self.run.log_dev_event(
+            'Processing checkpoint reached',
+            {
+                'current_batch_size': current_batch_size,
+                'memory_usage_mb': memory_usage,
+                'checkpoint_location': 'after_data_preprocessing',
+                'next_operation': 'file_saving',
+            },
+        )

synapse_sdk/plugins/categories/neural_net/actions/deployment.py

@@ -1,4 +1,3 @@
-from synapse_sdk.clients.exceptions import ClientError
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
 from synapse_sdk.plugins.enums import PluginCategory, RunMethod
@@ -25,12 +24,17 @@ class DeploymentAction(Action):
 
         self.ray_init()
 
-        deployment = serve.deployment(ray_actor_options=self.get_actor_options())(serve.ingress(app)(self.entrypoint))
+        ray_actor_options = self.get_actor_options()
+
+        deployment = serve.deployment(ray_actor_options=ray_actor_options)(serve.ingress(app)(self.entrypoint)).bind(
+            self.envs['SYNAPSE_PLUGIN_RUN_HOST']
+        )
+
         serve.delete(self.plugin_release.code)
 
         # TODO add run object
         serve.run(
-            deployment.bind(self.envs['SYNAPSE_PLUGIN_RUN_HOST']),
+            deployment,
             name=self.plugin_release.code,
             route_prefix=f'/{self.plugin_release.checksum}',
         )
@@ -41,13 +45,10 @@ class DeploymentAction(Action):
 
     def create_serve_application(self):
         if self.job_id:
-            try:
-                serve_application = self.ray_client.get_serve_application(self.plugin_release.code)
-                return self.client.create_serve_application({
-                    'job': self.job_id,
-                    'status': serve_application['status'],
-                    'data': serve_application,
-                })
-            except ClientError:
-                pass
+            serve_application = self.ray_client.get_serve_application(self.plugin_release.code)
+            return self.client.create_serve_application({
+                'job': self.job_id,
+                'status': serve_application['status'],
+                'data': serve_application,
+            })
         return None