synapse-sdk 1.0.0b5__py3-none-any.whl → 2025.12.3__py3-none-any.whl

synapse_sdk/plugins/categories/export/templates/plugin/export.py

@@ -1,184 +1,160 @@
-import json
 from pathlib import Path
+from typing import Generator
 
-import requests
+from . import BaseExporter
 
-from synapse_sdk.plugins.categories.export.enums import ExportStatus
 
+class Exporter(BaseExporter):
+    """Plugin export action interface for organizing files.
 
-def export(run, export_items, path_root, **params):
-    """Executes the export task.
-
-    Args:
-        run : Execution object
-        export_items (generator):
-            - data (dict): dm_schema_data information.
-            - files (dict): File information. Includes file URL, original file path, metadata, etc.
-            - id (int): ground_truth ID
-        path_root : pathlib object, the path to export
-        **params: Additional parameters
-
-    Returns:
-        dict: Result
+    This class provides a minimal interface for plugin developers to implement
+    their own export logic.
     """
 
-    export_path = path_root / params['name']
-    unique_export_path = export_path
-    counter = 1
-    while unique_export_path.exists():
-        unique_export_path = export_path.with_name(f'{export_path.name}({counter})')
-        counter += 1
-    unique_export_path.mkdir(parents=True)
-
-    run.log_message('Starting export process.')
-
-    # results contains all information fetched through the list API.
-    # example:
-    #   params.get('results', [])
-
-    save_original_file_flag = params.get('save_original_file')
-    errors_json_file_list = []
-    errors_original_file_list = []
-
-    # Path to save JSON files
-    json_output_path = unique_export_path / 'json'
-    json_output_path.mkdir(parents=True, exist_ok=True)
-
-    # Path to save original files
-    if save_original_file_flag:
-        origin_files_output_path = unique_export_path / 'origin_files'
-        origin_files_output_path.mkdir(parents=True, exist_ok=True)
-
-    total = params['count']
-    original_file_metrics_record = run.MetricsRecord(stand_by=total, success=0, failed=0)
-    data_file_metrics_record = run.MetricsRecord(stand_by=total, success=0, failed=0)
-    # progress init
-    run.set_progress(0, total, category='dataset_conversion')
-    for no, export_item in enumerate(export_items, start=1):
-        run.set_progress(no, total, category='dataset_conversion')
-        if no == 1:
-            run.log_message('Converting dataset.')
-        preprocessed_data = before_convert(export_item)
-        converted_data = convert_data(preprocessed_data)
-        final_data = after_convert(converted_data)
-
-        # Call if original file extraction is needed
-        if save_original_file_flag:
-            if no == 1:
-                run.log_message('Saving original file.')
-            original_status = save_original_file(run, final_data, origin_files_output_path, errors_original_file_list)
-
-            original_file_metrics_record.stand_by -= 1
-            if original_status == ExportStatus.FAILED:
-                original_file_metrics_record.failed += 1
-                continue
-            else:
-                original_file_metrics_record.success += 1
-
-        run.log_metrics(record=original_file_metrics_record, category='original_file')
-
-        # Extract data as JSON files
-        if no == 1:
-            run.log_message('Saving json file.')
-        data_status = save_as_json(run, final_data, json_output_path, errors_json_file_list)
-
-        data_file_metrics_record.stand_by -= 1
-        if data_status == ExportStatus.FAILED:
-            data_file_metrics_record.failed += 1
-            continue
-        else:
-            data_file_metrics_record.success += 1
-
-        run.log_metrics(record=data_file_metrics_record, category='data_file')
-
-    run.end_log()
-
-    # Save error list files
-    if len(errors_json_file_list) > 0 or len(errors_original_file_list) > 0:
-        export_error_file = {'json_file_name': errors_json_file_list, 'origin_file_name': errors_original_file_list}
-        with (unique_export_path / 'error_file_list.json').open('w', encoding='utf-8') as f:
-            json.dump(export_error_file, f, indent=4, ensure_ascii=False)
-
-    return {'export_path': str(path_root)}
-
-
-def convert_data(data):
-    """Converts the data."""
-    return data
-
-
-def before_convert(data):
-    """Preprocesses the data before conversion."""
-    return data
-
-
-def after_convert(data):
-    """Post-processes the data after conversion."""
-    return data
-
-
-def get_original_file_name(files):
-    """Retrieve the original file path from the given file information.
-
-    Args:
-        files (dict): A dictionary containing file information, including file URL,
-                      original file path, metadata, etc.
-
-    Returns:
-        file_name (str): The original file name extracted from the file information.
-    """
-    return files['file_name_original']
-
-
-def save_original_file(run, result, base_path, error_file_list):
-    """Saves the original file.
-
-    Args:
-        run : Execution object
-        result (dict): API response data containing file information.
-        base_path (Path): The directory where the file will be saved.
-        error_file_list (list): A list to store error files.
-    """
-    file_url = result['files']['url']
-    file_name = get_original_file_name(result['files'])
-    response = requests.get(file_url)
-    file_info = {'file_name': file_name}
-    error_msg = ''
-    try:
-        with (base_path / file_name).open('wb') as file:
-            file.write(response.content)
-        status = ExportStatus.SUCCESS
-    except Exception as e:
-        error_msg = str(e)
-        error_file_list.append([file_name, error_msg])
-        status = ExportStatus.FAILED
-
-    run.export_log_original_file(result['id'], file_info, status, error_msg)
-    return status
-
-
-def save_as_json(run, result, base_path, error_file_list):
-    """Saves the data as a JSON file.
-
-    Args:
-        run : Execution object
-        result (dict): API response data containing file information.
-        base_path (Path): The directory where the file will be saved.
-        error_file_list (list): A list to store error files.
-    """
-    # Default save file name: original file name
-    file_name = Path(get_original_file_name(result['files'])).stem
-    json_data = result['data']
-    file_info = {'file_name': f'{file_name}.json'}
-    error_msg = ''
-    try:
-        with (base_path / f'{file_name}.json').open('w', encoding='utf-8') as f:
-            json.dump(json_data, f, indent=4, ensure_ascii=False)
-        status = ExportStatus.SUCCESS
-    except Exception as e:
-        error_msg = str(e)
-        error_file_list.append([f'{file_name}.json', str(e)])
-        status = ExportStatus.FAILED
-
-    run.export_log_json_file(result['id'], file_info, status, error_msg)
-    return status
+    def __init__(self, run, export_items: Generator, path_root: Path, **params):
+        """Initialize the plugin export action class.
+        Args:
+            run: Plugin run object with logging capabilities.
+            export_items (generator):
+                - data (dict): dm_schema_data information.
+                - files (dict): File information. Includes file URL, original file path, metadata, etc.
+                - id (int): target ID (ex. assignment id, task id, ground_truth_event id)
+            path_root: pathlib object, the path to export
+            **params: Additional parameters
+                - name (str): The name of the action.
+                - description (str | None): The description of the action.
+                - storage (int): The storage ID to save the exported data.
+                - save_original_file (bool): Whether to save the original file.
+                - path (str): The path to save the exported data.
+                - target (str): The target source to export data from. (ex. ground_truth, assignment, task)
+                - filter (dict): The filter criteria to apply.
+                - extra_params (dict | None): Additional parameters for export customization.
+                    Example: {"include_metadata": True, "compression": "gzip"}
+                - count (int): Total number of results.
+                - results (list): List of results fetched through the list API.
+                - project_id (int): Project ID.
+                - configuration (dict): Project configuration.
+        """
+        super().__init__(run, export_items, path_root, **params)
+
+    def export(self, export_items=None, results=None, **kwargs) -> dict:
+        """Executes the export task using the base class implementation.
+
+        Args:
+            export_items: Optional export items to process. If not provided, uses self.export_items.
+            results: Optional results data to process alongside export_items.
+            **kwargs: Additional parameters for export customization.
+
+        Returns:
+            dict: Result
+        """
+        return super().export(export_items, results, **kwargs)
+
+    def convert_data(self, data):
+        """Converts the data."""
+        return data
+
+    def before_convert(self, data):
+        """Preprocesses the data before conversion."""
+        return data
+
+    def after_convert(self, data):
+        """Post-processes the data after conversion."""
+        return data
+
+    def sample_dev_log(self):
+        """Sample development logging examples for plugin developers.
+
+        This method demonstrates various ways to use log_dev_event() for debugging,
+        monitoring, and tracking plugin execution. The event_type is automatically
+        generated as 'export_dev_log' for export actions and cannot be modified.
+
+        Use Cases:
+        1. Process Tracking: Log when important processes start/complete
+        2. Error Handling: Capture detailed error information with appropriate severity
+        3. Performance Monitoring: Record timing and resource usage
+        4. Data Validation: Log validation results and data quality metrics
+        5. Debug Information: Track variable states and execution flow
+
+        Examples show different scenarios where development logging is beneficial:
+        - Basic process logging with structured data
+        - Error logging with exception details and danger level
+        - Performance tracking with timing information
+        - Validation logging with success/failure status
+        """
+        # Example 1: Basic Process Tracking
+        # Use when: Starting important processes that you want to monitor
+        # Benefits: Helps track execution flow and identify bottlenecks
+        self.run.log_dev_event(
+            'Starting data conversion process',
+            {'data_type': 'img', 'data_size': 'unknown', 'conversion_method': 'custom_format'},
+        )
+
+        # Example 2: Error Handling with Detailed Information
+        # Use when: Catching exceptions that you want to analyze later
+        # Benefits: Provides structured error data for debugging and monitoring
+        from synapse_sdk.shared.enums import Context
+
+        try:
+            # Simulated operation that might fail
+            pass
+        except Exception as e:
+            self.run.log_dev_event(
+                f'Data conversion failed: {str(e)}',
+                {
+                    'error_type': type(e).__name__,
+                    'error_details': str(e),
+                    'operation': 'data_conversion',
+                    'recovery_attempted': False,
+                },
+                level=Context.DANGER,
+            )
+
+        # Example 3: Performance Monitoring
+        # Use when: Tracking processing time for optimization
+        # Benefits: Identifies performance bottlenecks and optimization opportunities
+        import time
+
+        start_time = time.time()
+        # Simulated processing work
+        time.sleep(0.001)
+        processing_time = time.time() - start_time
+
+        self.run.log_dev_event(
+            'File processing completed',
+            {
+                'processing_time_ms': round(processing_time * 1000, 2),
+                'files_processed': 1,
+                'performance_rating': 'excellent' if processing_time < 0.1 else 'normal',
+            },
+        )
+
+        # Example 4: Data Validation Logging
+        # Use when: Validating data quality or structure
+        # Benefits: Helps identify data issues and track validation metrics
+        validation_passed = True  # Simulated validation result
+        self.run.log_dev_event(
+            'Data validation completed',
+            {
+                'validation_passed': validation_passed,
+                'validation_rules': ['format_check', 'required_fields', 'data_types'],
+                'data_quality_score': 95.5,
+            },
+            level=Context.SUCCESS if validation_passed else Context.WARNING,
+        )
+
+        # Example 5: Debug Information with Variable States
+        # Use when: Debugging complex logic or tracking variable changes
+        # Benefits: Provides insight into execution state at specific points
+        current_batch_size = 100
+        memory_usage = 45.2  # Simulated memory usage in MB
+
+        self.run.log_dev_event(
+            'Processing checkpoint reached',
+            {
+                'current_batch_size': current_batch_size,
+                'memory_usage_mb': memory_usage,
+                'checkpoint_location': 'after_data_preprocessing',
+                'next_operation': 'file_saving',
+            },
+        )