synapse-sdk 1.0.0a23__py3-none-any.whl → 2025.12.3__py3-none-any.whl

synapse_sdk/plugins/categories/upload/actions/upload/steps/base.py

@@ -0,0 +1,107 @@
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Optional
+
+from ..context import StepResult, UploadContext
+from ..enums import LogCode
+
+
+class BaseStep(ABC):
+    """Abstract base class for all workflow steps."""
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Step name for logging and tracking."""
+        pass
+
+    @property
+    @abstractmethod
+    def progress_weight(self) -> float:
+        """Relative weight for progress calculation (0.0 to 1.0)."""
+        pass
+
+    @abstractmethod
+    def execute(self, context: UploadContext) -> StepResult:
+        """Execute the step logic."""
+        pass
+
+    @abstractmethod
+    def can_skip(self, context: UploadContext) -> bool:
+        """Determine if this step can be skipped based on context."""
+        pass
+
+    @abstractmethod
+    def rollback(self, context: UploadContext) -> None:
+        """Rollback changes made by this step."""
+        pass
+
+    def validate_prerequisites(self, context: UploadContext) -> None:
+        """Validate step prerequisites. Raises exception if not met."""
+        pass
+
+    def log_step_start(self, context: UploadContext) -> None:
+        """Log step start."""
+        context.run.log_message_with_code(LogCode.STEP_STARTING, self.name)
+
+    def log_step_complete(self, context: UploadContext) -> None:
+        """Log step completion."""
+        context.run.log_message_with_code(LogCode.STEP_COMPLETED, self.name)
+
+    def log_step_skipped(self, context: UploadContext) -> None:
+        """Log step skipped."""
+        context.run.log_message_with_code(LogCode.STEP_SKIPPED, self.name)
+
+    def log_step_error(self, context: UploadContext, error: str) -> None:
+        """Log step error."""
+        context.run.log_message_with_code(LogCode.STEP_ERROR, self.name, error)
+
+    def create_success_result(
+        self,
+        data: Optional[Dict[str, Any]] = None,
+        rollback_data: Optional[Dict[str, Any]] = None,
+        skipped: bool = False,
+    ) -> StepResult:
+        """Create a successful step result."""
+        rollback_data = rollback_data or {}
+        rollback_data['step_name'] = self.name
+        return StepResult(success=True, data=data or {}, rollback_data=rollback_data, skipped=skipped)
+
+    def create_error_result(
+        self, error: str, rollback_data: Optional[Dict[str, Any]] = None, original_exception: Optional[Exception] = None
+    ) -> StepResult:
+        """Create an error step result."""
+        rollback_data = rollback_data or {}
+        rollback_data['step_name'] = self.name
+        return StepResult(
+            success=False, error=error, rollback_data=rollback_data, original_exception=original_exception
+        )
+
+    def safe_execute(self, context: UploadContext) -> StepResult:
+        """Execute step with error handling and logging."""
+        try:
+            self.validate_prerequisites(context)
+
+            if self.can_skip(context):
+                self.log_step_skipped(context)
+                return self.create_success_result(skipped=True)
+
+            self.log_step_start(context)
+            result = self.execute(context)
+
+            if result.success:
+                self.log_step_complete(context)
+            else:
+                self.log_step_error(context, result.error or 'Unknown error')
+
+            return result
+
+        except Exception as e:
+            error_msg = f'Exception in step {self.name}: {str(e)}'
+            self.log_step_error(context, error_msg)
+            return self.create_error_result(error_msg, original_exception=e)
+
+    def __str__(self):
+        return f'{self.__class__.__name__}(name={self.name})'
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}(name='{self.name}', weight={self.progress_weight})"

synapse_sdk/plugins/categories/upload/actions/upload/steps/cleanup.py

@@ -0,0 +1,62 @@
+import os
+import shutil
+from pathlib import Path
+
+from ..context import StepResult, UploadContext
+from ..enums import LogCode
+from .base import BaseStep
+
+
+class CleanupStep(BaseStep):
+    """Cleanup temporary resources and finalize workflow."""
+
+    @property
+    def name(self) -> str:
+        return 'cleanup'
+
+    @property
+    def progress_weight(self) -> float:
+        return 0.05
+
+    def execute(self, context: UploadContext) -> StepResult:
+        """Execute cleanup step."""
+        try:
+            # Cleanup temporary directory - commented out because duplicated process with ray cleanup process
+            # self._cleanup_temp_directory(context)
+
+            # Log completion
+            context.run.log_message_with_code(LogCode.IMPORT_COMPLETED)
+
+            return self.create_success_result(data={'cleanup_completed': True}, rollback_data={'temp_cleaned': True})
+
+        except Exception as e:
+            # Cleanup failures shouldn't stop the workflow
+            context.run.log_message_with_code(LogCode.CLEANUP_WARNING, str(e))
+            return self.create_success_result(
+                data={'cleanup_completed': False}, rollback_data={'cleanup_error': str(e)}
+            )
+
+    def can_skip(self, context: UploadContext) -> bool:
+        """Cleanup step can be skipped if disabled."""
+        return context.get_param('skip_cleanup', False)
+
+    def rollback(self, context: UploadContext) -> None:
+        """Rollback cleanup (nothing to rollback for cleanup)."""
+        context.run.log_message_with_code(LogCode.ROLLBACK_CLEANUP)
+
+    def _cleanup_temp_directory(self, context: UploadContext, temp_path: Path = None) -> None:
+        """Clean up temporary directory."""
+        if temp_path is None:
+            try:
+                temp_path = Path(os.getcwd()) / 'temp'
+            except (FileNotFoundError, OSError):
+                return
+
+        if not temp_path.exists():
+            return
+
+        try:
+            shutil.rmtree(temp_path, ignore_errors=True)
+            context.run.log_message_with_code(LogCode.CLEANUP_TEMP_DIR_SUCCESS, temp_path)
+        except Exception as e:
+            context.run.log_message_with_code(LogCode.CLEANUP_TEMP_DIR_FAILED, str(e))

synapse_sdk/plugins/categories/upload/actions/upload/steps/collection.py

@@ -0,0 +1,63 @@
+from synapse_sdk.plugins.exceptions import ActionError
+
+from ..context import StepResult, UploadContext
+from ..enums import LogCode
+from .base import BaseStep
+
+
+class AnalyzeCollectionStep(BaseStep):
+    """Analyze data collection to get file specifications."""
+
+    @property
+    def name(self) -> str:
+        return 'analyze_collection'
+
+    @property
+    def progress_weight(self) -> float:
+        return 0.05
+
+    def execute(self, context: UploadContext) -> StepResult:
+        """Execute collection analysis step."""
+        collection_id = context.get_param('data_collection')
+        if collection_id is None:
+            return self.create_error_result('Data collection parameter is required')
+
+        try:
+            # Set initial progress
+            context.run.set_progress(0, 2, category='analyze_collection')
+
+            # Get collection from client
+            collection = context.client.get_data_collection(collection_id)
+            context.run.set_progress(1, 2, category='analyze_collection')
+
+            # Extract file specifications
+            file_specifications = collection.get('file_specifications', [])
+            context.set_file_specifications(file_specifications)
+
+            # Complete progress
+            context.run.set_progress(2, 2, category='analyze_collection')
+
+            return self.create_success_result(
+                data={'file_specifications': file_specifications}, rollback_data={'collection_id': collection_id}
+            )
+
+        except Exception as e:
+            return self.create_error_result(f'Failed to analyze collection {collection_id}: {str(e)}')
+
+    def can_skip(self, context: UploadContext) -> bool:
+        """Collection analysis cannot be skipped."""
+        return False
+
+    def rollback(self, context: UploadContext) -> None:
+        """Rollback collection analysis."""
+        # Clear file specifications
+        context.file_specifications.clear()
+        context.run.log_message_with_code(LogCode.ROLLBACK_COLLECTION_ANALYSIS)
+
+    def validate_prerequisites(self, context: UploadContext) -> None:
+        """Validate prerequisites for collection analysis."""
+        if context.client is None:
+            raise ActionError('Client is required for collection analysis')
+
+        if context.get_param('data_collection') is None:
+            raise ActionError('Data collection parameter is required')

synapse_sdk/plugins/categories/upload/actions/upload/steps/generate.py

@@ -0,0 +1,91 @@
+from ..context import StepResult, UploadContext
+from ..enums import LogCode, UploadStatus
+from .base import BaseStep
+
+
+class GenerateDataUnitsStep(BaseStep):
+    """Generate data units from uploaded files."""
+
+    @property
+    def name(self) -> str:
+        return 'generate_data_units'
+
+    @property
+    def progress_weight(self) -> float:
+        return 0.20
+
+    def execute(self, context: UploadContext) -> StepResult:
+        """Execute data unit generation step."""
+        data_unit_strategy = context.strategies.get('data_unit')
+        if not data_unit_strategy:
+            return self.create_error_result('Data unit strategy not found')
+
+        if not context.uploaded_files:
+            context.run.log_message_with_code(LogCode.NO_DATA_UNITS_GENERATED)
+            return self.create_error_result('No uploaded files to generate data units from')
+
+        try:
+            # Setup progress tracking
+            upload_result_count = len(context.uploaded_files)
+            context.run.set_progress(0, upload_result_count, category='generate_data_units')
+            context.run.log_message_with_code(LogCode.GENERATING_DATA_UNITS)
+
+            # Initialize metrics
+            initial_metrics = {'stand_by': upload_result_count, 'success': 0, 'failed': 0}
+            context.update_metrics('data_units', initial_metrics)
+            context.run.set_metrics(initial_metrics, category='data_units')
+
+            # Get batch size from parameters
+            batch_size = context.get_param('creating_data_unit_batch_size', 1)
+
+            # Generate data units using strategy
+            generated_data_units = data_unit_strategy.generate(context.uploaded_files, batch_size)
+
+            # Update context
+            context.add_data_units(generated_data_units)
+
+            # Log data unit results
+            for data_unit in generated_data_units:
+                context.run.log_data_unit(
+                    data_unit.get('id'), UploadStatus.SUCCESS, data_unit_meta=data_unit.get('meta')
+                )
+
+            # Update final metrics
+            final_metrics = {'stand_by': 0, 'success': len(generated_data_units), 'failed': 0}
+            context.update_metrics('data_units', final_metrics)
+            context.run.set_metrics(final_metrics, category='data_units')
+
+            # Handle success vs failure cases
+            if generated_data_units:
+                # Success: Set completion progress with elapsed time
+                context.run.set_progress(upload_result_count, upload_result_count, category='generate_data_units')
+                return self.create_success_result(
+                    data={'generated_data_units': generated_data_units},
+                    rollback_data={'data_units_count': len(generated_data_units), 'batch_size': batch_size},
+                )
+            else:
+                # Failure: Mark as failed with elapsed time but no completion
+                context.run.set_progress_failed(category='generate_data_units')
+                return self.create_error_result('No data units were successfully generated')
+
+        except Exception as e:
+            # Exception: Mark as failed with elapsed time
+            context.run.set_progress_failed(category='generate_data_units')
+            context.run.log_message_with_code(LogCode.DATA_UNIT_BATCH_FAILED, str(e))
+            return self.create_error_result(f'Data unit generation failed: {str(e)}')
+
+    def can_skip(self, context: UploadContext) -> bool:
+        """Data unit generation cannot be skipped."""
+        return False
+
+    def rollback(self, context: UploadContext) -> None:
+        """Rollback data unit generation."""
+        # In a real implementation, this would delete generated data units
+        # For now, just clear the data units list and log
+        context.data_units.clear()
+        context.run.log_message_with_code(LogCode.ROLLBACK_DATA_UNIT_GENERATION)
+
+    def validate_prerequisites(self, context: UploadContext) -> None:
+        """Validate prerequisites for data unit generation."""
+        if not context.uploaded_files:
+            raise ValueError('No uploaded files available for data unit generation')

synapse_sdk/plugins/categories/upload/actions/upload/steps/initialize.py

@@ -0,0 +1,82 @@
+from synapse_sdk.plugins.exceptions import ActionError
+from synapse_sdk.utils.storage import get_pathlib
+
+from ..context import StepResult, UploadContext
+from ..enums import LogCode
+from .base import BaseStep
+
+
+class InitializeStep(BaseStep):
+    """Initialize upload workflow by setting up storage and paths."""
+
+    @property
+    def name(self) -> str:
+        return 'initialize'
+
+    @property
+    def progress_weight(self) -> float:
+        return 0.05
+
+    def execute(self, context: UploadContext) -> StepResult:
+        """Execute initialization step."""
+        # Get and validate storage
+        storage_id = context.get_param('storage')
+        if storage_id is None:
+            return self.create_error_result('Storage parameter is required')
+
+        try:
+            storage = context.client.get_storage(storage_id)
+            context.set_storage(storage)
+        except Exception as e:
+            return self.create_error_result(f'Failed to get storage {storage_id}: {str(e)}')
+
+        # Check if we're in multi-path mode
+        use_single_path = context.get_param('use_single_path', True)
+
+        # Get and validate path (only required in single-path mode)
+        path = context.get_param('path')
+        pathlib_cwd = None
+
+        if use_single_path:
+            # Single-path mode: global path is required
+            if path is None:
+                return self.create_error_result('Path parameter is required in single-path mode')
+
+            try:
+                pathlib_cwd = get_pathlib(storage, path)
+                context.set_pathlib_cwd(pathlib_cwd)
+            except Exception as e:
+                return self.create_error_result(f'Failed to get path {path}: {str(e)}')
+        else:
+            # Multi-path mode: global path is optional (each asset has its own path)
+            if path:
+                try:
+                    pathlib_cwd = get_pathlib(storage, path)
+                    context.set_pathlib_cwd(pathlib_cwd)
+                except Exception as e:
+                    return self.create_error_result(f'Failed to get path {path}: {str(e)}')
+
+        # Return success with rollback data
+        rollback_data = {'storage_id': storage_id, 'path': path, 'use_single_path': use_single_path}
+
+        return self.create_success_result(
+            data={'storage': storage, 'pathlib_cwd': pathlib_cwd}, rollback_data=rollback_data
+        )
+
+    def can_skip(self, context: UploadContext) -> bool:
+        """Initialize step cannot be skipped."""
+        return False
+
+    def rollback(self, context: UploadContext) -> None:
+        """Rollback initialization (cleanup if needed)."""
+        # For initialization, there's typically nothing to rollback
+        # But we could log the rollback action
+        context.run.log_message_with_code(LogCode.ROLLBACK_INITIALIZATION)
+
+    def validate_prerequisites(self, context: UploadContext) -> None:
+        """Validate prerequisites for initialization."""
+        if context.client is None:
+            raise ActionError('Client is required for initialization')
+
+        if context.run is None:
+            raise ActionError('Run instance is required for initialization')

synapse_sdk/plugins/categories/upload/actions/upload/steps/metadata.py

@@ -0,0 +1,235 @@
+from pathlib import Path
+
+from synapse_sdk.utils.storage import get_pathlib
+
+from ..context import StepResult, UploadContext
+from ..enums import LogCode
+from ..exceptions import ExcelParsingError, ExcelSecurityError
+from .base import BaseStep
+
+
+class ProcessMetadataStep(BaseStep):
+    """Process metadata from Excel files or other sources.
+
+    This step handles Excel metadata file processing, including path resolution,
+    file extraction, and validation. Supports multiple path resolution strategies:
+    absolute paths, storage-relative paths, and working directory-relative paths.
+    """
+
+    @property
+    def name(self) -> str:
+        return 'process_metadata'
+
+    @property
+    def progress_weight(self) -> float:
+        return 0.10
+
+    def execute(self, context: UploadContext) -> StepResult:
+        """Execute metadata processing step.
+
+        Processes Excel metadata files by resolving the file path, extracting
+        metadata, and validating the extracted data. Supports multiple path
+        resolution strategies and handles both explicit metadata paths and
+        default metadata file discovery.
+
+        Args:
+            context: Upload context containing parameters and state information.
+
+        Returns:
+            StepResult containing:
+                - success: True if metadata processing succeeded
+                - data: Dictionary with 'metadata' key containing extracted metadata
+                - rollback_data: Dictionary with 'metadata_processed' flag
+
+        Note:
+            If no metadata strategy is configured, the step completes successfully
+            with empty metadata. If excel_metadata_path is specified but the file
+            is not found, the step logs a warning and continues with empty metadata.
+        """
+        metadata_strategy = context.strategies.get('metadata')
+        if not metadata_strategy:
+            context.run.log_message_with_code(LogCode.NO_METADATA_STRATEGY)
+            return self.create_success_result(data={'metadata': {}})
+
+        excel_metadata = {}
+        temp_file_to_cleanup = None
+
+        try:
+            # Check if Excel metadata path is specified
+            excel_metadata_path_config = context.get_param('excel_metadata_path')
+
+            if excel_metadata_path_config:
+                # Path-based approach (only supported method)
+                excel_path = self._resolve_excel_path_from_string(excel_metadata_path_config, context)
+
+                if not excel_path or not excel_path.exists():
+                    context.run.log_message_with_code(LogCode.EXCEL_FILE_NOT_FOUND_PATH)
+                    return self.create_success_result(data={'metadata': {}})
+
+                excel_metadata = metadata_strategy.extract(excel_path)
+            else:
+                # Look for default metadata files (meta.xlsx, meta.xls)
+                # Only possible in single-path mode where pathlib_cwd is set
+                if context.pathlib_cwd:
+                    excel_path = self._find_excel_metadata_file(context.pathlib_cwd)
+                    if excel_path:
+                        excel_metadata = metadata_strategy.extract(excel_path)
+                else:
+                    context.run.log_message_with_code(LogCode.NO_METADATA_STRATEGY)
+
+            # Validate extracted metadata
+            if excel_metadata:
+                validation_result = metadata_strategy.validate(excel_metadata)
+                if not validation_result.valid:
+                    error_msg = f'Metadata validation failed: {", ".join(validation_result.errors)}'
+                    return self.create_error_result(error_msg)
+                context.run.log_message_with_code(LogCode.EXCEL_METADATA_LOADED, len(excel_metadata))
+
+            return self.create_success_result(
+                data={'metadata': excel_metadata}, rollback_data={'metadata_processed': len(excel_metadata) > 0}
+            )
+
+        except ExcelSecurityError as e:
+            context.run.log_message_with_code(LogCode.EXCEL_SECURITY_VIOLATION, str(e))
+            return self.create_error_result(f'Excel security violation: {str(e)}')
+
+        except ExcelParsingError as e:
+            # If excel_metadata_path was specified, this is an error
+            # If we were just looking for default files, it's not an error
+            if context.get_param('excel_metadata_path'):
+                context.run.log_message_with_code(LogCode.EXCEL_PARSING_ERROR, str(e))
+                return self.create_error_result(f'Excel parsing error: {str(e)}')
+            else:
+                context.run.log_message_with_code(LogCode.EXCEL_PARSING_ERROR, str(e))
+                return self.create_success_result(data={'metadata': {}})
+
+        except Exception as e:
+            return self.create_error_result(f'Unexpected error processing metadata: {str(e)}')
+
+        finally:
+            # Clean up temporary file if it was created from base64
+            if temp_file_to_cleanup and temp_file_to_cleanup.exists():
+                try:
+                    temp_file_to_cleanup.unlink()
+                    context.run.log_message_with_code(LogCode.METADATA_TEMP_FILE_CLEANUP, temp_file_to_cleanup)
+                except Exception as e:
+                    context.run.log_message_with_code(
+                        LogCode.METADATA_TEMP_FILE_CLEANUP_FAILED, temp_file_to_cleanup, str(e)
+                    )
+
+    def can_skip(self, context: UploadContext) -> bool:
+        """Check if metadata step can be skipped.
+
+        Args:
+            context: Upload context containing strategy configuration.
+
+        Returns:
+            True if no metadata strategy is configured, False otherwise.
+        """
+        return 'metadata' not in context.strategies
+
+    def rollback(self, context: UploadContext) -> None:
+        """Rollback metadata processing.
+
+        Clears any loaded metadata from the context to restore the state
+        before this step was executed.
+
+        Args:
+            context: Upload context containing metadata to clear.
+        """
+        # Clear any loaded metadata
+        context.metadata.clear()
+
+    def _resolve_excel_path_from_string(self, excel_path_str: str, context: UploadContext) -> Path | None:
+        """Resolve Excel metadata path from a string path.
+
+        Attempts to resolve the Excel metadata file path using multiple strategies
+        in the following order:
+        1. Absolute filesystem path
+        2. Relative to storage default path (if storage is available)
+        3. Relative to working directory (if pathlib_cwd is set in single-path mode)
+
+        Args:
+            excel_path_str: File path string to the Excel metadata file.
+            context: Upload context containing storage configuration and working
+                directory information for path resolution.
+
+        Returns:
+            Path object pointing to the Excel file if found, None otherwise.
+
+        Examples:
+            Absolute path:
+
+                >>> path = self._resolve_excel_path_from_string(
+                ...     "/data/meta.xlsx", context
+                ... )
+
+            Storage-relative path:
+
+                >>> path = self._resolve_excel_path_from_string(
+                ...     "metadata/meta.xlsx", context
+                ... )
+
+        Note:
+            When resolving storage-relative paths, the method logs the resolved
+            path for debugging purposes. Failed storage path resolution is logged
+            but does not raise an exception.
+        """
+        # Try absolute path first
+        path = Path(excel_path_str)
+        if path.exists() and path.is_file():
+            return path
+
+        # Try relative to storage default path (if storage is available)
+        if context.storage:
+            try:
+                storage_base_path = get_pathlib(context.storage, excel_path_str)
+                if storage_base_path.exists() and storage_base_path.is_file():
+                    context.run.log_message_with_code(LogCode.EXCEL_PATH_RESOLVED_STORAGE, str(storage_base_path))
+                    return storage_base_path
+            except (FileNotFoundError, PermissionError) as e:
+                # Expected errors when path doesn't exist or no permissions
+                context.run.log_message_with_code(LogCode.EXCEL_PATH_RESOLUTION_FAILED, type(e).__name__, str(e))
+            except Exception as e:
+                # Unexpected errors should be logged with more detail for debugging
+                context.run.log_message_with_code(LogCode.EXCEL_PATH_RESOLUTION_ERROR, type(e).__name__, str(e))
+
+        # Try relative to cwd (only if pathlib_cwd is set - single-path mode)
+        if context.pathlib_cwd:
+            path = context.pathlib_cwd / excel_path_str
+            if path.exists() and path.is_file():
+                return path
+
+        # Could not resolve path
+        return None
+
+    def _find_excel_metadata_file(self, pathlib_cwd: Path) -> Path:
+        """Find default Excel metadata file in working directory.
+
+        Searches for standard Excel metadata filenames (meta.xlsx, meta.xls)
+        in the specified working directory. Prioritizes .xlsx format over .xls.
+
+        Args:
+            pathlib_cwd: Working directory path to search in. Must not be None.
+
+        Returns:
+            Path object to the Excel metadata file if found, None otherwise.
+
+        Note:
+            This method only searches the immediate working directory, not
+            subdirectories. The search order is: meta.xlsx, then meta.xls.
+        """
+        if not pathlib_cwd:
+            return None
+
+        # Check .xlsx first as it's more common
+        excel_path = pathlib_cwd / 'meta.xlsx'
+        if excel_path.exists():
+            return excel_path
+
+        # Fallback to .xls
+        excel_path = pathlib_cwd / 'meta.xls'
+        if excel_path.exists():
+            return excel_path
+
+        return None