synapse-sdk 2025.9.1__py3-none-any.whl → 2025.9.4__py3-none-any.whl

synapse_sdk/plugins/categories/upload/actions/upload/registry.py

@@ -0,0 +1,113 @@
+from typing import Dict, List, Optional
+
+from .steps.base import BaseStep
+
+
+class StepRegistry:
+    """Registry for managing workflow steps."""
+
+    def __init__(self):
+        self._steps: List[BaseStep] = []
+        self._step_by_name: Dict[str, BaseStep] = {}
+
+    def register(self, step: BaseStep) -> None:
+        """Register a step in the workflow."""
+        if step.name in self._step_by_name:
+            raise ValueError(f"Step with name '{step.name}' already registered")
+
+        self._steps.append(step)
+        self._step_by_name[step.name] = step
+
+    def unregister(self, step_name: str) -> bool:
+        """Unregister a step by name. Returns True if step was found and removed."""
+        if step_name not in self._step_by_name:
+            return False
+
+        step = self._step_by_name[step_name]
+        self._steps.remove(step)
+        del self._step_by_name[step_name]
+        return True
+
+    def get_steps(self) -> List[BaseStep]:
+        """Get all registered steps in order."""
+        return self._steps.copy()
+
+    def get_step(self, name: str) -> Optional[BaseStep]:
+        """Get a step by name."""
+        return self._step_by_name.get(name)
+
+    def has_step(self, name: str) -> bool:
+        """Check if a step is registered."""
+        return name in self._step_by_name
+
+    def clear(self) -> None:
+        """Clear all registered steps."""
+        self._steps.clear()
+        self._step_by_name.clear()
+
+    def get_step_names(self) -> List[str]:
+        """Get list of all registered step names."""
+        return list(self._step_by_name.keys())
+
+    def get_total_progress_weight(self) -> float:
+        """Get total progress weight of all steps."""
+        return sum(step.progress_weight for step in self._steps)
+
+    def insert_step_after(self, after_step_name: str, new_step: BaseStep) -> None:
+        """Insert a step after an existing step."""
+        if after_step_name not in self._step_by_name:
+            raise ValueError(f"Step '{after_step_name}' not found")
+
+        if new_step.name in self._step_by_name:
+            raise ValueError(f"Step with name '{new_step.name}' already registered")
+
+        # Find the index of the step to insert after
+        after_step = self._step_by_name[after_step_name]
+        index = self._steps.index(after_step) + 1
+
+        # Insert the new step
+        self._steps.insert(index, new_step)
+        self._step_by_name[new_step.name] = new_step
+
+    def insert_step_before(self, before_step_name: str, new_step: BaseStep) -> None:
+        """Insert a step before an existing step."""
+        if before_step_name not in self._step_by_name:
+            raise ValueError(f"Step '{before_step_name}' not found")
+
+        if new_step.name in self._step_by_name:
+            raise ValueError(f"Step with name '{new_step.name}' already registered")
+
+        # Find the index of the step to insert before
+        before_step = self._step_by_name[before_step_name]
+        index = self._steps.index(before_step)
+
+        # Insert the new step
+        self._steps.insert(index, new_step)
+        self._step_by_name[new_step.name] = new_step
+
+    def reorder_steps(self, step_names: List[str]) -> None:
+        """Reorder steps according to the provided list of step names."""
+        if set(step_names) != set(self._step_by_name.keys()):
+            raise ValueError('Step names list must contain all registered steps')
+
+        # Reorder the steps list
+        self._steps = [self._step_by_name[name] for name in step_names]
+
+    def __len__(self) -> int:
+        """Return number of registered steps."""
+        return len(self._steps)
+
+    def __iter__(self):
+        """Iterate over registered steps."""
+        return iter(self._steps)
+
+    def __contains__(self, step_name: str) -> bool:
+        """Check if step name is registered."""
+        return step_name in self._step_by_name
+
+    def __str__(self) -> str:
+        step_names = [step.name for step in self._steps]
+        return f'StepRegistry({step_names})'
+
+    def __repr__(self) -> str:
+        return f'StepRegistry(steps={len(self._steps)})'

synapse_sdk/plugins/categories/upload/actions/upload/steps/__init__.py

@@ -0,0 +1 @@
+# Step-based workflow implementations for upload actions

synapse_sdk/plugins/categories/upload/actions/upload/steps/base.py

@@ -0,0 +1,106 @@
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Optional
+
+from ..context import StepResult, UploadContext
+
+
+class BaseStep(ABC):
+    """Abstract base class for all workflow steps."""
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Step name for logging and tracking."""
+        pass
+
+    @property
+    @abstractmethod
+    def progress_weight(self) -> float:
+        """Relative weight for progress calculation (0.0 to 1.0)."""
+        pass
+
+    @abstractmethod
+    def execute(self, context: UploadContext) -> StepResult:
+        """Execute the step logic."""
+        pass
+
+    @abstractmethod
+    def can_skip(self, context: UploadContext) -> bool:
+        """Determine if this step can be skipped based on context."""
+        pass
+
+    @abstractmethod
+    def rollback(self, context: UploadContext) -> None:
+        """Rollback changes made by this step."""
+        pass
+
+    def validate_prerequisites(self, context: UploadContext) -> None:
+        """Validate step prerequisites. Raises exception if not met."""
+        pass
+
+    def log_step_start(self, context: UploadContext) -> None:
+        """Log step start."""
+        context.run.log_message(f'Starting step: {self.name}')
+
+    def log_step_complete(self, context: UploadContext) -> None:
+        """Log step completion."""
+        context.run.log_message(f'Completed step: {self.name}')
+
+    def log_step_skipped(self, context: UploadContext) -> None:
+        """Log step skipped."""
+        context.run.log_message(f'Skipped step: {self.name}')
+
+    def log_step_error(self, context: UploadContext, error: str) -> None:
+        """Log step error."""
+        context.run.log_message(f'Error in step {self.name}: {error}')
+
+    def create_success_result(
+        self,
+        data: Optional[Dict[str, Any]] = None,
+        rollback_data: Optional[Dict[str, Any]] = None,
+        skipped: bool = False,
+    ) -> StepResult:
+        """Create a successful step result."""
+        rollback_data = rollback_data or {}
+        rollback_data['step_name'] = self.name
+        return StepResult(success=True, data=data or {}, rollback_data=rollback_data, skipped=skipped)
+
+    def create_error_result(
+        self, error: str, rollback_data: Optional[Dict[str, Any]] = None, original_exception: Optional[Exception] = None
+    ) -> StepResult:
+        """Create an error step result."""
+        rollback_data = rollback_data or {}
+        rollback_data['step_name'] = self.name
+        return StepResult(
+            success=False, error=error, rollback_data=rollback_data, original_exception=original_exception
+        )
+
+    def safe_execute(self, context: UploadContext) -> StepResult:
+        """Execute step with error handling and logging."""
+        try:
+            self.validate_prerequisites(context)
+
+            if self.can_skip(context):
+                self.log_step_skipped(context)
+                return self.create_success_result(skipped=True)
+
+            self.log_step_start(context)
+            result = self.execute(context)
+
+            if result.success:
+                self.log_step_complete(context)
+            else:
+                self.log_step_error(context, result.error or 'Unknown error')
+
+            return result
+
+        except Exception as e:
+            error_msg = f'Exception in step {self.name}: {str(e)}'
+            self.log_step_error(context, error_msg)
+            return self.create_error_result(error_msg, original_exception=e)
+
+    def __str__(self):
+        return f'{self.__class__.__name__}(name={self.name})'
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}(name='{self.name}', weight={self.progress_weight})"

synapse_sdk/plugins/categories/upload/actions/upload/steps/cleanup.py

@@ -0,0 +1,62 @@
+import os
+import shutil
+from pathlib import Path
+
+from ..context import StepResult, UploadContext
+from ..enums import LogCode
+from .base import BaseStep
+
+
+class CleanupStep(BaseStep):
+    """Cleanup temporary resources and finalize workflow."""
+
+    @property
+    def name(self) -> str:
+        return 'cleanup'
+
+    @property
+    def progress_weight(self) -> float:
+        return 0.05
+
+    def execute(self, context: UploadContext) -> StepResult:
+        """Execute cleanup step."""
+        try:
+            # Cleanup temporary directory
+            self._cleanup_temp_directory(context)
+
+            # Log completion
+            context.run.log_message_with_code(LogCode.IMPORT_COMPLETED)
+
+            return self.create_success_result(data={'cleanup_completed': True}, rollback_data={'temp_cleaned': True})
+
+        except Exception as e:
+            # Cleanup failures shouldn't stop the workflow
+            context.run.log_message(f'Cleanup warning: {str(e)}')
+            return self.create_success_result(
+                data={'cleanup_completed': False}, rollback_data={'cleanup_error': str(e)}
+            )
+
+    def can_skip(self, context: UploadContext) -> bool:
+        """Cleanup step can be skipped if disabled."""
+        return context.get_param('skip_cleanup', False)
+
+    def rollback(self, context: UploadContext) -> None:
+        """Rollback cleanup (nothing to rollback for cleanup)."""
+        context.run.log_message('Cleanup step rollback - no action needed')
+
+    def _cleanup_temp_directory(self, context: UploadContext, temp_path: Path = None) -> None:
+        """Clean up temporary directory."""
+        if temp_path is None:
+            try:
+                temp_path = Path(os.getcwd()) / 'temp'
+            except (FileNotFoundError, OSError):
+                return
+
+        if not temp_path.exists():
+            return
+
+        try:
+            shutil.rmtree(temp_path, ignore_errors=True)
+            context.run.log_message(f'Cleaned up temporary directory: {temp_path}')
+        except Exception as e:
+            context.run.log_message(f'Failed to cleanup temporary directory: {str(e)}')

synapse_sdk/plugins/categories/upload/actions/upload/steps/collection.py

@@ -0,0 +1,62 @@
+from synapse_sdk.plugins.exceptions import ActionError
+
+from ..context import StepResult, UploadContext
+from .base import BaseStep
+
+
+class AnalyzeCollectionStep(BaseStep):
+    """Analyze data collection to get file specifications."""
+
+    @property
+    def name(self) -> str:
+        return 'analyze_collection'
+
+    @property
+    def progress_weight(self) -> float:
+        return 0.05
+
+    def execute(self, context: UploadContext) -> StepResult:
+        """Execute collection analysis step."""
+        collection_id = context.get_param('data_collection')
+        if collection_id is None:
+            return self.create_error_result('Data collection parameter is required')
+
+        try:
+            # Set initial progress
+            context.run.set_progress(0, 2, category='analyze_collection')
+
+            # Get collection from client
+            collection = context.client.get_data_collection(collection_id)
+            context.run.set_progress(1, 2, category='analyze_collection')
+
+            # Extract file specifications
+            file_specifications = collection.get('file_specifications', [])
+            context.set_file_specifications(file_specifications)
+
+            # Complete progress
+            context.run.set_progress(2, 2, category='analyze_collection')
+
+            return self.create_success_result(
+                data={'file_specifications': file_specifications}, rollback_data={'collection_id': collection_id}
+            )
+
+        except Exception as e:
+            return self.create_error_result(f'Failed to analyze collection {collection_id}: {str(e)}')
+
+    def can_skip(self, context: UploadContext) -> bool:
+        """Collection analysis cannot be skipped."""
+        return False
+
+    def rollback(self, context: UploadContext) -> None:
+        """Rollback collection analysis."""
+        # Clear file specifications
+        context.file_specifications.clear()
+        context.run.log_message('Rolled back collection analysis')
+
+    def validate_prerequisites(self, context: UploadContext) -> None:
+        """Validate prerequisites for collection analysis."""
+        if context.client is None:
+            raise ActionError('Client is required for collection analysis')
+
+        if context.get_param('data_collection') is None:
+            raise ActionError('Data collection parameter is required')

synapse_sdk/plugins/categories/upload/actions/upload/steps/generate.py

@@ -0,0 +1,80 @@
+from ..context import StepResult, UploadContext
+from ..enums import LogCode, UploadStatus
+from .base import BaseStep
+
+
+class GenerateDataUnitsStep(BaseStep):
+    """Generate data units from uploaded files."""
+
+    @property
+    def name(self) -> str:
+        return 'generate_data_units'
+
+    @property
+    def progress_weight(self) -> float:
+        return 0.20
+
+    def execute(self, context: UploadContext) -> StepResult:
+        """Execute data unit generation step."""
+        data_unit_strategy = context.strategies.get('data_unit')
+        if not data_unit_strategy:
+            return self.create_error_result('Data unit strategy not found')
+
+        if not context.uploaded_files:
+            context.run.log_message_with_code(LogCode.NO_DATA_UNITS_GENERATED)
+            return self.create_error_result('No uploaded files to generate data units from')
+
+        try:
+            # Setup progress tracking
+            upload_result_count = len(context.uploaded_files)
+            context.run.set_progress(0, upload_result_count, category='generate_data_units')
+            context.run.log_message_with_code(LogCode.GENERATING_DATA_UNITS)
+
+            # Initialize metrics
+            context.update_metrics('data_units', {'stand_by': upload_result_count, 'success': 0, 'failed': 0})
+
+            # Get batch size from parameters
+            batch_size = context.get_param('creating_data_unit_batch_size', 1)
+
+            # Generate data units using strategy
+            generated_data_units = data_unit_strategy.generate(context.uploaded_files, batch_size)
+
+            # Update context
+            context.add_data_units(generated_data_units)
+
+            # Log data unit results
+            for data_unit in generated_data_units:
+                context.run.log_data_unit(
+                    data_unit.get('id'), UploadStatus.SUCCESS, data_unit_meta=data_unit.get('meta')
+                )
+
+            # Update final metrics
+            context.update_metrics('data_units', {'stand_by': 0, 'success': len(generated_data_units), 'failed': 0})
+
+            # Complete progress
+            context.run.set_progress(upload_result_count, upload_result_count, category='generate_data_units')
+
+            return self.create_success_result(
+                data={'generated_data_units': generated_data_units},
+                rollback_data={'data_units_count': len(generated_data_units), 'batch_size': batch_size},
+            )
+
+        except Exception as e:
+            context.run.log_message_with_code(LogCode.DATA_UNIT_BATCH_FAILED, str(e))
+            return self.create_error_result(f'Data unit generation failed: {str(e)}')
+
+    def can_skip(self, context: UploadContext) -> bool:
+        """Data unit generation cannot be skipped."""
+        return False
+
+    def rollback(self, context: UploadContext) -> None:
+        """Rollback data unit generation."""
+        # In a real implementation, this would delete generated data units
+        # For now, just clear the data units list and log
+        context.data_units.clear()
+        context.run.log_message('Rolled back data unit generation')
+
+    def validate_prerequisites(self, context: UploadContext) -> None:
+        """Validate prerequisites for data unit generation."""
+        if not context.uploaded_files:
+            raise ValueError('No uploaded files available for data unit generation')

synapse_sdk/plugins/categories/upload/actions/upload/steps/initialize.py

@@ -0,0 +1,66 @@
+from synapse_sdk.plugins.exceptions import ActionError
+from synapse_sdk.utils.storage import get_pathlib
+
+from ..context import StepResult, UploadContext
+from .base import BaseStep
+
+
+class InitializeStep(BaseStep):
+    """Initialize upload workflow by setting up storage and paths."""
+
+    @property
+    def name(self) -> str:
+        return 'initialize'
+
+    @property
+    def progress_weight(self) -> float:
+        return 0.05
+
+    def execute(self, context: UploadContext) -> StepResult:
+        """Execute initialization step."""
+        # Get and validate storage
+        storage_id = context.get_param('storage')
+        if storage_id is None:
+            return self.create_error_result('Storage parameter is required')
+
+        try:
+            storage = context.client.get_storage(storage_id)
+            context.set_storage(storage)
+        except Exception as e:
+            return self.create_error_result(f'Failed to get storage {storage_id}: {str(e)}')
+
+        # Get and validate path
+        path = context.get_param('path')
+        if path is None:
+            return self.create_error_result('Path parameter is required')
+
+        try:
+            pathlib_cwd = get_pathlib(storage, path)
+            context.set_pathlib_cwd(pathlib_cwd)
+        except Exception as e:
+            return self.create_error_result(f'Failed to get path {path}: {str(e)}')
+
+        # Return success with rollback data
+        rollback_data = {'storage_id': storage_id, 'path': path}
+
+        return self.create_success_result(
+            data={'storage': storage, 'pathlib_cwd': pathlib_cwd}, rollback_data=rollback_data
+        )
+
+    def can_skip(self, context: UploadContext) -> bool:
+        """Initialize step cannot be skipped."""
+        return False
+
+    def rollback(self, context: UploadContext) -> None:
+        """Rollback initialization (cleanup if needed)."""
+        # For initialization, there's typically nothing to rollback
+        # But we could log the rollback action
+        context.run.log_message('Rolling back initialization step')
+
+    def validate_prerequisites(self, context: UploadContext) -> None:
+        """Validate prerequisites for initialization."""
+        if context.client is None:
+            raise ActionError('Client is required for initialization')
+
+        if context.run is None:
+            raise ActionError('Run instance is required for initialization')

synapse_sdk/plugins/categories/upload/actions/upload/steps/metadata.py

@@ -0,0 +1,101 @@
+from pathlib import Path
+
+from ..context import StepResult, UploadContext
+from ..enums import LogCode
+from ..exceptions import ExcelParsingError, ExcelSecurityError
+from .base import BaseStep
+
+
+class ProcessMetadataStep(BaseStep):
+    """Process metadata from Excel files or other sources."""
+
+    @property
+    def name(self) -> str:
+        return 'process_metadata'
+
+    @property
+    def progress_weight(self) -> float:
+        return 0.10
+
+    def execute(self, context: UploadContext) -> StepResult:
+        """Execute metadata processing step."""
+        metadata_strategy = context.strategies.get('metadata')
+        if not metadata_strategy:
+            context.run.log_message('No metadata strategy configured - skipping metadata processing')
+            return self.create_success_result(data={'metadata': {}})
+
+        excel_metadata = {}
+
+        try:
+            # Check if Excel metadata path is specified
+            excel_metadata_path = context.get_param('excel_metadata_path')
+            if excel_metadata_path:
+                # Convert string to Path object
+                if isinstance(excel_metadata_path, str):
+                    excel_metadata_path = Path(excel_metadata_path)
+
+                if excel_metadata_path.exists() and excel_metadata_path.is_file():
+                    excel_path = excel_metadata_path
+                else:
+                    excel_path = context.pathlib_cwd / excel_metadata_path
+                if not excel_path.exists():
+                    context.run.log_message_with_code(LogCode.EXCEL_FILE_NOT_FOUND_PATH)
+                    return self.create_success_result(data={'metadata': {}})
+                excel_metadata = metadata_strategy.extract(excel_path)
+            else:
+                # Look for default metadata files (meta.xlsx, meta.xls)
+                excel_path = self._find_excel_metadata_file(context.pathlib_cwd)
+                if excel_path:
+                    excel_metadata = metadata_strategy.extract(excel_path)
+
+            # Validate extracted metadata
+            if excel_metadata:
+                validation_result = metadata_strategy.validate(excel_metadata)
+                if not validation_result.valid:
+                    error_msg = f'Metadata validation failed: {", ".join(validation_result.errors)}'
+                    return self.create_error_result(error_msg)
+                context.run.log_message_with_code(LogCode.EXCEL_METADATA_LOADED, len(excel_metadata))
+
+            return self.create_success_result(
+                data={'metadata': excel_metadata}, rollback_data={'metadata_processed': len(excel_metadata) > 0}
+            )
+
+        except ExcelSecurityError as e:
+            context.run.log_message_with_code(LogCode.EXCEL_SECURITY_VIOLATION, str(e))
+            return self.create_error_result(f'Excel security violation: {str(e)}')
+
+        except ExcelParsingError as e:
+            # If excel_metadata_path was specified, this is an error
+            # If we were just looking for default files, it's not an error
+            if context.get_param('excel_metadata_path'):
+                context.run.log_message_with_code(LogCode.EXCEL_PARSING_ERROR, str(e))
+                return self.create_error_result(f'Excel parsing error: {str(e)}')
+            else:
+                context.run.log_message_with_code(LogCode.EXCEL_PARSING_ERROR, str(e))
+                return self.create_success_result(data={'metadata': {}})
+
+        except Exception as e:
+            return self.create_error_result(f'Unexpected error processing metadata: {str(e)}')
+
+    def can_skip(self, context: UploadContext) -> bool:
+        """Metadata step can be skipped if no metadata strategy is configured."""
+        return 'metadata' not in context.strategies
+
+    def rollback(self, context: UploadContext) -> None:
+        """Rollback metadata processing."""
+        # Clear any loaded metadata
+        context.metadata.clear()
+
+    def _find_excel_metadata_file(self, pathlib_cwd: Path) -> Path:
+        """Find default Excel metadata file."""
+        # Check .xlsx first as it's more common
+        excel_path = pathlib_cwd / 'meta.xlsx'
+        if excel_path.exists():
+            return excel_path
+
+        # Fallback to .xls
+        excel_path = pathlib_cwd / 'meta.xls'
+        if excel_path.exists():
+            return excel_path
+
+        return None