synapse-sdk 1.0.0b5__py3-none-any.whl → 2025.12.3__py3-none-any.whl

synapse_sdk/plugins/categories/upload/actions/upload/steps/organize.py

@@ -0,0 +1,201 @@
+from ..context import StepResult, UploadContext
+from ..enums import LogCode
+from .base import BaseStep
+
+
+class OrganizeFilesStep(BaseStep):
+    """Organize files according to specifications using file discovery strategy."""
+
+    @property
+    def name(self) -> str:
+        return 'organize_files'
+
+    @property
+    def progress_weight(self) -> float:
+        return 0.15
+
+    def execute(self, context: UploadContext) -> StepResult:
+        """Execute file organization step."""
+        file_discovery_strategy = context.strategies.get('file_discovery')
+        if not file_discovery_strategy:
+            return self.create_error_result('File discovery strategy not found')
+
+        if not context.file_specifications:
+            return self.create_error_result('File specifications not available')
+
+        try:
+            # Check which mode we're in
+            use_single_path = context.get_param('use_single_path', True)
+
+            if use_single_path:
+                # Single path mode: all assets use same base path
+                return self._execute_single_path_mode(context, file_discovery_strategy)
+            else:
+                # Multi-path mode: each asset has its own path
+                return self._execute_multi_path_mode(context, file_discovery_strategy)
+
+        except Exception as e:
+            return self.create_error_result(f'File organization failed: {str(e)}')
+
+    def _execute_single_path_mode(self, context: UploadContext, file_discovery_strategy) -> StepResult:
+        """Execute file organization in single path mode (traditional)."""
+        # Create type directories mapping
+        type_dirs = {}
+        for spec in context.file_specifications:
+            spec_name = spec['name']
+            spec_dir = context.pathlib_cwd / spec_name
+            if spec_dir.exists() and spec_dir.is_dir():
+                type_dirs[spec_name] = spec_dir
+
+        if type_dirs:
+            context.run.log_message_with_code(LogCode.TYPE_DIRECTORIES_FOUND, list(type_dirs.keys()))
+        else:
+            context.run.log_message_with_code(LogCode.NO_TYPE_DIRECTORIES)
+            return self.create_success_result(data={'organized_files': []})
+
+        context.run.log_message_with_code(LogCode.TYPE_STRUCTURE_DETECTED)
+        context.run.log_message_with_code(LogCode.FILE_ORGANIZATION_STARTED)
+
+        # Discover files in type directories
+        all_files = []
+        is_recursive = context.get_param('is_recursive', True)
+
+        for spec_name, dir_path in type_dirs.items():
+            files_in_dir = file_discovery_strategy.discover(dir_path, is_recursive)
+            all_files.extend(files_in_dir)
+
+        if not all_files:
+            context.run.log_message_with_code(LogCode.NO_FILES_FOUND_WARNING)
+            return self.create_success_result(data={'organized_files': []})
+
+        # Organize files using strategy
+        organized_files = file_discovery_strategy.organize(
+            all_files, context.file_specifications, context.metadata or {}, type_dirs
+        )
+
+        if organized_files:
+            context.run.log_message_with_code(LogCode.FILES_DISCOVERED, len(organized_files))
+            context.add_organized_files(organized_files)
+
+        return self.create_success_result(
+            data={'organized_files': organized_files},
+            rollback_data={'files_count': len(organized_files), 'type_dirs': list(type_dirs.keys())},
+        )
+
+    def _execute_multi_path_mode(self, context: UploadContext, file_discovery_strategy) -> StepResult:
+        """Execute file organization in multi-path mode (each asset has own path)."""
+        from synapse_sdk.utils.storage import get_pathlib
+
+        assets = context.get_param('assets', {})
+        if not assets:
+            return self.create_error_result('Multi-path mode requires assets configuration')
+
+        # Validate that all required specs have asset paths
+        required_specs = [spec['name'] for spec in context.file_specifications if spec.get('is_required', False)]
+        missing_required = [spec for spec in required_specs if spec not in assets]
+
+        if missing_required:
+            return self.create_error_result(
+                f'Multi-path mode requires asset paths for required specs: {", ".join(missing_required)}'
+            )
+
+        context.run.log_message_with_code(LogCode.MULTI_PATH_MODE_ENABLED, len(assets))
+        context.run.log_message_with_code(LogCode.FILE_ORGANIZATION_STARTED)
+
+        # Collect all files and specs first
+        all_files = []
+        type_dirs = {}
+        specs_with_files = []
+
+        for spec in context.file_specifications:
+            spec_name = spec['name']
+            is_required = spec.get('is_required', False)
+
+            # Skip if no asset configuration for this spec (only allowed for optional specs)
+            if spec_name not in assets:
+                if is_required:
+                    # This should not happen due to validation above, but double-check
+                    return self.create_error_result(f'Required spec {spec_name} missing asset path')
+                context.run.log_message_with_code(LogCode.OPTIONAL_SPEC_SKIPPED, spec_name)
+                continue
+
+            asset_config = assets[spec_name]
+
+            # Get the asset path from storage
+            try:
+                asset_path = get_pathlib(context.storage, asset_config.get('path', ''))
+                type_dirs[spec_name] = asset_path
+            except Exception as e:
+                context.run.log_message_with_code(LogCode.ASSET_PATH_ACCESS_ERROR, spec_name, str(e))
+                continue
+
+            if not asset_path.exists():
+                context.run.log_message_with_code(LogCode.ASSET_PATH_NOT_FOUND, spec_name, asset_config.get('path', ''))
+                continue
+
+            # Discover files for this asset
+            is_recursive = asset_config.get('is_recursive', True)
+            context.run.log_message_with_code(LogCode.DISCOVERING_FILES_FOR_ASSET, spec_name, is_recursive)
+
+            files = file_discovery_strategy.discover(asset_path, is_recursive)
+
+            if not files:
+                context.run.log_message_with_code(LogCode.NO_FILES_FOUND_FOR_ASSET, spec_name)
+                continue
+
+            all_files.extend(files)
+            specs_with_files.append(spec)
+            context.run.log_message_with_code(LogCode.FILES_FOUND_FOR_ASSET, len(files), spec_name)
+
+        # Organize all files together to group by dataset_key
+        all_organized_files = []
+        if all_files and specs_with_files:
+            context.run.log_message_with_code(
+                LogCode.ORGANIZING_FILES_MULTI_PATH, len(all_files), len(specs_with_files)
+            )
+            context.run.log_message_with_code(LogCode.TYPE_DIRECTORIES_MULTI_PATH, list(type_dirs.keys()))
+
+            all_organized_files = file_discovery_strategy.organize(
+                all_files, specs_with_files, context.metadata or {}, type_dirs
+            )
+
+        if all_organized_files:
+            context.run.log_message_with_code(LogCode.FILES_DISCOVERED, len(all_organized_files))
+            context.run.log_message_with_code(
+                LogCode.DATA_UNITS_CREATED_FROM_FILES, len(all_organized_files), len(all_files)
+            )
+            context.add_organized_files(all_organized_files)
+        else:
+            context.run.log_message_with_code(LogCode.NO_FILES_FOUND_WARNING)
+
+        return self.create_success_result(
+            data={'organized_files': all_organized_files},
+            rollback_data={'files_count': len(all_organized_files), 'type_dirs': list(type_dirs.keys())},
+        )
+
+    def can_skip(self, context: UploadContext) -> bool:
+        """File organization cannot be skipped."""
+        return False
+
+    def rollback(self, context: UploadContext) -> None:
+        """Rollback file organization."""
+        # Clear organized files
+        context.organized_files.clear()
+        context.run.log_message_with_code(LogCode.ROLLBACK_FILE_ORGANIZATION)
+
+    def validate_prerequisites(self, context: UploadContext) -> None:
+        """Validate prerequisites for file organization."""
+        use_single_path = context.get_param('use_single_path', True)
+
+        # In single-path mode, pathlib_cwd is required
+        if use_single_path and not context.pathlib_cwd:
+            raise ValueError('Working directory path not set in single-path mode')
+
+        # In multi-path mode, pathlib_cwd is optional (each asset has its own path)
+        if not use_single_path:
+            assets = context.get_param('assets', {})
+            if not assets:
+                raise ValueError('Multi-path mode requires assets configuration')
+
+        if not context.file_specifications:
+            raise ValueError('File specifications not available')

synapse_sdk/plugins/categories/upload/actions/upload/steps/upload.py

@@ -0,0 +1,104 @@
+from synapse_sdk.plugins.exceptions import ActionError
+
+from ..context import StepResult, UploadContext
+from ..enums import LogCode, UploadStatus
+from ..strategies.base import UploadConfig
+from .base import BaseStep
+
+
+class UploadFilesStep(BaseStep):
+    """Upload organized files using upload strategy."""
+
+    @property
+    def name(self) -> str:
+        return 'upload_files'
+
+    @property
+    def progress_weight(self) -> float:
+        return 0.30
+
+    def execute(self, context: UploadContext) -> StepResult:
+        """Execute file upload step."""
+        upload_strategy = context.strategies.get('upload')
+        if not upload_strategy:
+            return self.create_error_result('Upload strategy not found')
+
+        if not context.organized_files:
+            context.run.log_message_with_code(LogCode.NO_FILES_UPLOADED)
+            return self.create_error_result('No organized files to upload')
+
+        try:
+            # Setup progress tracking
+            organized_files_count = len(context.organized_files)
+            context.run.set_progress(0, organized_files_count, category='upload_data_files')
+            context.run.log_message_with_code(LogCode.UPLOADING_DATA_FILES)
+
+            # Initialize metrics
+            initial_metrics = {'stand_by': organized_files_count, 'success': 0, 'failed': 0}
+            context.update_metrics('data_files', initial_metrics)
+            context.run.set_metrics(initial_metrics, category='data_files')
+
+            # Create upload configuration
+            # Note: Always uses synchronous upload to guarantee file order
+            upload_config = UploadConfig(
+                chunked_threshold_mb=context.get_param('max_file_size_mb', 50),
+                batch_size=context.get_param('upload_batch_size', 1),
+            )
+
+            # Execute upload using strategy
+            uploaded_files = upload_strategy.upload(context.organized_files, upload_config)
+
+            # Update context and metrics
+            context.add_uploaded_files(uploaded_files)
+
+            # Log upload results
+            for uploaded_file in uploaded_files:
+                context.run.log_data_file(uploaded_file, UploadStatus.SUCCESS)
+
+            # Update final metrics
+            final_metrics = {
+                'stand_by': 0,
+                'success': len(uploaded_files),
+                'failed': organized_files_count - len(uploaded_files),
+            }
+            context.update_metrics('data_files', final_metrics)
+            context.run.set_metrics(final_metrics, category='data_files')
+
+            # Handle success vs failure cases
+            if uploaded_files:
+                # Success: Set completion progress with elapsed time
+                context.run.set_progress(organized_files_count, organized_files_count, category='upload_data_files')
+                return self.create_success_result(
+                    data={'uploaded_files': uploaded_files},
+                    rollback_data={'uploaded_files_count': len(uploaded_files)},
+                )
+            else:
+                # Failure: Mark as failed with elapsed time but no completion
+                context.run.set_progress_failed(category='upload_data_files')
+                return self.create_error_result('No files were successfully uploaded')
+
+        except Exception as e:
+            # Exception: Mark as failed with elapsed time
+            context.run.set_progress_failed(category='upload_data_files')
+            context.run.log_message_with_code(LogCode.FILE_UPLOAD_FAILED, str(e))
+            return self.create_error_result(f'File upload failed: {str(e)}')
+
+    def can_skip(self, context: UploadContext) -> bool:
+        """File upload cannot be skipped."""
+        return False
+
+    def rollback(self, context: UploadContext) -> None:
+        """Rollback file upload."""
+        # In a real implementation, this would delete uploaded files
+        # For now, just clear the uploaded files list and log
+        context.uploaded_files.clear()
+        context.run.log_message_with_code(LogCode.ROLLBACK_FILE_UPLOADS)
+
+    def validate_prerequisites(self, context: UploadContext) -> None:
+        """Validate prerequisites for file upload."""
+        if not context.organized_files:
+            raise ValueError('No organized files available for upload')
+
+        collection_id = context.get_param('data_collection')
+        if collection_id is None:
+            raise ActionError('Data collection parameter is required for upload')

synapse_sdk/plugins/categories/upload/actions/upload/steps/validate.py

@@ -0,0 +1,71 @@
+from ..context import StepResult, UploadContext
+from ..enums import LogCode
+from .base import BaseStep
+
+
+class ValidateFilesStep(BaseStep):
+    """Validate organized files against specifications."""
+
+    @property
+    def name(self) -> str:
+        return 'validate_files'
+
+    @property
+    def progress_weight(self) -> float:
+        return 0.10
+
+    def execute(self, context: UploadContext) -> StepResult:
+        """Execute file validation step.
+
+        Validates organized files against specifications using validation strategy.
+
+        Args:
+            context (UploadContext): Upload workflow context containing organized files,
+                specifications, and strategies.
+
+        Returns:
+            StepResult: Success result with validation status, or error result if validation fails.
+        """
+        validation_strategy = context.strategies.get('validation')
+        if not validation_strategy:
+            return self.create_error_result('Validation strategy not found')
+
+        if not context.organized_files:
+            context.run.log_message_with_code(LogCode.NO_FILES_FOUND)
+            return self.create_error_result('No organized files to validate')
+
+        if not context.file_specifications:
+            return self.create_error_result('File specifications not available')
+
+        try:
+            # Validate organized files against specifications using strategy
+            validation_result = validation_strategy.validate_files(context.organized_files, context.file_specifications)
+
+            if not validation_result.valid:
+                context.run.log_message_with_code(LogCode.VALIDATION_FAILED)
+                error_msg = f'File validation failed: {", ".join(validation_result.errors)}'
+                return self.create_error_result(error_msg)
+
+            return self.create_success_result(
+                data={'validation_passed': True}, rollback_data={'validated_files_count': len(context.organized_files)}
+            )
+
+        except Exception as e:
+            return self.create_error_result(f'File validation failed: {str(e)}')
+
+    def can_skip(self, context: UploadContext) -> bool:
+        """File validation cannot be skipped."""
+        return False
+
+    def rollback(self, context: UploadContext) -> None:
+        """Rollback file validation."""
+        # Nothing specific to rollback for validation
+        context.run.log_message_with_code(LogCode.ROLLBACK_FILE_VALIDATION)
+
+    def validate_prerequisites(self, context: UploadContext) -> None:
+        """Validate prerequisites for file validation."""
+        if not context.organized_files:
+            raise ValueError('No organized files available for validation')
+
+        if not context.file_specifications:
+            raise ValueError('File specifications not available for validation')

synapse_sdk/plugins/categories/upload/actions/upload/strategies/__init__.py

@@ -0,0 +1 @@
+# Strategy pattern implementations for upload actions

synapse_sdk/plugins/categories/upload/actions/upload/strategies/base.py

@@ -0,0 +1,82 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Dict, List
+
+
+class ValidationResult:
+    """Result of validation operations."""
+
+    def __init__(self, valid: bool, errors: List[str] = None):
+        self.valid = valid
+        self.errors = errors or []
+
+    def __bool__(self):
+        return self.valid
+
+
+class ValidationStrategy(ABC):
+    """Strategy interface for validation operations."""
+
+    @abstractmethod
+    def validate_params(self, params: Dict) -> ValidationResult:
+        """Validate action parameters."""
+        pass
+
+    @abstractmethod
+    def validate_files(self, files: List[Dict], specs: Dict) -> ValidationResult:
+        """Validate organized files against specifications."""
+        pass
+
+
+class FileDiscoveryStrategy(ABC):
+    """Strategy interface for file discovery and organization."""
+
+    @abstractmethod
+    def discover(self, path: Path, recursive: bool) -> List[Path]:
+        """Discover files in the given path."""
+        pass
+
+    @abstractmethod
+    def organize(self, files: List[Path], specs: Dict, metadata: Dict, type_dirs: Dict = None) -> List[Dict]:
+        """Organize files according to specifications."""
+        pass
+
+
+class MetadataStrategy(ABC):
+    """Strategy interface for metadata extraction and processing."""
+
+    @abstractmethod
+    def extract(self, source_path: Path) -> Dict[str, Dict[str, Any]]:
+        """Extract metadata from source (e.g., Excel file)."""
+        pass
+
+    @abstractmethod
+    def validate(self, metadata: Dict) -> ValidationResult:
+        """Validate extracted metadata."""
+        pass
+
+
+class UploadConfig:
+    """Configuration for upload operations."""
+
+    def __init__(self, chunked_threshold_mb: int = 50, batch_size: int = 1):
+        self.chunked_threshold_mb = chunked_threshold_mb
+        self.batch_size = batch_size
+
+
+class UploadStrategy(ABC):
+    """Strategy interface for file upload operations."""
+
+    @abstractmethod
+    def upload(self, files: List[Dict], config: UploadConfig) -> List[Dict]:
+        """Upload files to storage."""
+        pass
+
+
+class DataUnitStrategy(ABC):
+    """Strategy interface for data unit generation."""
+
+    @abstractmethod
+    def generate(self, uploaded_files: List[Dict], batch_size: int) -> List[Dict]:
+        """Generate data units from uploaded files."""
+        pass

synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/__init__.py

@@ -0,0 +1 @@
+# Data unit strategy implementations

synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/batch.py

@@ -0,0 +1,39 @@
+from typing import Dict, List
+
+from synapse_sdk.clients.utils import get_batched_list
+
+from ...enums import LogCode, UploadStatus
+from ..base import DataUnitStrategy
+
+
+class BatchDataUnitStrategy(DataUnitStrategy):
+    """Batch data unit generation strategy."""
+
+    def __init__(self, context):
+        self.context = context
+
+    def generate(self, uploaded_files: List[Dict], batch_size: int) -> List[Dict]:
+        """Generate data units in batches."""
+        client = self.context.client
+        generated_data_units = []
+
+        # Use the same batching logic as the legacy implementation
+        batches = get_batched_list(uploaded_files, batch_size)
+
+        for batch in batches:
+            try:
+                created_data_units = client.create_data_units(batch)
+                generated_data_units.extend(created_data_units)
+
+                # Log each created data unit
+                for created_data_unit in created_data_units:
+                    self.context.run.log_data_unit(
+                        created_data_unit['id'], UploadStatus.SUCCESS, data_unit_meta=created_data_unit.get('meta')
+                    )
+            except Exception as e:
+                self.context.run.log_message_with_code(LogCode.DATA_UNIT_BATCH_FAILED, str(e))
+                # Log failed data units
+                for _ in batch:
+                    self.context.run.log_data_unit(None, UploadStatus.FAILED, data_unit_meta=None)
+
+        return generated_data_units

synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/single.py

@@ -0,0 +1,29 @@
+from typing import Dict, List
+
+from ...enums import LogCode, UploadStatus
+from ..base import DataUnitStrategy
+
+
+class SingleDataUnitStrategy(DataUnitStrategy):
+    """Single data unit generation strategy."""
+
+    def __init__(self, context):
+        self.context = context
+
+    def generate(self, uploaded_files: List[Dict], batch_size: int) -> List[Dict]:
+        """Generate data units individually."""
+        client = self.context.client
+        generated_data_units = []
+
+        for uploaded_file in uploaded_files:
+            try:
+                # Create data unit for single file (batch of 1)
+                created_data_units = client.create_data_units([uploaded_file])
+                generated_data_units.extend(created_data_units)
+
+            except Exception as e:
+                self.context.run.log_message_with_code(LogCode.DATA_UNIT_BATCH_FAILED, str(e))
+                # Log failed data unit
+                self.context.run.log_data_unit(None, UploadStatus.FAILED, data_unit_meta=None)
+
+        return generated_data_units

synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/__init__.py

@@ -0,0 +1 @@
+# File discovery strategy implementations