synapse-sdk 2025.9.1__py3-none-any.whl → 2025.9.3__py3-none-any.whl

synapse_sdk/plugins/categories/upload/actions/upload/steps/organize.py

@@ -0,0 +1,89 @@
+from ..context import StepResult, UploadContext
+from ..enums import LogCode
+from .base import BaseStep
+
+
+class OrganizeFilesStep(BaseStep):
+    """Organize files according to specifications using file discovery strategy."""
+
+    @property
+    def name(self) -> str:
+        return 'organize_files'
+
+    @property
+    def progress_weight(self) -> float:
+        return 0.15
+
+    def execute(self, context: UploadContext) -> StepResult:
+        """Execute file organization step."""
+        file_discovery_strategy = context.strategies.get('file_discovery')
+        if not file_discovery_strategy:
+            return self.create_error_result('File discovery strategy not found')
+
+        if not context.file_specifications:
+            return self.create_error_result('File specifications not available')
+
+        try:
+            # Create type directories mapping
+            type_dirs = {}
+            for spec in context.file_specifications:
+                spec_name = spec['name']
+                spec_dir = context.pathlib_cwd / spec_name
+                if spec_dir.exists() and spec_dir.is_dir():
+                    type_dirs[spec_name] = spec_dir
+
+            if type_dirs:
+                context.run.log_message_with_code(LogCode.TYPE_DIRECTORIES_FOUND, list(type_dirs.keys()))
+            else:
+                context.run.log_message_with_code(LogCode.NO_TYPE_DIRECTORIES)
+                return self.create_success_result(data={'organized_files': []})
+
+            context.run.log_message_with_code(LogCode.TYPE_STRUCTURE_DETECTED)
+            context.run.log_message_with_code(LogCode.FILE_ORGANIZATION_STARTED)
+
+            # Discover files in type directories
+            all_files = []
+            is_recursive = context.get_param('is_recursive', True)
+
+            for spec_name, dir_path in type_dirs.items():
+                files_in_dir = file_discovery_strategy.discover(dir_path, is_recursive)
+                all_files.extend(files_in_dir)
+
+            if not all_files:
+                context.run.log_message_with_code(LogCode.NO_FILES_FOUND_WARNING)
+                return self.create_success_result(data={'organized_files': []})
+
+            # Organize files using strategy
+            organized_files = file_discovery_strategy.organize(
+                all_files, context.file_specifications, context.metadata or {}, type_dirs
+            )
+
+            if organized_files:
+                context.run.log_message_with_code(LogCode.FILES_DISCOVERED, len(organized_files))
+                context.add_organized_files(organized_files)
+
+            return self.create_success_result(
+                data={'organized_files': organized_files},
+                rollback_data={'files_count': len(organized_files), 'type_dirs': list(type_dirs.keys())},
+            )
+
+        except Exception as e:
+            return self.create_error_result(f'File organization failed: {str(e)}')
+
+    def can_skip(self, context: UploadContext) -> bool:
+        """File organization cannot be skipped."""
+        return False
+
+    def rollback(self, context: UploadContext) -> None:
+        """Rollback file organization."""
+        # Clear organized files
+        context.organized_files.clear()
+        context.run.log_message('Rolled back file organization')
+
+    def validate_prerequisites(self, context: UploadContext) -> None:
+        """Validate prerequisites for file organization."""
+        if not context.pathlib_cwd:
+            raise ValueError('Working directory path not set')
+
+        if not context.file_specifications:
+            raise ValueError('File specifications not available')

synapse_sdk/plugins/categories/upload/actions/upload/steps/upload.py

@@ -0,0 +1,96 @@
+from synapse_sdk.plugins.exceptions import ActionError
+
+from ..context import StepResult, UploadContext
+from ..enums import LogCode, UploadStatus
+from ..strategies.base import UploadConfig
+from .base import BaseStep
+
+
+class UploadFilesStep(BaseStep):
+    """Upload organized files using upload strategy."""
+
+    @property
+    def name(self) -> str:
+        return 'upload_files'
+
+    @property
+    def progress_weight(self) -> float:
+        return 0.30
+
+    def execute(self, context: UploadContext) -> StepResult:
+        """Execute file upload step."""
+        upload_strategy = context.strategies.get('upload')
+        if not upload_strategy:
+            return self.create_error_result('Upload strategy not found')
+
+        if not context.organized_files:
+            context.run.log_message_with_code(LogCode.NO_FILES_UPLOADED)
+            return self.create_error_result('No organized files to upload')
+
+        try:
+            # Setup progress tracking
+            organized_files_count = len(context.organized_files)
+            context.run.set_progress(0, organized_files_count, category='upload_data_files')
+            context.run.log_message_with_code(LogCode.UPLOADING_DATA_FILES)
+
+            # Initialize metrics
+            context.update_metrics('data_files', {'stand_by': organized_files_count, 'success': 0, 'failed': 0})
+
+            # Create upload configuration
+            upload_config = UploadConfig(
+                use_async=context.get_param('use_async_upload', True),
+                max_concurrent=context.get_param('max_concurrent_uploads', 10),
+                chunked_threshold_mb=context.get_param('max_file_size_mb', 50),
+                batch_size=context.get_param('upload_batch_size', 1),
+            )
+
+            # Execute upload using strategy
+            uploaded_files = upload_strategy.upload(context.organized_files, upload_config)
+
+            # Update context and metrics
+            context.add_uploaded_files(uploaded_files)
+
+            # Log upload results
+            for uploaded_file in uploaded_files:
+                context.run.log_data_file(uploaded_file, UploadStatus.SUCCESS)
+
+            # Update final metrics
+            context.update_metrics(
+                'data_files',
+                {'stand_by': 0, 'success': len(uploaded_files), 'failed': organized_files_count - len(uploaded_files)},
+            )
+
+            # Complete progress
+            context.run.set_progress(organized_files_count, organized_files_count, category='upload_data_files')
+
+            return self.create_success_result(
+                data={'uploaded_files': uploaded_files},
+                rollback_data={
+                    'uploaded_files_count': len(uploaded_files),
+                    'upload_method': 'async' if upload_config.use_async else 'sync',
+                },
+            )
+
+        except Exception as e:
+            context.run.log_message_with_code(LogCode.FILE_UPLOAD_FAILED, str(e))
+            return self.create_error_result(f'File upload failed: {str(e)}')
+
+    def can_skip(self, context: UploadContext) -> bool:
+        """File upload cannot be skipped."""
+        return False
+
+    def rollback(self, context: UploadContext) -> None:
+        """Rollback file upload."""
+        # In a real implementation, this would delete uploaded files
+        # For now, just clear the uploaded files list and log
+        context.uploaded_files.clear()
+        context.run.log_message('Rolled back file uploads')
+
+    def validate_prerequisites(self, context: UploadContext) -> None:
+        """Validate prerequisites for file upload."""
+        if not context.organized_files:
+            raise ValueError('No organized files available for upload')
+
+        collection_id = context.get_param('data_collection')
+        if collection_id is None:
+            raise ActionError('Data collection parameter is required for upload')

synapse_sdk/plugins/categories/upload/actions/upload/steps/validate.py

@@ -0,0 +1,61 @@
+from ..context import StepResult, UploadContext
+from ..enums import LogCode
+from .base import BaseStep
+
+
+class ValidateFilesStep(BaseStep):
+    """Validate organized files against specifications."""
+
+    @property
+    def name(self) -> str:
+        return 'validate_files'
+
+    @property
+    def progress_weight(self) -> float:
+        return 0.10
+
+    def execute(self, context: UploadContext) -> StepResult:
+        """Execute file validation step."""
+        validation_strategy = context.strategies.get('validation')
+        if not validation_strategy:
+            return self.create_error_result('Validation strategy not found')
+
+        if not context.organized_files:
+            context.run.log_message_with_code(LogCode.NO_FILES_FOUND)
+            return self.create_error_result('No organized files to validate')
+
+        if not context.file_specifications:
+            return self.create_error_result('File specifications not available')
+
+        try:
+            # Validate organized files against specifications
+            validation_result = validation_strategy.validate_files(context.organized_files, context.file_specifications)
+
+            if not validation_result.valid:
+                context.run.log_message_with_code(LogCode.VALIDATION_FAILED)
+                error_msg = f'File validation failed: {", ".join(validation_result.errors)}'
+                return self.create_error_result(error_msg)
+
+            return self.create_success_result(
+                data={'validation_passed': True}, rollback_data={'validated_files_count': len(context.organized_files)}
+            )
+
+        except Exception as e:
+            return self.create_error_result(f'File validation failed: {str(e)}')
+
+    def can_skip(self, context: UploadContext) -> bool:
+        """File validation cannot be skipped."""
+        return False
+
+    def rollback(self, context: UploadContext) -> None:
+        """Rollback file validation."""
+        # Nothing specific to rollback for validation
+        context.run.log_message('Rolled back file validation')
+
+    def validate_prerequisites(self, context: UploadContext) -> None:
+        """Validate prerequisites for file validation."""
+        if not context.organized_files:
+            raise ValueError('No organized files available for validation')
+
+        if not context.file_specifications:
+            raise ValueError('File specifications not available for validation')

synapse_sdk/plugins/categories/upload/actions/upload/strategies/__init__.py

@@ -0,0 +1 @@
+# Strategy pattern implementations for upload actions

synapse_sdk/plugins/categories/upload/actions/upload/strategies/base.py

@@ -0,0 +1,86 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Dict, List
+
+
+class ValidationResult:
+    """Result of validation operations."""
+
+    def __init__(self, valid: bool, errors: List[str] = None):
+        self.valid = valid
+        self.errors = errors or []
+
+    def __bool__(self):
+        return self.valid
+
+
+class ValidationStrategy(ABC):
+    """Strategy interface for validation operations."""
+
+    @abstractmethod
+    def validate_params(self, params: Dict) -> ValidationResult:
+        """Validate action parameters."""
+        pass
+
+    @abstractmethod
+    def validate_files(self, files: List[Dict], specs: Dict) -> ValidationResult:
+        """Validate organized files against specifications."""
+        pass
+
+
+class FileDiscoveryStrategy(ABC):
+    """Strategy interface for file discovery and organization."""
+
+    @abstractmethod
+    def discover(self, path: Path, recursive: bool) -> List[Path]:
+        """Discover files in the given path."""
+        pass
+
+    @abstractmethod
+    def organize(self, files: List[Path], specs: Dict, metadata: Dict, type_dirs: Dict = None) -> List[Dict]:
+        """Organize files according to specifications."""
+        pass
+
+
+class MetadataStrategy(ABC):
+    """Strategy interface for metadata extraction and processing."""
+
+    @abstractmethod
+    def extract(self, source_path: Path) -> Dict[str, Dict[str, Any]]:
+        """Extract metadata from source (e.g., Excel file)."""
+        pass
+
+    @abstractmethod
+    def validate(self, metadata: Dict) -> ValidationResult:
+        """Validate extracted metadata."""
+        pass
+
+
+class UploadConfig:
+    """Configuration for upload operations."""
+
+    def __init__(
+        self, use_async: bool = True, max_concurrent: int = 10, chunked_threshold_mb: int = 50, batch_size: int = 1
+    ):
+        self.use_async = use_async
+        self.max_concurrent = max_concurrent
+        self.chunked_threshold_mb = chunked_threshold_mb
+        self.batch_size = batch_size
+
+
+class UploadStrategy(ABC):
+    """Strategy interface for file upload operations."""
+
+    @abstractmethod
+    def upload(self, files: List[Dict], config: UploadConfig) -> List[Dict]:
+        """Upload files to storage."""
+        pass
+
+
+class DataUnitStrategy(ABC):
+    """Strategy interface for data unit generation."""
+
+    @abstractmethod
+    def generate(self, uploaded_files: List[Dict], batch_size: int) -> List[Dict]:
+        """Generate data units from uploaded files."""
+        pass

synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/__init__.py

@@ -0,0 +1 @@
+# Data unit strategy implementations

synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/batch.py

@@ -0,0 +1,39 @@
+from typing import Dict, List
+
+from synapse_sdk.clients.utils import get_batched_list
+
+from ...enums import LogCode, UploadStatus
+from ..base import DataUnitStrategy
+
+
+class BatchDataUnitStrategy(DataUnitStrategy):
+    """Batch data unit generation strategy."""
+
+    def __init__(self, context):
+        self.context = context
+
+    def generate(self, uploaded_files: List[Dict], batch_size: int) -> List[Dict]:
+        """Generate data units in batches."""
+        client = self.context.client
+        generated_data_units = []
+
+        # Use the same batching logic as the legacy implementation
+        batches = get_batched_list(uploaded_files, batch_size)
+
+        for batch in batches:
+            try:
+                created_data_units = client.create_data_units(batch)
+                generated_data_units.extend(created_data_units)
+
+                # Log each created data unit
+                for created_data_unit in created_data_units:
+                    self.context.run.log_data_unit(
+                        created_data_unit['id'], UploadStatus.SUCCESS, data_unit_meta=created_data_unit.get('meta')
+                    )
+            except Exception as e:
+                self.context.run.log_message_with_code(LogCode.DATA_UNIT_BATCH_FAILED, str(e))
+                # Log failed data units
+                for _ in batch:
+                    self.context.run.log_data_unit(None, UploadStatus.FAILED, data_unit_meta=None)
+
+        return generated_data_units

synapse_sdk/plugins/categories/upload/actions/upload/strategies/data_unit/single.py

@@ -0,0 +1,34 @@
+from typing import Dict, List
+
+from ...enums import LogCode, UploadStatus
+from ..base import DataUnitStrategy
+
+
+class SingleDataUnitStrategy(DataUnitStrategy):
+    """Single data unit generation strategy."""
+
+    def __init__(self, context):
+        self.context = context
+
+    def generate(self, uploaded_files: List[Dict], batch_size: int) -> List[Dict]:
+        """Generate data units individually."""
+        client = self.context.client
+        generated_data_units = []
+
+        for uploaded_file in uploaded_files:
+            try:
+                # Create data unit for single file (batch of 1)
+                created_data_units = client.create_data_units([uploaded_file])
+                generated_data_units.extend(created_data_units)
+
+                # Log each created data unit
+                for created_data_unit in created_data_units:
+                    self.context.run.log_data_unit(
+                        created_data_unit['id'], UploadStatus.SUCCESS, data_unit_meta=created_data_unit.get('meta')
+                    )
+            except Exception as e:
+                self.context.run.log_message_with_code(LogCode.DATA_UNIT_BATCH_FAILED, str(e))
+                # Log failed data unit
+                self.context.run.log_data_unit(None, UploadStatus.FAILED, data_unit_meta=None)
+
+        return generated_data_units

synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/__init__.py

@@ -0,0 +1 @@
+# File discovery strategy implementations

synapse_sdk/plugins/categories/upload/actions/upload/strategies/file_discovery/flat.py

@@ -0,0 +1,233 @@
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List
+
+from ..base import FileDiscoveryStrategy
+
+
+class FlatFileDiscoveryStrategy(FileDiscoveryStrategy):
+    """Non-recursive file discovery strategy."""
+
+    def discover(self, path: Path, recursive: bool) -> List[Path]:
+        """Discover files non-recursively in the given path."""
+        return [file_path for file_path in path.glob('*') if file_path.is_file()]
+
+    def organize(self, files: List[Path], specs: Dict, metadata: Dict, type_dirs: Dict = None) -> List[Dict]:
+        """Organize files according to specifications with metadata."""
+        organized_files = []
+
+        # Use provided type_dirs or create fallback mapping
+        if type_dirs is None:
+            type_dirs = {}
+            for spec in specs:
+                spec_name = spec['name']
+                # Fallback: extract spec directory from file paths
+                for file_path in files:
+                    # Check if this file's path contains the spec_name as a directory
+                    path_parts = file_path.parts
+                    if spec_name in path_parts:
+                        # Find the index of spec_name and reconstruct the path up to that directory
+                        spec_index = path_parts.index(spec_name)
+                        spec_dir = Path(*path_parts[: spec_index + 1])
+                        if spec_dir.exists() and spec_dir.is_dir():
+                            type_dirs[spec_name] = spec_dir
+                            break
+
+        if not type_dirs:
+            return organized_files
+
+        # Performance optimization 2: Build metadata index for faster lookups
+        metadata_index = self._build_metadata_index(metadata)
+
+        # Group files by dataset (flat discovery)
+        dataset_files = {}
+        required_specs = [spec['name'] for spec in specs if spec.get('is_required', False)]
+
+        for file_path in files:
+            # Determine which type directory this file belongs to
+            for spec_name, dir_path in type_dirs.items():
+                if file_path.parent == dir_path:  # Only direct children
+                    file_name = file_path.stem
+
+                    if file_name not in dataset_files:
+                        dataset_files[file_name] = {}
+
+                    if spec_name not in dataset_files[file_name]:
+                        dataset_files[file_name][spec_name] = file_path
+                    else:
+                        # Keep the most recent file - only stat when needed
+                        existing_file = dataset_files[file_name][spec_name]
+                        try:
+                            if file_path.stat().st_mtime > existing_file.stat().st_mtime:
+                                dataset_files[file_name][spec_name] = file_path
+                        except (OSError, IOError):
+                            # If stat fails, keep existing file
+                            pass
+
+        # Create organized files for datasets with all required files
+        for file_name, files_dict in sorted(dataset_files.items()):
+            if all(req in files_dict for req in required_specs):
+                # Calculate most common file extension (optimized)
+                file_extensions = {}
+                for file_path in files_dict.values():
+                    ext = file_path.suffix.lower()
+                    if ext:
+                        file_extensions[ext] = file_extensions.get(ext, 0) + 1
+
+                origin_file_extension = max(file_extensions, key=file_extensions.get) if file_extensions else ''
+
+                meta_data = {
+                    'origin_file_stem': file_name,
+                    'origin_file_extension': origin_file_extension,
+                    'created_at': datetime.now().isoformat(),
+                }
+
+                # Add metadata if available - using optimized index lookup
+                if metadata_index:
+                    matched_metadata = self._find_matching_metadata_optimized(file_name, files_dict, metadata_index)
+                    if matched_metadata:
+                        meta_data.update(matched_metadata)
+
+                organized_files.append({'files': files_dict, 'meta': meta_data})
+
+        return organized_files
+
+    def _build_metadata_index(self, metadata: Dict) -> Dict:
+        """Build metadata index for faster lookups."""
+        if not metadata:
+            return {}
+
+        metadata_index = {'exact_stem': {}, 'exact_name': {}, 'stem_lookup': {}, 'partial_paths': {}, 'full_paths': {}}
+
+        for meta_key, meta_value in metadata.items():
+            meta_path = Path(meta_key)
+
+            # Index by stem
+            stem = meta_path.stem
+            if stem:
+                metadata_index['exact_stem'][stem] = meta_value
+                metadata_index['stem_lookup'][stem] = meta_value
+
+            # Index by full name
+            name = meta_path.name
+            if name:
+                metadata_index['exact_name'][name] = meta_value
+
+            # Index for partial path matching
+            metadata_index['partial_paths'][meta_key] = meta_value
+
+            # Index for full path matching
+            metadata_index['full_paths'][meta_key] = meta_value
+
+        return metadata_index
+
+    def _find_matching_metadata_optimized(self, file_name: str, files_dict: Dict, metadata_index: Dict) -> Dict:
+        """Find matching metadata using optimized index lookups."""
+        if not metadata_index:
+            return {}
+
+        # Strategy 1: Exact stem match (O(1) lookup)
+        if file_name in metadata_index['exact_stem']:
+            return metadata_index['exact_stem'][file_name]
+
+        # Strategy 2: Exact filename match with extension (O(1) lookup)
+        sample_file = list(files_dict.values())[0] if files_dict else None
+        if sample_file:
+            full_filename = f'{file_name}{sample_file.suffix}'
+            if full_filename in metadata_index['exact_name']:
+                return metadata_index['exact_name'][full_filename]
+
+            # Try sample file name
+            sample_filename = sample_file.name
+            if sample_filename in metadata_index['exact_name']:
+                return metadata_index['exact_name'][sample_filename]
+
+        # Strategy 3: Stem lookup (already optimized above)
+        # This is covered by exact_stem lookup
+
+        # Strategy 4 & 5: Partial and full path matching (fallback to original logic for complex cases)
+        if sample_file:
+            file_path_str = str(sample_file)
+            file_path_posix = sample_file.as_posix()
+
+            # Check partial paths
+            for meta_key in metadata_index['partial_paths']:
+                if (
+                    meta_key in file_path_str
+                    or meta_key in file_path_posix
+                    or file_path_str in meta_key
+                    or file_path_posix in meta_key
+                ):
+                    return metadata_index['partial_paths'][meta_key]
+
+        return {}
+
+    def _find_matching_metadata(self, file_name: str, files_dict: Dict, metadata: Dict) -> Dict:
+        """Find matching metadata using comprehensive pattern matching.
+
+        Matching priority:
+        1. Exact stem match (highest priority)
+        2. Exact filename match (with extension)
+        3. Metadata key stem matches file stem
+        4. Partial path matching
+        5. Full path matching
+        """
+        if not metadata:
+            return {}
+
+        # Get sample file for extension and path information
+        sample_file = list(files_dict.values())[0] if files_dict else None
+
+        # Strategy 1: Exact stem match (highest priority)
+        if file_name in metadata:
+            return metadata[file_name]
+
+        # Strategy 2: Exact filename match (with extension)
+        if sample_file:
+            full_filename = f'{file_name}{sample_file.suffix}'
+            if full_filename in metadata:
+                return metadata[full_filename]
+
+            # Also try with sample file name
+            sample_filename = sample_file.name
+            if sample_filename in metadata:
+                return metadata[sample_filename]
+
+        # Strategy 3: Metadata key stem matches file stem
+        for meta_key in metadata.keys():
+            meta_stem = Path(meta_key).stem
+            if meta_stem == file_name:
+                return metadata[meta_key]
+
+        # Strategy 4: Partial path matching
+        if sample_file:
+            file_path_parts = sample_file.parts
+            for meta_key in metadata.keys():
+                meta_path = Path(meta_key)
+                # Check if any part of the metadata key matches our file path parts
+                for part in file_path_parts:
+                    if part in str(meta_path) or str(meta_path) in part:
+                        # Additional validation: ensure it's a reasonable match
+                        if meta_path.stem == file_name or meta_path.name == sample_file.name or part == meta_path.stem:
+                            return metadata[meta_key]
+
+        # Strategy 5: Full path matching
+        if sample_file:
+            full_path_str = str(sample_file)
+            full_path_posix = sample_file.as_posix()
+
+            for meta_key in metadata.keys():
+                # Direct path match
+                if meta_key == full_path_str or meta_key == full_path_posix:
+                    return metadata[meta_key]
+
+                # Relative path match (check if meta_key is contained in our path)
+                if meta_key in full_path_str or meta_key in full_path_posix:
+                    return metadata[meta_key]
+
+                # Reverse match (check if our path is contained in meta_key)
+                if full_path_str in meta_key or full_path_posix in meta_key:
+                    return metadata[meta_key]
+
+        # No match found
+        return {}