synapse-sdk 1.0.0b5__py3-none-any.whl → 2025.12.3__py3-none-any.whl

synapse_sdk/plugins/categories/upload/actions/upload/models.py

@@ -0,0 +1,214 @@
+from typing import Annotated
+
+from pydantic import AfterValidator, BaseModel, ValidationInfo, field_validator, model_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.utils.pydantic.validators import non_blank
+
+from .enums import VALIDATION_ERROR_MESSAGES, ValidationErrorCode
+
+
+class AssetConfig(BaseModel):
+    """Configuration for individual asset in multi-path mode.
+
+    Used when use_single_path=False to specify unique paths and recursive
+    settings for each file specification.
+
+    Attributes:
+        path: File system path for this specific asset.
+        is_recursive: Whether to recursively search subdirectories for this
+            asset. Defaults to True.
+
+    Examples:
+        >>> asset_config = AssetConfig(
+        ...     path="/sensors/camera/front",
+        ...     is_recursive=True
+        ... )
+    """
+
+    path: str
+    is_recursive: bool = True
+
+
+class UploadParams(BaseModel):
+    """Upload action parameter validation model.
+
+    Defines and validates all parameters required for upload operations.
+    Uses Pydantic for type validation and custom validators to ensure
+    storage, data_collection, and project resources exist before processing.
+
+    The model supports two operational modes controlled by the use_single_path
+    flag:
+
+    Single Path Mode (use_single_path=True, DEFAULT):
+        Traditional mode where all file specifications share one base path.
+        Requires 'path' parameter. Ignores 'assets' parameter.
+
+    Multi-Path Mode (use_single_path=False):
+        Advanced mode where each file specification has its own path.
+        Requires 'assets' parameter. Ignores 'path' and 'is_recursive' parameters.
+
+    Attributes:
+        name: Human-readable name for the upload operation. Must be non-blank.
+        description: Optional description of the upload operation.
+        use_single_path: Mode selector. True for single path mode, False for
+            multi-path mode. Defaults to True.
+        path: Base path for single path mode. Required when use_single_path=True.
+        is_recursive: Global recursive setting for single path mode.
+            Defaults to True.
+        assets: Per-asset path configurations for multi-path mode. Dictionary
+            mapping file specification names to AssetConfig objects. Required
+            when use_single_path=False.
+        storage: Storage ID where files will be uploaded. Must exist and be
+            accessible via client API.
+        data_collection: Data collection ID for organizing uploads. Must exist
+            and be accessible via client API.
+        project: Optional project ID for grouping. Must exist if specified.
+        excel_metadata_path: Path to Excel metadata file. Can be:
+            - Absolute path: '/data/metadata.xlsx'
+            - Relative to storage default path: 'metadata.xlsx'
+            - Relative to working directory (single-path mode): 'metadata.xlsx'
+        max_file_size_mb: Maximum file size limit in megabytes. Defaults to 50.
+        creating_data_unit_batch_size: Batch size for data unit creation.
+            Defaults to 1.
+        use_async_upload: Whether to use asynchronous upload processing.
+            Defaults to True.
+        extra_params: Extra parameters for the action. Optional.
+
+    Examples:
+        Single Path Mode (Traditional):
+
+            >>> params = UploadParams(
+            ...     name="Standard Upload",
+            ...     use_single_path=True,
+            ...     path="/data/experiment_1",
+            ...     is_recursive=True,
+            ...     storage=1,
+            ...     data_collection=5
+            ... )
+
+        Multi-Path Mode (Advanced):
+
+            >>> params = UploadParams(
+            ...     name="Multi-Source Upload",
+            ...     use_single_path=False,
+            ...     assets={
+            ...         "image_1": AssetConfig(path="/sensors/camera", is_recursive=True),
+            ...         "pcd_1": AssetConfig(path="/sensors/lidar", is_recursive=False)
+            ...     },
+            ...     storage=1,
+            ...     data_collection=5
+            ... )
+
+        With Excel Metadata:
+
+            >>> params = UploadParams(
+            ...     name="Upload with Metadata",
+            ...     path="/data/files",
+            ...     storage=1,
+            ...     data_collection=5,
+            ...     excel_metadata_path="metadata.xlsx"
+            ... )
+    """
+
+    name: Annotated[str, AfterValidator(non_blank)]
+    description: str | None = None
+
+    # Mode selector flag (True = single path mode, False = multi-path mode)
+    use_single_path: bool = True
+
+    # Single path mode fields (used when use_single_path=True)
+    path: str | None = None
+    is_recursive: bool = True
+
+    # Multi-path mode fields (used when use_single_path=False)
+    assets: dict[str, AssetConfig] | None = None
+
+    storage: int
+    data_collection: int
+    project: int | None = None
+
+    # Excel metadata file path (absolute or relative to working directory)
+    excel_metadata_path: str | None = None
+
+    max_file_size_mb: int = 50
+    creating_data_unit_batch_size: int = 1
+    extra_params: dict | None = None
+
+    @field_validator('storage', mode='before')
+    @classmethod
+    def check_storage_exists(cls, value, info: ValidationInfo) -> int:
+        if info.context is None:
+            error_code = ValidationErrorCode.MISSING_CONTEXT
+            raise PydanticCustomError(error_code.value, VALIDATION_ERROR_MESSAGES[error_code])
+
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_storage(value)
+        except ClientError as e:
+            error_code = ValidationErrorCode.STORAGE_NOT_FOUND
+            error_message = VALIDATION_ERROR_MESSAGES[error_code].format(value, str(e))
+            raise PydanticCustomError(error_code.value, error_message)
+        return value
+
+    @field_validator('data_collection', mode='before')
+    @classmethod
+    def check_data_collection_exists(cls, value, info: ValidationInfo) -> int:
+        if info.context is None:
+            error_code = ValidationErrorCode.MISSING_CONTEXT
+            raise PydanticCustomError(error_code.value, VALIDATION_ERROR_MESSAGES[error_code])
+
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_data_collection(value)
+        except ClientError as e:
+            error_code = ValidationErrorCode.DATA_COLLECTION_NOT_FOUND
+            error_message = VALIDATION_ERROR_MESSAGES[error_code].format(value, str(e))
+            raise PydanticCustomError(error_code.value, error_message)
+        return value
+
+    @field_validator('project', mode='before')
+    @classmethod
+    def check_project_exists(cls, value, info: ValidationInfo) -> int | None:
+        if not value:
+            return value
+
+        if info.context is None:
+            error_code = ValidationErrorCode.MISSING_CONTEXT
+            raise PydanticCustomError(error_code.value, VALIDATION_ERROR_MESSAGES[error_code])
+
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_project(value)
+        except ClientError as e:
+            error_code = ValidationErrorCode.PROJECT_NOT_FOUND
+            error_message = VALIDATION_ERROR_MESSAGES[error_code].format(value, str(e))
+            raise PydanticCustomError(error_code.value, error_message)
+        return value
+
+    @model_validator(mode='after')
+    def validate_path_configuration(self) -> 'UploadParams':
+        """Validate path configuration based on use_single_path mode."""
+        if self.use_single_path:
+            # Single path mode: requires path
+            if not self.path:
+                raise PydanticCustomError(
+                    'missing_path', "When use_single_path=true (single path mode), 'path' is required"
+                )
+            # Warn if assets is provided in single path mode (it will be ignored)
+            # For now, we'll silently ignore it
+        else:
+            # Multi-path mode: requires assets
+            if not self.assets:
+                raise PydanticCustomError(
+                    'missing_assets',
+                    "When use_single_path=false (multi-path mode), 'assets' must be provided "
+                    'with path configurations for each file specification',
+                )
+            # path and is_recursive are ignored in multi-path mode
+
+        return self

synapse_sdk/plugins/categories/upload/actions/upload/orchestrator.py

@@ -0,0 +1,183 @@
+import traceback
+from typing import Any, Dict, List
+
+from .context import UploadContext
+from .enums import LogCode
+from .registry import StepRegistry
+from .steps.base import BaseStep
+
+
+class UploadOrchestrator:
+    """Facade that orchestrates the upload workflow using strategies and steps."""
+
+    def __init__(self, context: UploadContext, step_registry: StepRegistry, strategies: Dict[str, Any]):
+        self.context = context
+        self.step_registry = step_registry
+        self.strategies = strategies
+        self.executed_steps: List[BaseStep] = []
+        self.current_step_index = 0
+
+    def execute(self) -> Dict[str, Any]:
+        """Execute the complete upload workflow."""
+        try:
+            self._log_workflow_start()
+            self._inject_strategies_into_context()
+
+            steps = self.step_registry.get_steps()
+            total_steps = len(steps)
+
+            for i, step in enumerate(steps):
+                self.current_step_index = i
+
+                try:
+                    result = step.safe_execute(self.context)
+                    self.context.update(result)
+
+                    if result.success:
+                        if not result.skipped:
+                            self.executed_steps.append(step)
+                        self._update_progress(i + 1, total_steps)
+                    else:
+                        # Step failed, initiate rollback
+                        self._log_step_failure(step, result.error)
+                        self._rollback()
+                        # Re-raise original exception if available, otherwise create new one
+                        if result.original_exception:
+                            raise result.original_exception
+                        else:
+                            raise Exception(f"Step '{step.name}' failed: {result.error}")
+
+                except Exception as e:
+                    self._log_step_exception(step, str(e))
+                    self._rollback()
+                    raise
+
+            self._log_workflow_complete()
+            return self.context.get_result()
+
+        except Exception as e:
+            self._log_workflow_error(str(e))
+            # Ensure rollback is called if not already done
+            if not hasattr(self, '_rollback_executed'):
+                self._rollback()
+            raise
+
+    def _inject_strategies_into_context(self) -> None:
+        """Inject strategies into context for steps to use."""
+        if not hasattr(self.context, 'strategies'):
+            self.context.strategies = {}
+        self.context.strategies.update(self.strategies)
+
+    def _rollback(self) -> None:
+        """Rollback executed steps in reverse order."""
+        if hasattr(self, '_rollback_executed'):
+            return  # Prevent multiple rollbacks
+
+        self._rollback_executed = True
+        self._log_rollback_start()
+
+        # Rollback in reverse order
+        for step in reversed(self.executed_steps):
+            try:
+                self._log_step_rollback(step)
+                step.rollback(self.context)
+            except Exception as e:
+                # Log rollback error but continue with other steps
+                self._log_rollback_error(step, str(e))
+
+        self._log_rollback_complete()
+
+    def _update_progress(self, current_step: int, total_steps: int) -> None:
+        """Update overall progress based on step completion."""
+        if total_steps == 0:
+            return
+
+        # Calculate progress based on step weights
+        completed_weight = 0.0
+        total_weight = self.step_registry.get_total_progress_weight()
+
+        for i, step in enumerate(self.executed_steps):
+            completed_weight += step.progress_weight
+
+        progress_percentage = (completed_weight / total_weight) * 100 if total_weight > 0 else 0
+
+        # Update context with progress information
+        self.context.update_metrics(
+            'workflow',
+            {
+                'current_step': current_step,
+                'total_steps': total_steps,
+                'progress_percentage': progress_percentage,
+                'completed_weight': completed_weight,
+                'total_weight': total_weight,
+            },
+        )
+
+    def _log_workflow_start(self) -> None:
+        """Log workflow start."""
+        steps = self.step_registry.get_steps()
+        step_names = [step.name for step in steps]
+        self.context.run.log_message_with_code(LogCode.WORKFLOW_STARTING, len(steps), step_names)
+
+    def _log_workflow_complete(self) -> None:
+        """Log workflow completion."""
+        self.context.run.log_message_with_code(LogCode.WORKFLOW_COMPLETED)
+
+    def _log_workflow_error(self, error: str) -> None:
+        """Log workflow error."""
+        self.context.run.log_message_with_code(LogCode.WORKFLOW_FAILED, error)
+
+    def _log_step_failure(self, step: BaseStep, error: str) -> None:
+        """Log step failure."""
+        self.context.run.log_message_with_code(LogCode.STEP_FAILED, step.name, error)
+
+    def _log_step_exception(self, step: BaseStep, error: str) -> None:
+        """Log step exception."""
+        self.context.run.log_message_with_code(LogCode.STEP_EXCEPTION, step.name, error)
+        # Log full traceback for debugging
+        self.context.run.log_message_with_code(LogCode.STEP_TRACEBACK, traceback.format_exc())
+
+    def _log_rollback_start(self) -> None:
+        """Log rollback start."""
+        self.context.run.log_message_with_code(LogCode.ROLLBACK_STARTING, len(self.executed_steps))
+
+    def _log_rollback_complete(self) -> None:
+        """Log rollback completion."""
+        self.context.run.log_message_with_code(LogCode.ROLLBACK_COMPLETED)
+
+    def _log_step_rollback(self, step: BaseStep) -> None:
+        """Log step rollback."""
+        self.context.run.log_message_with_code(LogCode.STEP_ROLLBACK, step.name)
+
+    def _log_rollback_error(self, step: BaseStep, error: str) -> None:
+        """Log rollback error."""
+        self.context.run.log_message_with_code(LogCode.ROLLBACK_ERROR, step.name, error)
+
+    def get_executed_steps(self) -> List[BaseStep]:
+        """Get list of successfully executed steps."""
+        return self.executed_steps.copy()
+
+    def get_current_step_index(self) -> int:
+        """Get current step index."""
+        return self.current_step_index
+
+    def get_total_steps(self) -> int:
+        """Get total number of steps."""
+        return len(self.step_registry.get_steps())
+
+    def is_rollback_executed(self) -> bool:
+        """Check if rollback has been executed."""
+        return hasattr(self, '_rollback_executed')
+
+    def get_workflow_summary(self) -> Dict[str, Any]:
+        """Get workflow execution summary."""
+        steps = self.step_registry.get_steps()
+        return {
+            'total_steps': len(steps),
+            'executed_steps': len(self.executed_steps),
+            'current_step_index': self.current_step_index,
+            'step_names': [step.name for step in steps],
+            'executed_step_names': [step.name for step in self.executed_steps],
+            'rollback_executed': self.is_rollback_executed(),
+            'strategies': list(self.strategies.keys()) if self.strategies else [],
+        }

synapse_sdk/plugins/categories/upload/actions/upload/registry.py

@@ -0,0 +1,113 @@
+from typing import Dict, List, Optional
+
+from .steps.base import BaseStep
+
+
+class StepRegistry:
+    """Registry for managing workflow steps."""
+
+    def __init__(self):
+        self._steps: List[BaseStep] = []
+        self._step_by_name: Dict[str, BaseStep] = {}
+
+    def register(self, step: BaseStep) -> None:
+        """Register a step in the workflow."""
+        if step.name in self._step_by_name:
+            raise ValueError(f"Step with name '{step.name}' already registered")
+
+        self._steps.append(step)
+        self._step_by_name[step.name] = step
+
+    def unregister(self, step_name: str) -> bool:
+        """Unregister a step by name. Returns True if step was found and removed."""
+        if step_name not in self._step_by_name:
+            return False
+
+        step = self._step_by_name[step_name]
+        self._steps.remove(step)
+        del self._step_by_name[step_name]
+        return True
+
+    def get_steps(self) -> List[BaseStep]:
+        """Get all registered steps in order."""
+        return self._steps.copy()
+
+    def get_step(self, name: str) -> Optional[BaseStep]:
+        """Get a step by name."""
+        return self._step_by_name.get(name)
+
+    def has_step(self, name: str) -> bool:
+        """Check if a step is registered."""
+        return name in self._step_by_name
+
+    def clear(self) -> None:
+        """Clear all registered steps."""
+        self._steps.clear()
+        self._step_by_name.clear()
+
+    def get_step_names(self) -> List[str]:
+        """Get list of all registered step names."""
+        return list(self._step_by_name.keys())
+
+    def get_total_progress_weight(self) -> float:
+        """Get total progress weight of all steps."""
+        return sum(step.progress_weight for step in self._steps)
+
+    def insert_step_after(self, after_step_name: str, new_step: BaseStep) -> None:
+        """Insert a step after an existing step."""
+        if after_step_name not in self._step_by_name:
+            raise ValueError(f"Step '{after_step_name}' not found")
+
+        if new_step.name in self._step_by_name:
+            raise ValueError(f"Step with name '{new_step.name}' already registered")
+
+        # Find the index of the step to insert after
+        after_step = self._step_by_name[after_step_name]
+        index = self._steps.index(after_step) + 1
+
+        # Insert the new step
+        self._steps.insert(index, new_step)
+        self._step_by_name[new_step.name] = new_step
+
+    def insert_step_before(self, before_step_name: str, new_step: BaseStep) -> None:
+        """Insert a step before an existing step."""
+        if before_step_name not in self._step_by_name:
+            raise ValueError(f"Step '{before_step_name}' not found")
+
+        if new_step.name in self._step_by_name:
+            raise ValueError(f"Step with name '{new_step.name}' already registered")
+
+        # Find the index of the step to insert before
+        before_step = self._step_by_name[before_step_name]
+        index = self._steps.index(before_step)
+
+        # Insert the new step
+        self._steps.insert(index, new_step)
+        self._step_by_name[new_step.name] = new_step
+
+    def reorder_steps(self, step_names: List[str]) -> None:
+        """Reorder steps according to the provided list of step names."""
+        if set(step_names) != set(self._step_by_name.keys()):
+            raise ValueError('Step names list must contain all registered steps')
+
+        # Reorder the steps list
+        self._steps = [self._step_by_name[name] for name in step_names]
+
+    def __len__(self) -> int:
+        """Return number of registered steps."""
+        return len(self._steps)
+
+    def __iter__(self):
+        """Iterate over registered steps."""
+        return iter(self._steps)
+
+    def __contains__(self, step_name: str) -> bool:
+        """Check if step name is registered."""
+        return step_name in self._step_by_name
+
+    def __str__(self) -> str:
+        step_names = [step.name for step in self._steps]
+        return f'StepRegistry({step_names})'
+
+    def __repr__(self) -> str:
+        return f'StepRegistry(steps={len(self._steps)})'

synapse_sdk/plugins/categories/upload/actions/upload/run.py

@@ -0,0 +1,179 @@
+import json
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel
+
+from synapse_sdk.plugins.models import Run
+from synapse_sdk.shared.enums import Context
+
+from .enums import LOG_MESSAGES, LogCode, UploadStatus
+from .utils import PathAwareJSONEncoder
+
+
+class UploadRun(Run):
+    """Upload-specific run management class.
+
+    Extends the base Run class with upload-specific logging capabilities
+    and event tracking. Provides type-safe logging using LogCode enums
+    and specialized methods for tracking upload progress.
+
+    Manages logging for upload events, data files, data units, and tasks
+    throughout the upload lifecycle. Each log entry includes status,
+    timestamps, and relevant metadata.
+
+    Attributes:
+        Inherits all attributes from base Run class plus upload-specific
+        logging methods and nested model classes for structured logging.
+
+    Example:
+        >>> run = UploadRun(job_id, context)
+        >>> run.log_message_with_code(LogCode.UPLOAD_STARTED)
+        >>> run.log_upload_event(LogCode.FILES_DISCOVERED, file_count)
+    """
+
+    class UploadEventLog(BaseModel):
+        """Model for upload event log entries.
+
+        Records significant events during upload processing with
+        status information and timestamps.
+
+        Attributes:
+            info (str | None): Optional additional information
+            status (Context): Event status/severity level
+            created (str): Timestamp when event occurred
+        """
+
+        info: Optional[str] = None
+        status: Context
+        created: str
+
+    class DataFileLog(BaseModel):
+        """Model for data file processing log entries.
+
+        Tracks the processing status of individual data files
+        during upload operations.
+
+        Attributes:
+            data_file_info (str | None): Information about the data file
+            status (UploadStatus): Processing status (SUCCESS/FAILED)
+            created (str): Timestamp when log entry was created
+        """
+
+        data_file_info: str | None
+        status: UploadStatus
+        created: str
+
+    class DataUnitLog(BaseModel):
+        """Model for data unit creation log entries.
+
+        Records the creation status of data units generated from
+        uploaded files, including metadata and identifiers.
+
+        Attributes:
+            data_unit_id (int | None): ID of created data unit
+            status (UploadStatus): Creation status (SUCCESS/FAILED)
+            created (str): Timestamp when log entry was created
+            data_unit_meta (dict | None): Metadata associated with data unit
+        """
+
+        data_unit_id: int | None
+        status: UploadStatus
+        created: str
+        data_unit_meta: dict | None
+
+    class TaskLog(BaseModel):
+        """Model for task execution log entries.
+
+        Tracks the execution status of background tasks related
+        to upload processing.
+
+        Attributes:
+            task_id (int | None): ID of the executed task
+            status (UploadStatus): Task execution status (SUCCESS/FAILED)
+            created (str): Timestamp when log entry was created
+        """
+
+        task_id: int | None
+        status: UploadStatus
+        created: str
+
+    class MetricsRecord(BaseModel):
+        """Model for upload metrics tracking.
+
+        Records count-based metrics for monitoring upload
+        progress and success rates.
+
+        Attributes:
+            stand_by (int): Number of items waiting to be processed
+            failed (int): Number of items that failed processing
+            success (int): Number of items successfully processed
+        """
+
+        stand_by: int
+        failed: int
+        success: int
+
+    def log_message_with_code(self, code: LogCode, *args, level: Optional[Context] = None):
+        if code not in LOG_MESSAGES:
+            # Use direct log_message to avoid recursion
+            self.log_message(f'Unknown log code: {code}')
+            return
+
+        log_config = LOG_MESSAGES[code]
+        message = log_config['message'].format(*args) if args else log_config['message']
+        log_level = level or log_config['level'] or Context.INFO
+
+        # Always call log_message for basic logging
+        if log_level:
+            self.log_message(message, context=log_level.value)
+        else:
+            self.log_message(message)
+
+    def log_upload_event(self, code: LogCode, *args, level: Optional[Context] = None):
+        # Call log_message_with_code to handle the basic logging
+        self.log_message_with_code(code, *args, level=level)
+
+        # Also log the event for upload-specific tracking
+        if code not in LOG_MESSAGES:
+            now = datetime.now().isoformat()
+            self.log(
+                'upload_event',
+                self.UploadEventLog(info=f'Unknown log code: {code}', status=Context.DANGER, created=now).model_dump(),
+            )
+            return
+
+        log_config = LOG_MESSAGES[code]
+        message = log_config['message'].format(*args) if args else log_config['message']
+        log_level = level or log_config['level'] or Context.INFO
+
+        now = datetime.now().isoformat()
+        self.log(
+            'upload_event',
+            self.UploadEventLog(info=message, status=log_level, created=now).model_dump(),
+        )
+
+    def log_data_file(self, data_file_info: dict, status: UploadStatus):
+        now = datetime.now().isoformat()
+        data_file_info_str = json.dumps(data_file_info, ensure_ascii=False, cls=PathAwareJSONEncoder)
+        self.log(
+            'upload_data_file',
+            self.DataFileLog(data_file_info=data_file_info_str, status=status, created=now).model_dump(),
+        )
+
+    def log_data_unit(self, data_unit_id: int, status: UploadStatus, data_unit_meta: dict | None = None):
+        now = datetime.now().isoformat()
+        self.log(
+            'upload_data_unit',
+            self.DataUnitLog(
+                data_unit_id=data_unit_id, status=status, created=now, data_unit_meta=data_unit_meta
+            ).model_dump(),
+        )
+
+    def log_task(self, task_id: int, status: UploadStatus):
+        now = datetime.now().isoformat()
+        self.log('upload_task', self.TaskLog(task_id=task_id, status=status, created=now).model_dump())
+
+    def log_metrics(self, record: MetricsRecord, category: str):
+        record = self.MetricsRecord.model_validate(record)
+        self.set_metrics(value=record.model_dump(), category=category)

synapse_sdk/plugins/categories/upload/actions/upload/steps/__init__.py

@@ -0,0 +1 @@
+# Step-based workflow implementations for upload actions