karaoke-gen 0.75.54__py3-none-any.whl

karaoke_gen/pipeline/__init__.py

@@ -0,0 +1,87 @@
+"""
+Shared pipeline architecture for karaoke-gen.
+
+This module provides a unified pipeline abstraction that can be executed
+either locally (in-process) or remotely (via backend workers).
+
+The pipeline consists of stages:
+- Separation: Audio separation into stems
+- Transcription: Lyrics transcription and synchronization
+- Screens: Title and end screen generation
+- Render: Video rendering with synchronized lyrics
+- Finalize: Encoding, packaging, and distribution
+
+Each stage can be executed by either a LocalExecutor or RemoteExecutor,
+allowing the same pipeline definition to work in both CLI and cloud contexts.
+
+Example usage:
+    from karaoke_gen.pipeline import (
+        PipelineContext,
+        LocalExecutor,
+        SeparationStage,
+        TranscriptionStage,
+    )
+    
+    # Create context
+    context = PipelineContext(
+        job_id="my-job",
+        artist="Artist Name",
+        title="Song Title",
+        input_audio_path="/path/to/audio.flac",
+        output_dir="/path/to/output",
+    )
+    
+    # Create stages
+    stages = [
+        SeparationStage(),
+        TranscriptionStage(),
+    ]
+    
+    # Run pipeline
+    executor = LocalExecutor()
+    results = await executor.run_pipeline(stages, context)
+"""
+from karaoke_gen.pipeline.base import (
+    PipelineStage,
+    PipelineExecutor,
+    StageResult,
+    StageStatus,
+)
+from karaoke_gen.pipeline.context import PipelineContext
+
+# Import stages
+from karaoke_gen.pipeline.stages import (
+    SeparationStage,
+    TranscriptionStage,
+    ScreensStage,
+    RenderStage,
+    FinalizeStage,
+)
+
+# Import executors
+from karaoke_gen.pipeline.executors import (
+    LocalExecutor,
+    RemoteExecutor,
+    create_local_executor,
+    create_remote_executor,
+)
+
+__all__ = [
+    # Base classes
+    "PipelineStage",
+    "PipelineExecutor",
+    "StageResult",
+    "StageStatus",
+    "PipelineContext",
+    # Stages
+    "SeparationStage",
+    "TranscriptionStage",
+    "ScreensStage",
+    "RenderStage",
+    "FinalizeStage",
+    # Executors
+    "LocalExecutor",
+    "RemoteExecutor",
+    "create_local_executor",
+    "create_remote_executor",
+]

karaoke_gen/pipeline/base.py

@@ -0,0 +1,215 @@
+"""
+Base classes and protocols for the pipeline architecture.
+
+This module defines the interfaces that all pipeline stages must implement,
+ensuring consistent behavior across different execution contexts.
+"""
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Dict, List, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from karaoke_gen.pipeline.context import PipelineContext
+
+
+class StageStatus(str, Enum):
+    """Status of a pipeline stage execution."""
+    PENDING = "pending"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    SKIPPED = "skipped"
+
+
+@dataclass
+class StageResult:
+    """
+    Result of a pipeline stage execution.
+    
+    Contains the outputs produced by the stage, any error information,
+    and metadata about the execution.
+    """
+    status: StageStatus
+    outputs: Dict[str, Any] = field(default_factory=dict)
+    error_message: Optional[str] = None
+    error_details: Optional[Dict[str, Any]] = None
+    duration_seconds: Optional[float] = None
+    
+    @property
+    def success(self) -> bool:
+        """Check if the stage completed successfully."""
+        return self.status == StageStatus.COMPLETED
+    
+    @property
+    def failed(self) -> bool:
+        """Check if the stage failed."""
+        return self.status == StageStatus.FAILED
+
+
+class PipelineStage(ABC):
+    """
+    Abstract base class for pipeline stages.
+    
+    Each stage represents a discrete unit of work in the karaoke generation
+    pipeline. Stages declare their dependencies (required inputs) and outputs,
+    allowing the pipeline executor to validate data flow.
+    
+    Stages should be stateless - all state is passed through PipelineContext.
+    """
+    
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """
+        Unique name for this stage.
+        
+        Used for logging, progress tracking, and output dictionary keys.
+        """
+        ...
+    
+    @property
+    def required_inputs(self) -> List[str]:
+        """
+        List of required input keys from previous stages.
+        
+        Override this to declare dependencies on outputs from other stages.
+        The pipeline executor will validate that these keys exist in
+        context.stage_outputs before running this stage.
+        
+        Returns:
+            List of required input keys (empty by default)
+        """
+        return []
+    
+    @property
+    def optional_inputs(self) -> List[str]:
+        """
+        List of optional input keys from previous stages.
+        
+        These inputs will be used if available but are not required.
+        
+        Returns:
+            List of optional input keys (empty by default)
+        """
+        return []
+    
+    @property
+    def output_keys(self) -> List[str]:
+        """
+        List of output keys this stage produces.
+        
+        Override this to declare what outputs this stage will add to
+        context.stage_outputs. This is used for documentation and
+        validation purposes.
+        
+        Returns:
+            List of output keys this stage produces
+        """
+        return []
+    
+    def validate_inputs(self, context: "PipelineContext") -> bool:
+        """
+        Validate that all required inputs are present.
+        
+        Args:
+            context: Pipeline context with stage outputs from previous stages
+            
+        Returns:
+            True if all required inputs are present, False otherwise
+        """
+        for key in self.required_inputs:
+            if key not in context.stage_outputs:
+                return False
+        return True
+    
+    def get_missing_inputs(self, context: "PipelineContext") -> List[str]:
+        """
+        Get list of missing required inputs.
+        
+        Args:
+            context: Pipeline context to check
+            
+        Returns:
+            List of missing input keys
+        """
+        return [key for key in self.required_inputs if key not in context.stage_outputs]
+    
+    @abstractmethod
+    async def execute(self, context: "PipelineContext") -> StageResult:
+        """
+        Execute the stage.
+        
+        This is the main entry point for stage execution. Implementations
+        should:
+        1. Read inputs from context.stage_outputs (using keys from required_inputs)
+        2. Perform the stage's work
+        3. Return a StageResult with outputs (using keys from output_keys)
+        
+        Args:
+            context: Pipeline context with all job parameters and stage outputs
+            
+        Returns:
+            StageResult containing outputs or error information
+        """
+        ...
+    
+    async def cleanup(self, context: "PipelineContext") -> None:
+        """
+        Clean up any resources after stage execution.
+        
+        Override this to perform cleanup operations like removing
+        temporary files. Called by the executor after stage completion.
+        
+        Args:
+            context: Pipeline context
+        """
+        pass
+    
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}(name={self.name})>"
+
+
+class PipelineExecutor(ABC):
+    """
+    Abstract base class for pipeline executors.
+    
+    Executors are responsible for running pipeline stages in a specific
+    context (local in-process, remote via API, etc.).
+    """
+    
+    @abstractmethod
+    async def run_stage(
+        self,
+        stage: PipelineStage,
+        context: "PipelineContext",
+    ) -> StageResult:
+        """
+        Execute a single pipeline stage.
+        
+        Args:
+            stage: The stage to execute
+            context: Pipeline context
+            
+        Returns:
+            Result of stage execution
+        """
+        ...
+    
+    @abstractmethod
+    async def run_pipeline(
+        self,
+        stages: List[PipelineStage],
+        context: "PipelineContext",
+    ) -> Dict[str, StageResult]:
+        """
+        Execute a full pipeline of stages.
+        
+        Args:
+            stages: List of stages to execute in order
+            context: Pipeline context
+            
+        Returns:
+            Dictionary mapping stage names to their results
+        """
+        ...

karaoke_gen/pipeline/context.py

@@ -0,0 +1,230 @@
+"""
+Pipeline context for passing state through pipeline stages.
+
+The PipelineContext is the central data structure that flows through
+all pipeline stages, carrying:
+- Job metadata (ID, artist, title)
+- File paths (input audio, output directory)
+- Style configuration
+- Outputs from each stage
+- Logging and progress callbacks
+"""
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional
+import logging
+
+
+@dataclass
+class PipelineContext:
+    """
+    Shared state passed through pipeline stages.
+    
+    This context object carries all information needed by pipeline stages
+    to perform their work, and accumulates outputs from each stage.
+    """
+    
+    # Job identification
+    job_id: str
+    artist: str
+    title: str
+    
+    # File paths
+    input_audio_path: str
+    output_dir: str
+    
+    # Style configuration (loaded from style_params.json)
+    style_params: Dict[str, Any] = field(default_factory=dict)
+    
+    # Processing options
+    enable_cdg: bool = True
+    enable_txt: bool = True
+    brand_prefix: Optional[str] = None
+    enable_youtube_upload: bool = False
+    discord_webhook_url: Optional[str] = None
+    
+    # Distribution options (native API for remote, rclone for local)
+    dropbox_path: Optional[str] = None
+    gdrive_folder_id: Optional[str] = None
+    organised_dir_rclone_root: Optional[str] = None
+    rclone_destination: Optional[str] = None
+    
+    # Accumulated outputs from each stage
+    # Keys are typically: separation, transcription, screens, render, finalize
+    stage_outputs: Dict[str, Any] = field(default_factory=dict)
+    
+    # Progress tracking
+    current_stage: Optional[str] = None
+    progress_percent: int = 0
+    
+    # Callbacks for progress updates and logging
+    on_progress: Optional[Callable[[str, int, str], None]] = None
+    on_log: Optional[Callable[[str, str, str], None]] = None
+    
+    # Logger instance
+    logger: Optional[logging.Logger] = None
+    
+    # Temporary files/directories to clean up
+    temp_paths: List[str] = field(default_factory=list)
+    
+    @property
+    def base_name(self) -> str:
+        """Standard base name for output files: 'Artist - Title'"""
+        return f"{self.artist} - {self.title}"
+    
+    @property
+    def output_path(self) -> Path:
+        """Output directory as Path object."""
+        return Path(self.output_dir)
+    
+    @property
+    def input_path(self) -> Path:
+        """Input audio path as Path object."""
+        return Path(self.input_audio_path)
+    
+    def get_stage_output(self, stage: str, key: str, default: Any = None) -> Any:
+        """
+        Get a specific output from a stage.
+        
+        Args:
+            stage: Stage name
+            key: Output key within that stage
+            default: Default value if not found
+            
+        Returns:
+            The output value, or default if not found
+        """
+        stage_data = self.stage_outputs.get(stage, {})
+        if isinstance(stage_data, dict):
+            return stage_data.get(key, default)
+        return default
+    
+    def set_stage_output(self, stage: str, outputs: Dict[str, Any]) -> None:
+        """
+        Set outputs for a stage.
+        
+        Args:
+            stage: Stage name
+            outputs: Dictionary of outputs to store
+        """
+        if stage not in self.stage_outputs:
+            self.stage_outputs[stage] = {}
+        
+        if isinstance(self.stage_outputs[stage], dict):
+            self.stage_outputs[stage].update(outputs)
+        else:
+            self.stage_outputs[stage] = outputs
+    
+    def update_progress(self, stage: str, percent: int, message: str = "") -> None:
+        """
+        Update progress and notify callback if set.
+        
+        Args:
+            stage: Current stage name
+            percent: Progress percentage (0-100)
+            message: Optional progress message
+        """
+        self.current_stage = stage
+        self.progress_percent = percent
+        
+        if self.on_progress:
+            self.on_progress(stage, percent, message)
+    
+    def log(self, level: str, message: str) -> None:
+        """
+        Log a message using the configured logger or callback.
+        
+        Args:
+            level: Log level (DEBUG, INFO, WARNING, ERROR)
+            message: Message to log
+        """
+        if self.logger:
+            log_method = getattr(self.logger, level.lower(), self.logger.info)
+            log_method(f"[{self.current_stage or 'pipeline'}] {message}")
+        
+        if self.on_log:
+            self.on_log(self.current_stage or "pipeline", level, message)
+    
+    def add_temp_path(self, path: str) -> None:
+        """
+        Register a temporary path for cleanup.
+        
+        Args:
+            path: Path to temporary file or directory
+        """
+        self.temp_paths.append(path)
+    
+    def cleanup_temp_paths(self) -> None:
+        """Remove all registered temporary paths."""
+        import shutil
+        import os
+        
+        for path in self.temp_paths:
+            try:
+                if os.path.isdir(path):
+                    shutil.rmtree(path)
+                elif os.path.isfile(path):
+                    os.remove(path)
+            except Exception:
+                pass  # Best effort cleanup
+        
+        self.temp_paths.clear()
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Convert context to a dictionary for serialization.
+        
+        Excludes non-serializable fields (callbacks, logger).
+        """
+        return {
+            "job_id": self.job_id,
+            "artist": self.artist,
+            "title": self.title,
+            "input_audio_path": self.input_audio_path,
+            "output_dir": self.output_dir,
+            "style_params": self.style_params,
+            "enable_cdg": self.enable_cdg,
+            "enable_txt": self.enable_txt,
+            "brand_prefix": self.brand_prefix,
+            "enable_youtube_upload": self.enable_youtube_upload,
+            "discord_webhook_url": self.discord_webhook_url,
+            "dropbox_path": self.dropbox_path,
+            "gdrive_folder_id": self.gdrive_folder_id,
+            "organised_dir_rclone_root": self.organised_dir_rclone_root,
+            "rclone_destination": self.rclone_destination,
+            "stage_outputs": self.stage_outputs,
+            "current_stage": self.current_stage,
+            "progress_percent": self.progress_percent,
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "PipelineContext":
+        """
+        Create a context from a dictionary.
+        
+        Args:
+            data: Dictionary with context fields
+            
+        Returns:
+            New PipelineContext instance
+        """
+        return cls(
+            job_id=data.get("job_id", ""),
+            artist=data.get("artist", ""),
+            title=data.get("title", ""),
+            input_audio_path=data.get("input_audio_path", ""),
+            output_dir=data.get("output_dir", ""),
+            style_params=data.get("style_params", {}),
+            enable_cdg=data.get("enable_cdg", True),
+            enable_txt=data.get("enable_txt", True),
+            brand_prefix=data.get("brand_prefix"),
+            enable_youtube_upload=data.get("enable_youtube_upload", False),
+            discord_webhook_url=data.get("discord_webhook_url"),
+            dropbox_path=data.get("dropbox_path"),
+            gdrive_folder_id=data.get("gdrive_folder_id"),
+            organised_dir_rclone_root=data.get("organised_dir_rclone_root"),
+            rclone_destination=data.get("rclone_destination"),
+            stage_outputs=data.get("stage_outputs", {}),
+            current_stage=data.get("current_stage"),
+            progress_percent=data.get("progress_percent", 0),
+        )

karaoke_gen/pipeline/executors/__init__.py

@@ -0,0 +1,21 @@
+"""
+Pipeline executors for running stages in different contexts.
+
+Executors handle the actual execution of pipeline stages, providing
+different implementations for different execution contexts:
+
+- LocalExecutor: Runs stages directly in-process (for CLI)
+- RemoteExecutor: Runs stages via backend API (for remote CLI)
+
+Both executors use the same stage interface, ensuring consistent
+behavior regardless of execution context.
+"""
+from karaoke_gen.pipeline.executors.local import LocalExecutor, create_local_executor
+from karaoke_gen.pipeline.executors.remote import RemoteExecutor, create_remote_executor
+
+__all__ = [
+    "LocalExecutor",
+    "RemoteExecutor",
+    "create_local_executor",
+    "create_remote_executor",
+]