karaoke-gen 0.57.0__py3-none-any.whl → 0.71.27__py3-none-any.whl

karaoke_gen/pipeline/executors/local.py

@@ -0,0 +1,159 @@
+"""
+Local pipeline executor.
+
+This executor runs pipeline stages directly in-process,
+suitable for CLI usage where all processing happens locally.
+"""
+import logging
+import time
+from typing import Dict, List
+
+from karaoke_gen.pipeline.base import PipelineExecutor, PipelineStage, StageResult, StageStatus
+from karaoke_gen.pipeline.context import PipelineContext
+
+
+class LocalExecutor(PipelineExecutor):
+    """
+    Runs pipeline stages directly in-process.
+    
+    This executor is used by the local CLI (karaoke-gen) to run
+    all processing stages sequentially on the local machine.
+    """
+    
+    def __init__(
+        self,
+        logger: logging.Logger = None,
+        stop_on_failure: bool = True,
+    ):
+        """
+        Initialize the local executor.
+        
+        Args:
+            logger: Logger instance
+            stop_on_failure: If True, stop pipeline on first failure
+        """
+        self.logger = logger or logging.getLogger(__name__)
+        self.stop_on_failure = stop_on_failure
+    
+    async def run_stage(
+        self,
+        stage: PipelineStage,
+        context: PipelineContext,
+    ) -> StageResult:
+        """
+        Execute a single pipeline stage.
+        
+        Args:
+            stage: The stage to execute
+            context: Pipeline context
+            
+        Returns:
+            Result of stage execution
+        """
+        self.logger.info(f"Starting stage: {stage.name}")
+        context.update_progress(stage.name, 0, f"Starting {stage.name}")
+        
+        # Validate inputs
+        if not stage.validate_inputs(context):
+            missing = stage.get_missing_inputs(context)
+            error_msg = f"Stage {stage.name} missing required inputs: {missing}"
+            self.logger.error(error_msg)
+            return StageResult(
+                status=StageStatus.FAILED,
+                error_message=error_msg,
+            )
+        
+        start_time = time.time()
+        
+        try:
+            # Execute the stage
+            result = await stage.execute(context)
+            
+            # Store outputs in context
+            if result.success and result.outputs:
+                context.set_stage_output(stage.name, result.outputs)
+            
+            # Log result
+            duration = time.time() - start_time
+            if result.success:
+                self.logger.info(f"Stage {stage.name} completed in {duration:.1f}s")
+            elif result.status == StageStatus.SKIPPED:
+                self.logger.info(f"Stage {stage.name} skipped")
+            else:
+                self.logger.error(f"Stage {stage.name} failed: {result.error_message}")
+            
+            return result
+            
+        except Exception as e:
+            duration = time.time() - start_time
+            self.logger.error(f"Stage {stage.name} raised exception: {e}", exc_info=True)
+            return StageResult(
+                status=StageStatus.FAILED,
+                error_message=str(e),
+                error_details={"exception_type": type(e).__name__},
+                duration_seconds=duration,
+            )
+        finally:
+            # Run cleanup
+            try:
+                await stage.cleanup(context)
+            except Exception as e:
+                self.logger.warning(f"Stage {stage.name} cleanup failed: {e}")
+    
+    async def run_pipeline(
+        self,
+        stages: List[PipelineStage],
+        context: PipelineContext,
+    ) -> Dict[str, StageResult]:
+        """
+        Execute a full pipeline of stages.
+        
+        Runs stages sequentially in order, stopping on failure
+        if stop_on_failure is True.
+        
+        Args:
+            stages: List of stages to execute in order
+            context: Pipeline context
+            
+        Returns:
+            Dictionary mapping stage names to their results
+        """
+        results: Dict[str, StageResult] = {}
+        
+        self.logger.info(f"Starting pipeline with {len(stages)} stages")
+        pipeline_start = time.time()
+        
+        for i, stage in enumerate(stages):
+            self.logger.info(f"Running stage {i+1}/{len(stages)}: {stage.name}")
+            
+            # Calculate overall progress
+            base_progress = int((i / len(stages)) * 100)
+            context.update_progress(stage.name, base_progress, f"Starting {stage.name}")
+            
+            # Run the stage
+            result = await self.run_stage(stage, context)
+            results[stage.name] = result
+            
+            # Check for failure
+            if result.failed and self.stop_on_failure:
+                self.logger.error(f"Pipeline stopped due to failure in stage: {stage.name}")
+                break
+        
+        pipeline_duration = time.time() - pipeline_start
+        
+        # Count results
+        completed = sum(1 for r in results.values() if r.status == StageStatus.COMPLETED)
+        failed = sum(1 for r in results.values() if r.status == StageStatus.FAILED)
+        skipped = sum(1 for r in results.values() if r.status == StageStatus.SKIPPED)
+        
+        self.logger.info(
+            f"Pipeline completed in {pipeline_duration:.1f}s: "
+            f"{completed} completed, {failed} failed, {skipped} skipped"
+        )
+        
+        return results
+
+
+def create_local_executor(logger: logging.Logger = None) -> LocalExecutor:
+    """Factory function to create a LocalExecutor."""
+    return LocalExecutor(logger=logger)

karaoke_gen/pipeline/executors/remote.py

@@ -0,0 +1,257 @@
+"""
+Remote pipeline executor.
+
+This executor runs pipeline stages via the backend API,
+suitable for the remote CLI where processing happens in the cloud.
+
+Note: This is a placeholder implementation. The actual remote
+execution is handled by the existing backend workers. This executor
+provides a compatible interface for potential future unified
+pipeline execution.
+"""
+import logging
+import time
+from typing import Any, Dict, List, Optional
+
+from karaoke_gen.pipeline.base import PipelineExecutor, PipelineStage, StageResult, StageStatus
+from karaoke_gen.pipeline.context import PipelineContext
+
+
+class RemoteExecutor(PipelineExecutor):
+    """
+    Runs pipeline stages via backend API.
+    
+    This executor is used by the remote CLI (karaoke-gen-remote) to
+    submit jobs to the cloud backend and monitor their progress.
+    
+    Note: The current implementation is a compatibility layer.
+    The actual processing is handled by the existing backend workers,
+    not by executing PipelineStage instances remotely.
+    """
+    
+    def __init__(
+        self,
+        service_url: str,
+        auth_token: Optional[str] = None,
+        logger: logging.Logger = None,
+        poll_interval: int = 5,
+    ):
+        """
+        Initialize the remote executor.
+        
+        Args:
+            service_url: Backend service URL
+            auth_token: Authentication token
+            logger: Logger instance
+            poll_interval: Seconds between status polls
+        """
+        self.service_url = service_url.rstrip('/')
+        self.auth_token = auth_token
+        self.logger = logger or logging.getLogger(__name__)
+        self.poll_interval = poll_interval
+        self._session = None
+    
+    @property
+    def session(self):
+        """Get or create HTTP session."""
+        if self._session is None:
+            import requests
+            self._session = requests.Session()
+            if self.auth_token:
+                self._session.headers['Authorization'] = f'Bearer {self.auth_token}'
+        return self._session
+    
+    async def run_stage(
+        self,
+        stage: PipelineStage,
+        context: PipelineContext,
+    ) -> StageResult:
+        """
+        Execute a single pipeline stage via backend.
+        
+        Note: This is a placeholder. The backend handles stage
+        execution internally through its worker system.
+        
+        Args:
+            stage: The stage to execute
+            context: Pipeline context
+            
+        Returns:
+            Result of stage execution
+        """
+        self.logger.warning(
+            f"RemoteExecutor.run_stage called for {stage.name}. "
+            "Remote execution is handled by backend workers, not via this interface."
+        )
+        
+        return StageResult(
+            status=StageStatus.SKIPPED,
+            error_message="Remote execution handled by backend workers",
+        )
+    
+    async def run_pipeline(
+        self,
+        stages: List[PipelineStage],
+        context: PipelineContext,
+    ) -> Dict[str, StageResult]:
+        """
+        Execute a full pipeline via backend.
+        
+        This submits a job to the backend and monitors its progress.
+        The backend handles individual stage execution through its
+        worker system.
+        
+        Args:
+            stages: List of stages (used for validation only)
+            context: Pipeline context with job parameters
+            
+        Returns:
+            Dictionary mapping stage names to their results
+        """
+        results: Dict[str, StageResult] = {}
+        
+        try:
+            # Submit job to backend
+            job_id = await self._submit_job(context)
+            context.log("INFO", f"Job submitted: {job_id}")
+            
+            # Monitor job progress
+            final_status = await self._monitor_job(job_id, context)
+            
+            # Build results based on final status
+            if final_status == "complete":
+                # Mark all stages as completed
+                for stage in stages:
+                    results[stage.name] = StageResult(status=StageStatus.COMPLETED)
+            else:
+                # Mark stages based on where failure occurred
+                for stage in stages:
+                    results[stage.name] = StageResult(
+                        status=StageStatus.FAILED,
+                        error_message=f"Job ended with status: {final_status}",
+                    )
+            
+            return results
+            
+        except Exception as e:
+            self.logger.error(f"Remote pipeline execution failed: {e}")
+            for stage in stages:
+                results[stage.name] = StageResult(
+                    status=StageStatus.FAILED,
+                    error_message=str(e),
+                )
+            return results
+    
+    async def _submit_job(self, context: PipelineContext) -> str:
+        """
+        Submit a job to the backend.
+        
+        Args:
+            context: Pipeline context with job parameters
+            
+        Returns:
+            Job ID
+        """
+        import os
+        
+        # Build form data
+        data = {
+            'artist': context.artist,
+            'title': context.title,
+            'enable_cdg': str(context.enable_cdg).lower(),
+            'enable_txt': str(context.enable_txt).lower(),
+        }
+        
+        if context.brand_prefix:
+            data['brand_prefix'] = context.brand_prefix
+        if context.discord_webhook_url:
+            data['discord_webhook_url'] = context.discord_webhook_url
+        if context.enable_youtube_upload:
+            data['enable_youtube_upload'] = str(context.enable_youtube_upload).lower()
+        if context.dropbox_path:
+            data['dropbox_path'] = context.dropbox_path
+        if context.gdrive_folder_id:
+            data['gdrive_folder_id'] = context.gdrive_folder_id
+        
+        # Upload audio file
+        files = {}
+        if os.path.isfile(context.input_audio_path):
+            files['file'] = (
+                os.path.basename(context.input_audio_path),
+                open(context.input_audio_path, 'rb'),
+            )
+        
+        try:
+            response = self.session.post(
+                f"{self.service_url}/api/jobs/upload",
+                data=data,
+                files=files,
+            )
+            response.raise_for_status()
+            result = response.json()
+            return result['job_id']
+        finally:
+            # Close file handles
+            for name, (filename, fh) in files.items():
+                fh.close()
+    
+    async def _monitor_job(self, job_id: str, context: PipelineContext) -> str:
+        """
+        Monitor job progress until completion.
+        
+        Args:
+            job_id: Job ID to monitor
+            context: Pipeline context for progress updates
+            
+        Returns:
+            Final job status
+        """
+        import asyncio
+        
+        while True:
+            try:
+                response = self.session.get(f"{self.service_url}/api/jobs/{job_id}")
+                response.raise_for_status()
+                job_data = response.json()
+                
+                status = job_data.get('status', 'unknown')
+                progress = job_data.get('progress', 0)
+                
+                context.update_progress(status, progress, f"Status: {status}")
+                
+                # Check for terminal states
+                if status in ['complete', 'failed', 'cancelled', 'error']:
+                    return status
+                
+                await asyncio.sleep(self.poll_interval)
+                
+            except Exception as e:
+                self.logger.warning(f"Error polling job status: {e}")
+                await asyncio.sleep(self.poll_interval)
+    
+    def get_job_status(self, job_id: str) -> Dict[str, Any]:
+        """
+        Get current job status.
+        
+        Args:
+            job_id: Job ID
+            
+        Returns:
+            Job status data
+        """
+        response = self.session.get(f"{self.service_url}/api/jobs/{job_id}")
+        response.raise_for_status()
+        return response.json()
+
+
+def create_remote_executor(
+    service_url: str,
+    auth_token: Optional[str] = None,
+    logger: logging.Logger = None,
+) -> RemoteExecutor:
+    """Factory function to create a RemoteExecutor."""
+    return RemoteExecutor(
+        service_url=service_url,
+        auth_token=auth_token,
+        logger=logger,
+    )

karaoke_gen/pipeline/stages/__init__.py

@@ -0,0 +1,27 @@
+"""
+Pipeline stages for karaoke generation.
+
+Each stage represents a discrete unit of work in the karaoke generation
+process. Stages are designed to be composable and can be executed either
+locally or remotely.
+
+Available stages:
+- SeparationStage: Audio separation into stems
+- TranscriptionStage: Lyrics transcription and synchronization
+- ScreensStage: Title and end screen generation
+- RenderStage: Video rendering with synchronized lyrics
+- FinalizeStage: Encoding, packaging, and distribution
+"""
+from karaoke_gen.pipeline.stages.separation import SeparationStage
+from karaoke_gen.pipeline.stages.transcription import TranscriptionStage
+from karaoke_gen.pipeline.stages.screens import ScreensStage
+from karaoke_gen.pipeline.stages.render import RenderStage
+from karaoke_gen.pipeline.stages.finalize import FinalizeStage
+
+__all__ = [
+    "SeparationStage",
+    "TranscriptionStage",
+    "ScreensStage",
+    "RenderStage",
+    "FinalizeStage",
+]

karaoke_gen/pipeline/stages/finalize.py

@@ -0,0 +1,202 @@
+"""
+Finalization pipeline stage.
+
+This stage handles the final processing:
+- Multi-format video encoding (4K lossless, 4K lossy, 720p)
+- CDG/TXT package generation
+- Distribution (YouTube, Dropbox, Google Drive)
+- Discord notifications
+
+This wraps the existing KaraokeFinalise module.
+"""
+import logging
+import os
+from typing import Any, Dict, List, Optional
+
+from karaoke_gen.pipeline.base import PipelineStage, StageResult, StageStatus
+from karaoke_gen.pipeline.context import PipelineContext
+
+
+class FinalizeStage(PipelineStage):
+    """
+    Finalization stage.
+    
+    Handles encoding, packaging, and distribution of the final
+    karaoke video and related files.
+    """
+    
+    def __init__(
+        self,
+        enable_cdg: bool = True,
+        enable_txt: bool = True,
+        non_interactive: bool = False,
+        server_side_mode: bool = False,
+        youtube_credentials: Optional[Dict[str, Any]] = None,
+        youtube_description_template: Optional[str] = None,
+        logger: Optional[logging.Logger] = None,
+    ):
+        """
+        Initialize the finalize stage.
+        
+        Args:
+            enable_cdg: Generate CDG+MP3 package
+            enable_txt: Generate TXT+MP3 package
+            non_interactive: Run without user prompts
+            server_side_mode: Running on server (disables local-only features)
+            youtube_credentials: Pre-loaded YouTube OAuth credentials
+            youtube_description_template: YouTube video description template
+            logger: Logger instance
+        """
+        self.enable_cdg = enable_cdg
+        self.enable_txt = enable_txt
+        self.non_interactive = non_interactive
+        self.server_side_mode = server_side_mode
+        self.youtube_credentials = youtube_credentials
+        self.youtube_description_template = youtube_description_template
+        self.logger = logger or logging.getLogger(__name__)
+    
+    @property
+    def name(self) -> str:
+        return "finalize"
+    
+    @property
+    def required_inputs(self) -> List[str]:
+        # Requires render output (with_vocals video)
+        return ["render", "screens"]
+    
+    @property
+    def optional_inputs(self) -> List[str]:
+        return ["separation"]
+    
+    @property
+    def output_keys(self) -> List[str]:
+        return [
+            "final_video_lossless_4k_mp4",
+            "final_video_lossless_4k_mkv",
+            "final_video_lossy_4k_mp4",
+            "final_video_lossy_720p_mp4",
+            "cdg_zip_path",
+            "txt_zip_path",
+            "brand_code",
+            "youtube_url",
+            "dropbox_link",
+        ]
+    
+    async def execute(self, context: PipelineContext) -> StageResult:
+        """
+        Execute finalization.
+        
+        Args:
+            context: Pipeline context with render outputs
+            
+        Returns:
+            StageResult with final file paths
+        """
+        import time
+        start_time = time.time()
+        
+        try:
+            context.update_progress(self.name, 0, "Starting finalization")
+            context.log("INFO", f"Finalizing: {context.artist} - {context.title}")
+            
+            from karaoke_gen.karaoke_finalise.karaoke_finalise import KaraokeFinalise
+            
+            # Get the selected instrumental path
+            separation = context.stage_outputs.get("separation", {})
+            
+            # Prefer combined instrumental with backing if available
+            instrumental_path = None
+            combined = separation.get("combined_instrumentals", {})
+            if combined:
+                # Get first combined instrumental
+                instrumental_path = list(combined.values())[0]
+            elif separation.get("clean_instrumental", {}).get("instrumental"):
+                instrumental_path = separation["clean_instrumental"]["instrumental"]
+            
+            if not instrumental_path or not os.path.exists(instrumental_path):
+                return StageResult(
+                    status=StageStatus.FAILED,
+                    error_message="No instrumental audio file found",
+                )
+            
+            # Build CDG styles from context style_params
+            cdg_styles = None
+            if context.style_params and context.style_params.get("cdg"):
+                cdg_styles = context.style_params["cdg"]
+            
+            context.update_progress(self.name, 10, "Initializing KaraokeFinalise")
+            
+            # Create KaraokeFinalise instance
+            finalise = KaraokeFinalise(
+                logger=self.logger,
+                log_level=logging.INFO,
+                dry_run=False,
+                instrumental_format="flac",
+                enable_cdg=self.enable_cdg or context.enable_cdg,
+                enable_txt=self.enable_txt or context.enable_txt,
+                cdg_styles=cdg_styles,
+                brand_prefix=context.brand_prefix,
+                organised_dir=None,  # Not used directly
+                organised_dir_rclone_root=context.organised_dir_rclone_root,
+                public_share_dir=None,
+                discord_webhook_url=context.discord_webhook_url,
+                youtube_client_secrets_file=None,
+                youtube_description_file=None,  # Use template instead
+                user_youtube_credentials=self.youtube_credentials,
+                rclone_destination=context.rclone_destination,
+                email_template_file=None,
+                non_interactive=self.non_interactive,
+                server_side_mode=self.server_side_mode,
+                selected_instrumental_file=instrumental_path,
+            )
+            
+            # Change to output directory for KaraokeFinalise
+            original_cwd = os.getcwd()
+            try:
+                os.chdir(context.output_dir)
+                
+                context.update_progress(self.name, 20, "Processing finalization")
+                
+                # Run finalization
+                result = finalise.process()
+                
+            finally:
+                os.chdir(original_cwd)
+            
+            # Build outputs from result
+            outputs = {
+                "final_video_lossless_4k_mp4": result.get("final_video"),
+                "final_video_lossless_4k_mkv": result.get("final_video_mkv"),
+                "final_video_lossy_4k_mp4": result.get("final_video_lossy"),
+                "final_video_lossy_720p_mp4": result.get("final_video_720p"),
+                "cdg_zip_path": result.get("final_karaoke_cdg_zip"),
+                "txt_zip_path": result.get("final_karaoke_txt_zip"),
+                "brand_code": result.get("brand_code"),
+                "youtube_url": result.get("youtube_url"),
+            }
+            
+            context.update_progress(self.name, 100, "Finalization complete")
+            
+            duration = time.time() - start_time
+            context.log("INFO", f"Finalization completed in {duration:.1f}s")
+            
+            if outputs.get("brand_code"):
+                context.log("INFO", f"Brand code: {outputs['brand_code']}")
+            if outputs.get("youtube_url"):
+                context.log("INFO", f"YouTube URL: {outputs['youtube_url']}")
+            
+            return StageResult(
+                status=StageStatus.COMPLETED,
+                outputs=outputs,
+                duration_seconds=duration,
+            )
+            
+        except Exception as e:
+            duration = time.time() - start_time
+            context.log("ERROR", f"Finalization failed: {str(e)}")
+            return StageResult(
+                status=StageStatus.FAILED,
+                error_message=str(e),
+                error_details={"exception_type": type(e).__name__},
+                duration_seconds=duration,
+            )