karaoke-gen 0.86.7__py3-none-any.whl → 0.96.0__py3-none-any.whl

backend/workers/render_video_worker.py

@@ -0,0 +1,483 @@
+"""
+Render Video Worker
+
+Generates the karaoke video with synchronized lyrics AFTER human review.
+
+This worker:
+1. Downloads the corrected lyrics data from GCS
+2. Downloads the audio file
+3. Downloads style assets from GCS
+4. Uses LyricsTranscriber's OutputGenerator to render video
+5. Uploads the with_vocals.mkv to GCS
+6. Transitions to AWAITING_INSTRUMENTAL_SELECTION
+
+Key insight: We use OutputGenerator from lyrics_transcriber library
+WITHOUT using its blocking ReviewServer. This allows async operation
+in Cloud Run.
+
+Observability:
+- All operations wrapped in tracing spans for Cloud Trace visibility
+- Logs include [job:ID] prefix for easy filtering in Cloud Logging
+- Worker start/end timing logged with WORKER_START/WORKER_END markers
+"""
+import logging
+import os
+import tempfile
+import time
+import json
+from typing import Optional, Dict, Any
+from dataclasses import dataclass
+
+from backend.models.job import JobStatus
+from backend.services.job_manager import JobManager
+from backend.services.storage_service import StorageService
+from backend.config import get_settings
+from backend.workers.worker_logging import create_job_logger, setup_job_logging, job_logging_context
+from backend.services.tracing import job_span, add_span_event, add_span_attribute
+
+# Import from lyrics_transcriber (submodule)
+from lyrics_transcriber.output.generator import OutputGenerator
+from lyrics_transcriber.output.countdown_processor import CountdownProcessor
+from lyrics_transcriber.types import CorrectionResult
+from lyrics_transcriber.core.config import OutputConfig
+
+# Import from the unified style loader
+from karaoke_gen.style_loader import load_styles_from_gcs
+from karaoke_gen.utils import sanitize_filename
+
+
+logger = logging.getLogger(__name__)
+
+
+# Loggers to capture for render video worker
+RENDER_VIDEO_WORKER_LOGGERS = [
+    "lyrics_transcriber.output",
+    "lyrics_transcriber.output.generator",
+    "lyrics_transcriber.output.video",
+    "lyrics_transcriber.output.ass",
+]
+
+
+async def process_render_video(job_id: str) -> bool:
+    """
+    Render karaoke video with corrected lyrics.
+    
+    Called after human review is complete (REVIEW_COMPLETE state).
+    Uses OutputGenerator from lyrics_transcriber to generate the video.
+    
+    Args:
+        job_id: Job ID to process
+        
+    Returns:
+        True if successful, False otherwise
+    """
+    start_time = time.time()
+    job_manager = JobManager()
+    storage = StorageService()
+    settings = get_settings()
+    
+    # Create job logger for remote debugging FIRST
+    job_log = create_job_logger(job_id, "render_video")
+    
+    # Log with structured markers for easy Cloud Logging queries
+    logger.info(f"[job:{job_id}] WORKER_START worker=render-video")
+    job_log.info("=== RENDER VIDEO WORKER STARTED ===")
+    job_log.info(f"Job ID: {job_id}")
+    
+    # Set up log capture for OutputGenerator
+    log_handler = setup_job_logging(job_id, "render_video", *RENDER_VIDEO_WORKER_LOGGERS)
+    job_log.info(f"Log handler attached for {len(RENDER_VIDEO_WORKER_LOGGERS)} loggers")
+    
+    job = job_manager.get_job(job_id)
+    if not job:
+        logger.error(f"[job:{job_id}] Job not found in Firestore")
+        job_log.error(f"Job {job_id} not found in Firestore!")
+        return False
+    
+    job_log.info(f"Starting video render for {job.artist} - {job.title}")
+    logger.info(f"[job:{job_id}] Starting video render (post-review)")
+    
+    try:
+        # Wrap entire worker in a tracing span
+        with job_span("render-video-worker", job_id, {"artist": job.artist, "title": job.title}) as root_span:
+            # Use job_logging_context for proper log isolation when multiple jobs run concurrently
+            # This ensures logs from third-party libraries (lyrics_transcriber.output) are only
+            # captured by this job's handler, not handlers from other concurrent jobs
+            with job_logging_context(job_id):
+                # Transition to RENDERING_VIDEO
+                job_manager.transition_to_state(
+                    job_id=job_id,
+                    new_status=JobStatus.RENDERING_VIDEO,
+                    progress=75,
+                    message="Rendering karaoke video with corrected lyrics"
+                )
+                
+                with tempfile.TemporaryDirectory() as temp_dir:
+                    job_log.info(f"Created temp directory: {temp_dir}")
+                    
+                    # Try updated corrections first (from human review), fall back to original
+                    corrections_gcs_updated = f"jobs/{job_id}/lyrics/corrections_updated.json"
+                    corrections_gcs_original = f"jobs/{job_id}/lyrics/corrections.json"
+                    
+                    # Get GCS path from file_urls
+                    corrections_url = job.file_urls.get('lyrics', {}).get('corrections_updated')
+                    if not corrections_url:
+                        corrections_url = job.file_urls.get('lyrics', {}).get('corrections')
+                    
+                    if not corrections_url:
+                        # Try direct GCS paths
+                        if storage.file_exists(corrections_gcs_updated):
+                            corrections_gcs = corrections_gcs_updated
+                        elif storage.file_exists(corrections_gcs_original):
+                            corrections_gcs = corrections_gcs_original
+                        else:
+                            raise FileNotFoundError(f"No corrections file found for job {job_id}")
+                    else:
+                        # Extract GCS path from URL
+                        corrections_gcs = _extract_gcs_path(corrections_url)
+                    
+                    # 2. Load the ORIGINAL corrections (has full structure)
+                    original_corrections_gcs = f"jobs/{job_id}/lyrics/corrections.json"
+                    original_corrections_path = os.path.join(temp_dir, "corrections_original.json")
+                    
+                    job_log.info(f"Downloading original corrections from {original_corrections_gcs}")
+                    logger.info(f"Job {job_id}: Downloading original corrections from {original_corrections_gcs}")
+                    storage.download_file(original_corrections_gcs, original_corrections_path)
+                    
+                    with open(original_corrections_path, 'r', encoding='utf-8') as f:
+                        original_data = json.load(f)
+                    
+                    # 3. Check if there are updated corrections (from review UI)
+                    # The frontend sends only partial data: {corrections, corrected_segments}
+                    updated_corrections_gcs = f"jobs/{job_id}/lyrics/corrections_updated.json"
+                    
+                    if storage.file_exists(updated_corrections_gcs):
+                        job_log.info("Found updated corrections from review, merging")
+                        logger.info(f"Job {job_id}: Found updated corrections, merging")
+                        updated_path = os.path.join(temp_dir, "corrections_updated.json")
+                        storage.download_file(updated_corrections_gcs, updated_path)
+                        
+                        with open(updated_path, 'r', encoding='utf-8') as f:
+                            updated_data = json.load(f)
+                        
+                        # Merge: update the original with the user's corrections
+                        if 'corrections' in updated_data:
+                            original_data['corrections'] = updated_data['corrections']
+                        if 'corrected_segments' in updated_data:
+                            original_data['corrected_segments'] = updated_data['corrected_segments']
+                        
+                        job_log.info("Merged user corrections into original data")
+                        logger.info(f"Job {job_id}: Merged user corrections into original data")
+                    
+                    # 4. Convert to CorrectionResult
+                    correction_result = CorrectionResult.from_dict(original_data)
+                    job_log.info(f"Loaded CorrectionResult with {len(correction_result.corrected_segments)} segments")
+                    logger.info(f"Job {job_id}: Loaded CorrectionResult with {len(correction_result.corrected_segments)} segments")
+                    
+                    # 5. Download audio file
+                    audio_path = os.path.join(temp_dir, "audio.flac")
+                    audio_gcs_path = job.input_media_gcs_path
+                    
+                    if not audio_gcs_path:
+                        raise FileNotFoundError(f"No input audio path for job {job_id}")
+                    
+                    job_log.info(f"Downloading audio from {audio_gcs_path}")
+                    logger.info(f"Job {job_id}: Downloading audio from {audio_gcs_path}")
+                    storage.download_file(audio_gcs_path, audio_path)
+                    job_log.info(f"Audio downloaded: {os.path.getsize(audio_path)} bytes")
+                    
+                    # Process countdown intro if needed (for songs that start too quickly)
+                    # This adds "3... 2... 1..." segment and pads audio with 3s silence
+                    # Note: Countdown is deferred to this stage (not during lyrics transcription)
+                    # so the review UI can show the original timing without the 3s shift
+                    countdown_processor = CountdownProcessor(cache_dir=temp_dir, logger=logger)
+                    correction_result, audio_path, padding_added, padding_seconds = countdown_processor.process(
+                        correction_result=correction_result,
+                        audio_filepath=audio_path,
+                    )
+                    if padding_added:
+                        job_log.info(f"Countdown added: {padding_seconds}s padding applied to audio and timestamps shifted")
+                    else:
+                        job_log.info("No countdown needed - song starts after 3 seconds")
+                    
+                    # 6. Get or create styles using the unified style loader
+                    job_log.info("Loading style configuration...")
+                    job_log.info(f"  job.style_params_gcs_path: {job.style_params_gcs_path}")
+                    job_log.info(f"  job.style_assets: {list(job.style_assets.keys()) if job.style_assets else 'None'}")
+                    
+                    styles_path, style_data = load_styles_from_gcs(
+                        style_params_gcs_path=job.style_params_gcs_path,
+                        style_assets=job.style_assets,
+                        temp_dir=temp_dir,
+                        download_func=storage.download_file,
+                        logger=job_log,
+                    )
+                    
+                    # 7. Configure OutputGenerator
+                    output_dir = os.path.join(temp_dir, "output")
+                    cache_dir = os.path.join(temp_dir, "cache")
+                    os.makedirs(output_dir, exist_ok=True)
+                    os.makedirs(cache_dir, exist_ok=True)
+                    
+                    job_log.info(f"Using styles from: {styles_path}")
+                    
+                    # Get subtitle offset from job (user-specified timing adjustment)
+                    subtitle_offset = getattr(job, 'subtitle_offset_ms', 0) or 0
+                    if subtitle_offset != 0:
+                        job_log.info(f"Applying subtitle offset: {subtitle_offset}ms")
+                    
+                    config = OutputConfig(
+                        output_dir=output_dir,
+                        cache_dir=cache_dir,
+                        output_styles_json=styles_path,
+                        render_video=True,
+                        generate_cdg=False,  # CDG optional, generated separately
+                        generate_plain_text=True,
+                        generate_lrc=True,
+                        video_resolution="4k",
+                        subtitle_offset_ms=subtitle_offset
+                    )
+                    
+                    job_log.info(f"OutputConfig: output_styles_json={config.output_styles_json}, render_video={config.render_video}")
+                    
+                    output_generator = OutputGenerator(config, logger)
+                    
+                    # 8. Generate outputs (video, LRC, ASS, etc.)
+                    # Sanitize artist/title to handle Unicode characters (curly quotes, em dashes, etc.)
+                    safe_artist = sanitize_filename(job.artist) if job.artist else "Unknown"
+                    safe_title = sanitize_filename(job.title) if job.title else "Unknown"
+                    output_prefix = f"{safe_artist} - {safe_title}"
+                    job_log.info(f"Generating outputs with prefix '{output_prefix}'")
+                    logger.info(f"Job {job_id}: Generating outputs with prefix '{output_prefix}'")
+                    
+                    outputs = output_generator.generate_outputs(
+                        transcription_corrected=correction_result,
+                        lyrics_results={},  # Reference lyrics already in correction_result
+                        output_prefix=output_prefix,
+                        audio_filepath=audio_path,
+                        artist=job.artist,
+                        title=job.title
+                    )
+                    
+                    # 9. Upload video to GCS
+                    if outputs.video and os.path.exists(outputs.video):
+                        video_size = os.path.getsize(outputs.video)
+                        job_log.info(f"Video generated: {video_size} bytes")
+                        video_gcs_path = f"jobs/{job_id}/videos/with_vocals.mkv"
+                        video_url = storage.upload_file(outputs.video, video_gcs_path)
+                        job_manager.update_file_url(job_id, 'videos', 'with_vocals', video_url)
+                        job_log.info(f"Uploaded with_vocals.mkv to GCS")
+                        logger.info(f"Job {job_id}: Uploaded with_vocals.mkv ({video_size} bytes)")
+                    else:
+                        job_log.error("Video generation failed - no output file produced!")
+                        raise Exception("Video generation failed - no output file produced")
+                    
+                    # 10. Upload LRC file
+                    if outputs.lrc and os.path.exists(outputs.lrc):
+                        lrc_gcs_path = f"jobs/{job_id}/lyrics/karaoke.lrc"
+                        lrc_url = storage.upload_file(outputs.lrc, lrc_gcs_path)
+                        job_manager.update_file_url(job_id, 'lyrics', 'lrc', lrc_url)
+                        job_log.info("Uploaded karaoke.lrc")
+                        logger.info(f"Job {job_id}: Uploaded karaoke.lrc")
+                    
+                    # 11. Upload ASS subtitle file
+                    if outputs.ass and os.path.exists(outputs.ass):
+                        ass_gcs_path = f"jobs/{job_id}/lyrics/karaoke.ass"
+                        ass_url = storage.upload_file(outputs.ass, ass_gcs_path)
+                        job_manager.update_file_url(job_id, 'lyrics', 'ass', ass_url)
+                        job_log.info("Uploaded karaoke.ass")
+                        logger.info(f"Job {job_id}: Uploaded karaoke.ass")
+                    
+                    # 12. Upload corrected text files
+                    if outputs.corrected_txt and os.path.exists(outputs.corrected_txt):
+                        txt_gcs_path = f"jobs/{job_id}/lyrics/corrected.txt"
+                        txt_url = storage.upload_file(outputs.corrected_txt, txt_gcs_path)
+                        job_manager.update_file_url(job_id, 'lyrics', 'corrected_txt', txt_url)
+                        job_log.info("Uploaded corrected.txt")
+                        logger.info(f"Job {job_id}: Uploaded corrected.txt")
+                    
+                    # 13. Analyze backing vocals for intelligent instrumental selection
+                    job_log.info("Analyzing backing vocals for instrumental selection...")
+                    await _analyze_backing_vocals(job_id, job_manager, storage, job_log)
+                    
+                    # 14. Transition based on prep_only flag and existing instrumental
+                    if getattr(job, 'prep_only', False):
+                        # Prep-only mode: stop here and mark as prep complete
+                        job_manager.transition_to_state(
+                            job_id=job_id,
+                            new_status=JobStatus.PREP_COMPLETE,
+                            progress=100,
+                            message="Prep phase complete - download outputs to continue locally"
+                        )
+                        job_log.info("=== RENDER VIDEO WORKER COMPLETE (PREP ONLY) ===")
+                        duration = time.time() - start_time
+                        root_span.set_attribute("duration_seconds", duration)
+                        root_span.set_attribute("prep_only", True)
+                        logger.info(f"[job:{job_id}] WORKER_END worker=render-video status=success duration={duration:.1f}s prep_only=true")
+                    elif getattr(job, 'existing_instrumental_gcs_path', None):
+                        # Existing instrumental provided - skip selection, auto-use it
+                        # Store selection as 'custom' to indicate user-provided instrumental
+                        job_manager.update_state_data(job_id, 'instrumental_selection', 'custom')
+                        job_log.info("Existing instrumental provided - skipping selection, using user-provided file")
+                        
+                        job_manager.transition_to_state(
+                            job_id=job_id,
+                            new_status=JobStatus.INSTRUMENTAL_SELECTED,
+                            progress=82,
+                            message="Using user-provided instrumental"
+                        )
+                        job_log.info("=== RENDER VIDEO WORKER COMPLETE (EXISTING INSTRUMENTAL) ===")
+                        duration = time.time() - start_time
+                        root_span.set_attribute("duration_seconds", duration)
+                        root_span.set_attribute("existing_instrumental", True)
+                        logger.info(f"[job:{job_id}] WORKER_END worker=render-video status=success duration={duration:.1f}s existing_instrumental=true")
+                        
+                        # Trigger video worker directly since no user selection needed
+                        from backend.workers.video_worker import run_video_worker
+                        job_log.info("Triggering video worker for final encoding...")
+                        await run_video_worker(job_id, job_manager, storage)
+                    else:
+                        # Normal mode: proceed to instrumental selection
+                        job_manager.transition_to_state(
+                            job_id=job_id,
+                            new_status=JobStatus.AWAITING_INSTRUMENTAL_SELECTION,
+                            progress=80,
+                            message="Video rendered - select your instrumental"
+                        )
+                        job_log.info("=== RENDER VIDEO WORKER COMPLETE ===")
+                        duration = time.time() - start_time
+                        root_span.set_attribute("duration_seconds", duration)
+                        logger.info(f"[job:{job_id}] WORKER_END worker=render-video status=success duration={duration:.1f}s")
+                    return True
+            
+    except Exception as e:
+        duration = time.time() - start_time
+        job_log.error(f"Video render failed: {e}")
+        logger.error(f"[job:{job_id}] WORKER_END worker=render-video status=error duration={duration:.1f}s error={e}")
+        job_manager.fail_job(job_id, f"Video render failed: {str(e)}")
+        return False
+
+
+def _extract_gcs_path(url: str) -> str:
+    """Extract GCS path from a URL or return as-is if already a path."""
+    if url.startswith('gs://'):
+        # Already a GCS path
+        return url.replace('gs://', '').split('/', 1)[1] if '/' in url else url
+    if url.startswith('https://storage.googleapis.com/'):
+        # Signed URL - extract path
+        path = url.replace('https://storage.googleapis.com/', '')
+        # Remove query params
+        if '?' in path:
+            path = path.split('?')[0]
+        # Skip bucket name
+        parts = path.split('/', 1)
+        return parts[1] if len(parts) > 1 else path
+    return url
+
+
+async def _analyze_backing_vocals(
+    job_id: str,
+    job_manager: JobManager,
+    storage: StorageService,
+    job_log: logging.Logger,
+) -> None:
+    """
+    Analyze backing vocals to help with intelligent instrumental selection.
+    
+    This function:
+    1. Downloads the backing vocals stem from GCS
+    2. Runs audio analysis to detect audible content
+    3. Generates a waveform visualization image
+    4. Stores analysis results and waveform URL in job state
+    
+    The analysis data is then used by the frontend to provide an
+    intelligent instrumental selection experience.
+    """
+    from backend.services.audio_analysis_service import AudioAnalysisService
+    
+    try:
+        # Get the job to access file URLs
+        job = job_manager.get_job(job_id)
+        if not job:
+            job_log.warning(f"Could not get job {job_id} for backing vocals analysis")
+            return
+        
+        # Get backing vocals path
+        backing_vocals_path = job.file_urls.get('stems', {}).get('backing_vocals')
+        if not backing_vocals_path:
+            job_log.warning("No backing vocals file found - skipping analysis")
+            return
+        
+        job_log.info(f"Analyzing backing vocals: {backing_vocals_path}")
+        
+        # Create analysis service and run analysis
+        analysis_service = AudioAnalysisService()
+        
+        # Define output path for waveform
+        waveform_gcs_path = f"jobs/{job_id}/analysis/backing_vocals_waveform.png"
+        
+        # Run analysis and generate waveform
+        result, waveform_path = analysis_service.analyze_and_generate_waveform(
+            gcs_audio_path=backing_vocals_path,
+            job_id=job_id,
+            gcs_waveform_destination=waveform_gcs_path,
+        )
+        
+        # Store analysis results in job state_data
+        analysis_data = {
+            'has_audible_content': result.has_audible_content,
+            'total_duration_seconds': result.total_duration_seconds,
+            'audible_segments': [
+                {
+                    'start_seconds': seg.start_seconds,
+                    'end_seconds': seg.end_seconds,
+                    'duration_seconds': seg.duration_seconds,
+                    'avg_amplitude_db': seg.avg_amplitude_db,
+                    'peak_amplitude_db': seg.peak_amplitude_db,
+                }
+                for seg in result.audible_segments
+            ],
+            'recommended_selection': result.recommended_selection.value,
+            'total_audible_duration_seconds': result.total_audible_duration_seconds,
+            'audible_percentage': result.audible_percentage,
+            'silence_threshold_db': result.silence_threshold_db,
+        }
+        
+        job_manager.update_state_data(job_id, 'backing_vocals_analysis', analysis_data)
+        
+        # Store waveform URL
+        job_manager.update_file_url(job_id, 'analysis', 'backing_vocals_waveform', waveform_path)
+        
+        job_log.info(
+            f"Backing vocals analysis complete: "
+            f"has_audible={result.has_audible_content}, "
+            f"segments={result.segment_count}, "
+            f"recommendation={result.recommended_selection.value}"
+        )
+        
+        # Log segment details if there are any
+        if result.audible_segments:
+            job_log.info(f"Audible segments ({len(result.audible_segments)}):")
+            for i, seg in enumerate(result.audible_segments[:5]):  # Log first 5
+                job_log.info(
+                    f"  [{i+1}] {seg.start_seconds:.1f}s - {seg.end_seconds:.1f}s "
+                    f"({seg.duration_seconds:.1f}s, avg: {seg.avg_amplitude_db:.1f}dB)"
+                )
+            if len(result.audible_segments) > 5:
+                job_log.info(f"  ... and {len(result.audible_segments) - 5} more")
+        
+    except Exception as e:
+        # Log the error but don't fail the job - analysis is a nice-to-have
+        job_log.warning(f"Backing vocals analysis failed (non-fatal): {e}")
+        logger.warning(f"Job {job_id}: Backing vocals analysis failed: {e}")
+        
+        # Store empty analysis so the frontend knows analysis was attempted
+        job_manager.update_state_data(job_id, 'backing_vocals_analysis', {
+            'has_audible_content': None,
+            'analysis_error': str(e),
+            'recommended_selection': 'review_needed',
+        })
+
+
+# For compatibility with worker service
+render_video_worker = process_render_video