karaoke-gen 0.90.1__py3-none-any.whl → 0.96.0__py3-none-any.whl

backend/api/routes/internal.py

@@ -0,0 +1,435 @@
+"""
+Internal API routes for worker coordination.
+
+These endpoints are for internal use only (backend → workers).
+They are protected by admin authentication.
+
+With Cloud Tasks integration, these endpoints may be called multiple times
+(retry on failure). Idempotency checks prevent duplicate processing.
+
+Observability:
+- Extracts trace context from incoming requests (propagated via Cloud Tasks)
+- Creates worker spans linked to the original request trace
+- All logs include job_id for easy filtering in Cloud Logging
+"""
+import logging
+import asyncio
+import time
+from typing import Tuple, Optional
+from fastapi import APIRouter, HTTPException, BackgroundTasks, Depends, Request
+from pydantic import BaseModel
+
+from backend.workers.audio_worker import process_audio_separation
+from backend.workers.lyrics_worker import process_lyrics_transcription
+from backend.workers.screens_worker import generate_screens
+from backend.workers.video_worker import generate_video
+from backend.workers.render_video_worker import process_render_video
+from backend.api.dependencies import require_admin
+from backend.services.auth_service import UserType
+from backend.services.job_manager import JobManager
+from backend.services.tracing import (
+    extract_trace_context,
+    start_span_with_context,
+    add_span_attribute,
+    add_span_event,
+)
+
+
+logger = logging.getLogger(__name__)
+router = APIRouter(prefix="/internal", tags=["internal"])
+
+
+class WorkerRequest(BaseModel):
+    """Request to trigger a worker."""
+    job_id: str
+
+
+class WorkerResponse(BaseModel):
+    """Response from worker trigger."""
+    status: str
+    job_id: str
+    message: str
+
+
+def _check_worker_idempotency(job_id: str, worker_name: str) -> Optional[WorkerResponse]:
+    """
+    Check if a worker is already running or completed for this job.
+    
+    This provides idempotency for Cloud Tasks retries - if a task is retried
+    but the worker is already running or has completed, we skip processing.
+    
+    Args:
+        job_id: Job ID to check
+        worker_name: Worker name (audio, lyrics, screens, render, video)
+        
+    Returns:
+        WorkerResponse if should skip (already running/complete), None to proceed
+    """
+    job_manager = JobManager()
+    job = job_manager.get_job(job_id)
+    
+    if not job:
+        logger.warning(f"[job:{job_id}] Job not found for {worker_name} worker")
+        return WorkerResponse(
+            status="not_found",
+            job_id=job_id,
+            message=f"Job {job_id} not found"
+        )
+    
+    # Check worker-specific progress in state_data
+    progress_key = f"{worker_name}_progress"
+    worker_progress = job.state_data.get(progress_key, {})
+    stage = worker_progress.get('stage')
+    
+    if stage == 'running':
+        logger.info(f"[job:{job_id}] {worker_name.capitalize()} worker already running, skipping")
+        return WorkerResponse(
+            status="already_running",
+            job_id=job_id,
+            message=f"{worker_name.capitalize()} worker already in progress"
+        )
+    
+    if stage == 'complete':
+        logger.info(f"[job:{job_id}] {worker_name.capitalize()} worker already complete, skipping")
+        return WorkerResponse(
+            status="already_complete",
+            job_id=job_id,
+            message=f"{worker_name.capitalize()} worker already completed"
+        )
+    
+    # Mark as running before starting (for idempotency on next retry)
+    job_manager.update_state_data(job_id, progress_key, {'stage': 'running'})
+    return None
+
+
+@router.post("/workers/audio", response_model=WorkerResponse)
+async def trigger_audio_worker(
+    request: WorkerRequest,
+    http_request: Request,
+    background_tasks: BackgroundTasks,
+    auth_data: Tuple[str, UserType, int] = Depends(require_admin)
+):
+    """
+    Trigger audio separation worker for a job.
+    
+    This endpoint is called internally after job creation to start
+    the audio processing track (parallel with lyrics processing).
+    
+    Idempotency: If worker is already running or complete, returns early.
+    
+    The worker runs in the background and updates job state as it progresses.
+    """
+    job_id = request.job_id
+    
+    # Extract trace context from incoming request (propagated via Cloud Tasks)
+    trace_context = extract_trace_context(dict(http_request.headers))
+    
+    logger.info(f"[job:{job_id}] WORKER_TRIGGER worker=audio")
+    add_span_attribute("job_id", job_id)
+    add_span_attribute("worker", "audio")
+    
+    # Idempotency check
+    skip_response = _check_worker_idempotency(job_id, "audio")
+    if skip_response:
+        add_span_event("worker_skipped", {"reason": skip_response.status})
+        return skip_response
+    
+    # Add task to background tasks
+    # This allows the HTTP response to return immediately
+    # while the worker continues processing
+    background_tasks.add_task(process_audio_separation, job_id)
+    
+    add_span_event("worker_started")
+    return WorkerResponse(
+        status="started",
+        job_id=job_id,
+        message="Audio separation worker started"
+    )
+
+
+@router.post("/workers/lyrics", response_model=WorkerResponse)
+async def trigger_lyrics_worker(
+    request: WorkerRequest,
+    http_request: Request,
+    background_tasks: BackgroundTasks,
+    auth_data: Tuple[str, UserType, int] = Depends(require_admin)
+):
+    """
+    Trigger lyrics transcription worker for a job.
+    
+    This endpoint is called internally after job creation to start
+    the lyrics processing track (parallel with audio processing).
+    
+    Idempotency: If worker is already running or complete, returns early.
+    
+    The worker runs in the background and updates job state as it progresses.
+    """
+    job_id = request.job_id
+    
+    # Extract trace context from incoming request
+    trace_context = extract_trace_context(dict(http_request.headers))
+    
+    logger.info(f"[job:{job_id}] WORKER_TRIGGER worker=lyrics")
+    add_span_attribute("job_id", job_id)
+    add_span_attribute("worker", "lyrics")
+    
+    # Idempotency check
+    skip_response = _check_worker_idempotency(job_id, "lyrics")
+    if skip_response:
+        add_span_event("worker_skipped", {"reason": skip_response.status})
+        return skip_response
+    
+    # Add task to background tasks
+    background_tasks.add_task(process_lyrics_transcription, job_id)
+    
+    add_span_event("worker_started")
+    return WorkerResponse(
+        status="started",
+        job_id=job_id,
+        message="Lyrics transcription worker started"
+    )
+
+
+@router.post("/workers/screens", response_model=WorkerResponse)
+async def trigger_screens_worker(
+    request: WorkerRequest,
+    http_request: Request,
+    background_tasks: BackgroundTasks,
+    auth_data: Tuple[str, UserType, int] = Depends(require_admin)
+):
+    """
+    Trigger title/end screen generation worker.
+    
+    This is called automatically when both audio and lyrics are complete.
+    
+    Idempotency: If worker is already running or complete, returns early.
+    """
+    job_id = request.job_id
+    
+    # Extract trace context from incoming request
+    trace_context = extract_trace_context(dict(http_request.headers))
+    
+    logger.info(f"[job:{job_id}] WORKER_TRIGGER worker=screens")
+    add_span_attribute("job_id", job_id)
+    add_span_attribute("worker", "screens")
+    
+    # Idempotency check
+    skip_response = _check_worker_idempotency(job_id, "screens")
+    if skip_response:
+        add_span_event("worker_skipped", {"reason": skip_response.status})
+        return skip_response
+    
+    # Add task to background tasks
+    background_tasks.add_task(generate_screens, job_id)
+    
+    add_span_event("worker_started")
+    return WorkerResponse(
+        status="started",
+        job_id=job_id,
+        message="Screens generation worker started"
+    )
+
+
+@router.post("/workers/video", response_model=WorkerResponse)
+async def trigger_video_worker(
+    request: WorkerRequest,
+    http_request: Request,
+    background_tasks: BackgroundTasks,
+    auth_data: Tuple[str, UserType, int] = Depends(require_admin)
+):
+    """
+    Trigger final video generation and encoding worker.
+    
+    This is called after user selects their preferred instrumental.
+    This is the longest-running stage (15-20 minutes).
+    
+    Idempotency: If worker is already running or complete, returns early.
+    """
+    job_id = request.job_id
+    
+    # Extract trace context from incoming request
+    trace_context = extract_trace_context(dict(http_request.headers))
+    
+    logger.info(f"[job:{job_id}] WORKER_TRIGGER worker=video")
+    add_span_attribute("job_id", job_id)
+    add_span_attribute("worker", "video")
+    
+    # Idempotency check
+    skip_response = _check_worker_idempotency(job_id, "video")
+    if skip_response:
+        add_span_event("worker_skipped", {"reason": skip_response.status})
+        return skip_response
+    
+    # Add task to background tasks
+    background_tasks.add_task(generate_video, job_id)
+    
+    add_span_event("worker_started")
+    return WorkerResponse(
+        status="started",
+        job_id=job_id,
+        message="Video generation worker started"
+    )
+
+
+@router.post("/workers/render-video", response_model=WorkerResponse)
+async def trigger_render_video_worker(
+    request: WorkerRequest,
+    http_request: Request,
+    background_tasks: BackgroundTasks,
+    auth_data: Tuple[str, UserType, int] = Depends(require_admin)
+):
+    """
+    Trigger render video worker (post-review).
+    
+    This is called after human review is complete.
+    Uses OutputGenerator from LyricsTranscriber to generate the karaoke video
+    with the corrected lyrics.
+    
+    Idempotency: If worker is already running or complete, returns early.
+    
+    Output: with_vocals.mkv in GCS
+    Next state: AWAITING_INSTRUMENTAL_SELECTION
+    """
+    job_id = request.job_id
+    
+    # Extract trace context from incoming request
+    trace_context = extract_trace_context(dict(http_request.headers))
+    
+    logger.info(f"[job:{job_id}] WORKER_TRIGGER worker=render-video")
+    add_span_attribute("job_id", job_id)
+    add_span_attribute("worker", "render-video")
+    
+    # Idempotency check
+    skip_response = _check_worker_idempotency(job_id, "render")
+    if skip_response:
+        add_span_event("worker_skipped", {"reason": skip_response.status})
+        return skip_response
+    
+    # Add task to background tasks
+    background_tasks.add_task(process_render_video, job_id)
+    
+    add_span_event("worker_started")
+    return WorkerResponse(
+        status="started",
+        job_id=job_id,
+        message="Render video worker started (post-review)"
+    )
+
+
+@router.post("/jobs/{job_id}/check-idle-reminder")
+async def check_idle_reminder(
+    job_id: str,
+    http_request: Request,
+    auth_data: Tuple[str, UserType, int] = Depends(require_admin)
+):
+    """
+    Check if a job needs an idle reminder email.
+
+    This endpoint is called by a Cloud Tasks scheduled task 5 minutes after
+    a job enters a blocking state (AWAITING_REVIEW or AWAITING_INSTRUMENTAL_SELECTION).
+
+    If the job is still in the blocking state and no reminder has been sent yet,
+    sends a reminder email to the user.
+
+    Idempotency: Only one reminder per job (tracked via reminder_sent flag).
+    """
+    from backend.models.job import JobStatus
+    from backend.services.job_notification_service import get_job_notification_service
+
+    # Extract trace context from incoming request
+    trace_context = extract_trace_context(dict(http_request.headers))
+
+    logger.info(f"[job:{job_id}] IDLE_REMINDER_CHECK starting")
+    add_span_attribute("job_id", job_id)
+    add_span_attribute("operation", "idle_reminder_check")
+
+    job_manager = JobManager()
+    job = job_manager.get_job(job_id)
+
+    if not job:
+        logger.warning(f"[job:{job_id}] Job not found for idle reminder check")
+        add_span_event("job_not_found")
+        return {"status": "not_found", "job_id": job_id, "message": "Job not found"}
+
+    # Check if job is still in a blocking state
+    blocking_states = [JobStatus.AWAITING_REVIEW, JobStatus.AWAITING_INSTRUMENTAL_SELECTION]
+    if job.status not in [s.value for s in blocking_states]:
+        logger.info(f"[job:{job_id}] Job no longer in blocking state ({job.status}), skipping reminder")
+        add_span_event("not_blocking", {"current_status": job.status})
+        return {
+            "status": "skipped",
+            "job_id": job_id,
+            "message": f"Job not in blocking state (current: {job.status})"
+        }
+
+    # Normalize state_data to prevent None errors
+    state_data = job.state_data or {}
+
+    # Check if reminder was already sent (idempotency)
+    if state_data.get('reminder_sent'):
+        logger.info(f"[job:{job_id}] Reminder already sent, skipping")
+        add_span_event("already_sent")
+        return {"status": "already_sent", "job_id": job_id, "message": "Reminder already sent"}
+
+    # Check if user has an email
+    if not job.user_email:
+        logger.warning(f"[job:{job_id}] No user email, cannot send reminder")
+        add_span_event("no_email")
+        return {"status": "no_email", "job_id": job_id, "message": "No user email configured"}
+
+    # Determine action type
+    action_type = state_data.get('blocking_action_type')
+    if not action_type:
+        action_type = "lyrics" if job.status == JobStatus.AWAITING_REVIEW.value else "instrumental"
+
+    # Send the reminder email
+    try:
+        notification_service = get_job_notification_service()
+
+        success = await notification_service.send_action_reminder_email(
+            job_id=job.job_id,
+            user_email=job.user_email,
+            action_type=action_type,
+            user_name=None,  # Could fetch from user service if needed
+            artist=job.artist,
+            title=job.title,
+            audio_hash=job.audio_hash,
+            review_token=job.review_token,
+            instrumental_token=job.instrumental_token,
+        )
+
+        if success:
+            # Mark reminder as sent (prevents duplicate sends)
+            job_manager.firestore.update_job(job_id, {
+                'state_data': {**state_data, 'reminder_sent': True}
+            })
+            logger.info(f"[job:{job_id}] Sent {action_type} reminder email to {job.user_email}")
+            add_span_event("reminder_sent", {"action_type": action_type})
+            return {
+                "status": "sent",
+                "job_id": job_id,
+                "message": f"Sent {action_type} reminder to {job.user_email}"
+            }
+        else:
+            logger.error(f"[job:{job_id}] Failed to send reminder email")
+            add_span_event("send_failed")
+            return {"status": "failed", "job_id": job_id, "message": "Failed to send reminder"}
+
+    except Exception as e:
+        logger.exception(f"[job:{job_id}] Error sending reminder: {e}")
+        add_span_event("error", {"error": str(e)})
+        return {"status": "error", "job_id": job_id, "message": str(e)}
+
+
+@router.get("/health")
+async def internal_health(
+    auth_data: Tuple[str, UserType, int] = Depends(require_admin)
+):
+    """
+    Internal health check endpoint.
+
+    Used to verify the internal API is responsive.
+    Requires admin authentication.
+    """
+    return {"status": "healthy", "service": "karaoke-backend-internal"}
+