karaoke-gen 0.96.0__py3-none-any.whl → 0.99.3__py3-none-any.whl

backend/main.py

@@ -10,6 +10,9 @@ from backend.config import settings
 from backend.api.routes import health, jobs, internal, file_upload, review, auth, audio_search, themes, users, admin
 from backend.services.tracing import setup_tracing, instrument_app, get_current_trace_id
 from backend.services.structured_logging import setup_structured_logging
+from backend.services.spacy_preloader import preload_spacy_model
+from backend.services.nltk_preloader import preload_all_nltk_resources
+from backend.services.langfuse_preloader import preload_langfuse_handler
 from backend.middleware.audit_logging import AuditLoggingMiddleware
 
 
@@ -67,7 +70,28 @@ async def lifespan(app: FastAPI):
     logger.info(f"Environment: {settings.environment}")
     logger.info(f"GCS Bucket: {settings.gcs_bucket_name}")
     logger.info(f"Tracing enabled: {tracing_enabled}")
-    
+
+    # Preload NLP models and resources to avoid cold start delays
+    # See docs/archive/2026-01-08-performance-investigation.md for background
+
+    # 1. SpaCy model (60+ second delay without preload)
+    try:
+        preload_spacy_model("en_core_web_sm")
+    except Exception as e:
+        logger.warning(f"SpaCy preload failed (will load lazily): {e}")
+
+    # 2. NLTK cmudict (50-100+ second delay without preload)
+    try:
+        preload_all_nltk_resources()
+    except Exception as e:
+        logger.warning(f"NLTK preload failed (will load lazily): {e}")
+
+    # 3. Langfuse callback handler (200+ second delay without preload)
+    try:
+        preload_langfuse_handler()
+    except Exception as e:
+        logger.warning(f"Langfuse preload failed (will initialize lazily): {e}")
+
     # Validate OAuth credentials (non-blocking)
     try:
         await validate_credentials_on_startup()

backend/services/encoding_service.py

@@ -25,6 +25,11 @@ from backend.config import get_settings
 
 logger = logging.getLogger(__name__)
 
+# Retry configuration for handling transient failures (e.g., worker restarts)
+MAX_RETRIES = 3
+INITIAL_BACKOFF_SECONDS = 2.0
+MAX_BACKOFF_SECONDS = 10.0
+
 
 class EncodingService:
     """Service for dispatching encoding jobs to GCE worker."""
@@ -68,6 +73,81 @@ class EncodingService:
         """Check if GCE preview encoding is enabled and configured."""
         return self.settings.use_gce_preview_encoding and self.is_configured
 
+    async def _request_with_retry(
+        self,
+        method: str,
+        url: str,
+        headers: Dict[str, str],
+        json_payload: Optional[Dict[str, Any]] = None,
+        timeout: float = 30.0,
+        job_id: str = "unknown",
+    ) -> Dict[str, Any]:
+        """
+        Make an HTTP request with retry logic for transient failures.
+
+        This handles connection errors that occur when the GCE worker is
+        restarting (e.g., during deployments) by retrying with exponential backoff.
+
+        Args:
+            method: HTTP method (GET, POST)
+            url: Request URL
+            headers: Request headers
+            json_payload: JSON body for POST requests
+            timeout: Request timeout in seconds
+            job_id: Job ID for logging
+
+        Returns:
+            Dict with keys:
+                - status (int): HTTP status code
+                - json (Any): Parsed JSON response body (if status 200, else None)
+                - text (str): Raw response text (if status != 200, else None)
+
+        Raises:
+            aiohttp.ClientConnectorError: If all retries fail due to connection errors
+            aiohttp.ServerDisconnectedError: If all retries fail due to server disconnect
+            asyncio.TimeoutError: If all retries fail due to timeout
+        """
+        last_exception = None
+        backoff = INITIAL_BACKOFF_SECONDS
+
+        for attempt in range(MAX_RETRIES + 1):
+            try:
+                async with aiohttp.ClientSession() as session:
+                    if method.upper() == "POST":
+                        async with session.post(
+                            url, json=json_payload, headers=headers, timeout=timeout
+                        ) as resp:
+                            # Return a copy of the response data since we exit the context
+                            return {
+                                "status": resp.status,
+                                "json": await resp.json() if resp.status == 200 else None,
+                                "text": await resp.text() if resp.status != 200 else None,
+                            }
+                    else:  # GET
+                        async with session.get(
+                            url, headers=headers, timeout=timeout
+                        ) as resp:
+                            return {
+                                "status": resp.status,
+                                "json": await resp.json() if resp.status == 200 else None,
+                                "text": await resp.text() if resp.status != 200 else None,
+                            }
+            except (aiohttp.ClientConnectorError, aiohttp.ServerDisconnectedError, asyncio.TimeoutError) as e:
+                last_exception = e
+                if attempt < MAX_RETRIES:
+                    logger.warning(
+                        f"[job:{job_id}] GCE worker connection failed (attempt {attempt + 1}/{MAX_RETRIES + 1}): {e}. "
+                        f"Retrying in {backoff:.1f}s..."
+                    )
+                    await asyncio.sleep(backoff)
+                    backoff = min(backoff * 2, MAX_BACKOFF_SECONDS)
+                else:
+                    logger.error(
+                        f"[job:{job_id}] GCE worker connection failed after {MAX_RETRIES + 1} attempts: {e}"
+                    )
+
+        raise last_exception
+
     async def submit_encoding_job(
         self,
         job_id: str,
@@ -106,17 +186,23 @@ class EncodingService:
 
         logger.info(f"[job:{job_id}] Submitting encoding job to GCE worker: {url}")
 
-        async with aiohttp.ClientSession() as session:
-            async with session.post(url, json=payload, headers=headers, timeout=30) as resp:
-                if resp.status == 401:
-                    raise RuntimeError("Invalid API key for encoding worker")
-                if resp.status == 409:
-                    raise RuntimeError(f"Encoding job {job_id} already exists")
-                if resp.status != 200:
-                    text = await resp.text()
-                    raise RuntimeError(f"Failed to submit encoding job: {resp.status} - {text}")
+        resp = await self._request_with_retry(
+            method="POST",
+            url=url,
+            headers=headers,
+            json_payload=payload,
+            timeout=30.0,
+            job_id=job_id,
+        )
+
+        if resp["status"] == 401:
+            raise RuntimeError("Invalid API key for encoding worker")
+        if resp["status"] == 409:
+            raise RuntimeError(f"Encoding job {job_id} already exists")
+        if resp["status"] != 200:
+            raise RuntimeError(f"Failed to submit encoding job: {resp['status']} - {resp['text']}")
 
-                return await resp.json()
+        return resp["json"]
 
     async def get_job_status(self, job_id: str) -> Dict[str, Any]:
         """
@@ -136,17 +222,22 @@ class EncodingService:
         url = f"{self._url}/status/{job_id}"
         headers = {"X-API-Key": self._api_key}
 
-        async with aiohttp.ClientSession() as session:
-            async with session.get(url, headers=headers, timeout=30) as resp:
-                if resp.status == 401:
-                    raise RuntimeError("Invalid API key for encoding worker")
-                if resp.status == 404:
-                    raise RuntimeError(f"Encoding job {job_id} not found")
-                if resp.status != 200:
-                    text = await resp.text()
-                    raise RuntimeError(f"Failed to get job status: {resp.status} - {text}")
+        resp = await self._request_with_retry(
+            method="GET",
+            url=url,
+            headers=headers,
+            timeout=30.0,
+            job_id=job_id,
+        )
 
-                return await resp.json()
+        if resp["status"] == 401:
+            raise RuntimeError("Invalid API key for encoding worker")
+        if resp["status"] == 404:
+            raise RuntimeError(f"Encoding job {job_id} not found")
+        if resp["status"] != 200:
+            raise RuntimeError(f"Failed to get job status: {resp['status']} - {resp['text']}")
+
+        return resp["json"]
 
     async def wait_for_completion(
         self,
@@ -296,17 +387,23 @@ class EncodingService:
 
         logger.info(f"[job:{job_id}] Submitting preview encoding job to GCE worker: {url}")
 
-        async with aiohttp.ClientSession() as session:
-            async with session.post(url, json=payload, headers=headers, timeout=30) as resp:
-                if resp.status == 401:
-                    raise RuntimeError("Invalid API key for encoding worker")
-                if resp.status == 409:
-                    raise RuntimeError(f"Preview encoding job {job_id} already exists")
-                if resp.status != 200:
-                    text = await resp.text()
-                    raise RuntimeError(f"Failed to submit preview encoding job: {resp.status} - {text}")
-
-                return await resp.json()
+        resp = await self._request_with_retry(
+            method="POST",
+            url=url,
+            headers=headers,
+            json_payload=payload,
+            timeout=30.0,
+            job_id=job_id,
+        )
+
+        if resp["status"] == 401:
+            raise RuntimeError("Invalid API key for encoding worker")
+        if resp["status"] == 409:
+            raise RuntimeError(f"Preview encoding job {job_id} already exists")
+        if resp["status"] != 200:
+            raise RuntimeError(f"Failed to submit preview encoding job: {resp['status']} - {resp['text']}")
+
+        return resp["json"]
 
     async def encode_preview_video(
         self,

backend/services/job_manager.py

@@ -34,10 +34,21 @@ class JobManager:
     def create_job(self, job_create: JobCreate) -> Job:
         """
         Create a new job with initial state PENDING.
-        
+
         Jobs start in PENDING state and transition to DOWNLOADING
         when a worker picks them up.
+
+        Raises:
+            ValueError: If theme_id is not provided (all jobs require a theme)
         """
+        # Enforce theme requirement - all jobs must have a theme
+        # This prevents unstyled videos from ever being generated
+        if not job_create.theme_id:
+            raise ValueError(
+                "theme_id is required for all jobs. "
+                "Use get_theme_service().get_default_theme_id() to get the default theme."
+            )
+
         job_id = str(uuid.uuid4())[:8]
         
         now = datetime.utcnow()

backend/services/langfuse_preloader.py

@@ -0,0 +1,98 @@
+"""Langfuse callback handler preloader for container startup.
+
+Initializes the Langfuse callback handler at container startup to avoid
+slow initialization during request processing. The CallbackHandler()
+constructor makes blocking network calls to the Langfuse API, which can
+take 3+ minutes on Cloud Run cold starts.
+
+See docs/archive/2026-01-08-performance-investigation.md for background.
+"""
+
+import logging
+import os
+import time
+from typing import Optional, Any
+
+logger = logging.getLogger(__name__)
+
+# Singleton storage for preloaded handler
+_preloaded_handler: Optional[Any] = None
+_initialization_attempted: bool = False
+
+
+def preload_langfuse_handler() -> Optional[Any]:
+    """Preload Langfuse callback handler at startup.
+
+    Only initializes if LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY
+    environment variables are set.
+
+    Returns:
+        The preloaded CallbackHandler, or None if not configured
+    """
+    global _preloaded_handler, _initialization_attempted
+
+    if _initialization_attempted:
+        logger.debug("Langfuse initialization already attempted")
+        return _preloaded_handler
+
+    _initialization_attempted = True
+
+    # Check if Langfuse is configured
+    public_key = os.getenv("LANGFUSE_PUBLIC_KEY")
+    secret_key = os.getenv("LANGFUSE_SECRET_KEY")
+
+    if not (public_key and secret_key):
+        logger.info("Langfuse not configured (missing keys), skipping preload")
+        return None
+
+    logger.info("Preloading Langfuse callback handler...")
+    start_time = time.time()
+
+    try:
+        from langfuse.langchain import CallbackHandler
+
+        # Initialize the handler - this is the slow part that makes network calls
+        _preloaded_handler = CallbackHandler()
+
+        elapsed = time.time() - start_time
+        host = os.getenv("LANGFUSE_HOST", "cloud.langfuse.com")
+        logger.info(f"Langfuse handler preloaded in {elapsed:.2f}s (host: {host})")
+
+        return _preloaded_handler
+
+    except ImportError:
+        logger.warning("langfuse package not installed, skipping preload")
+        return None
+    except Exception as e:
+        elapsed = time.time() - start_time
+        logger.error(f"Failed to preload Langfuse handler after {elapsed:.2f}s: {e}")
+        # Don't raise - Langfuse is optional
+        return None
+
+
+def get_preloaded_langfuse_handler() -> Optional[Any]:
+    """Get the preloaded Langfuse callback handler if available.
+
+    Returns:
+        The preloaded CallbackHandler, or None if not preloaded/configured
+    """
+    return _preloaded_handler
+
+
+def is_langfuse_preloaded() -> bool:
+    """Check if Langfuse handler has been preloaded."""
+    return _preloaded_handler is not None
+
+
+def is_langfuse_configured() -> bool:
+    """Check if Langfuse environment variables are configured."""
+    return bool(
+        os.getenv("LANGFUSE_PUBLIC_KEY") and os.getenv("LANGFUSE_SECRET_KEY")
+    )
+
+
+def clear_preloaded_handler() -> None:
+    """Clear preloaded handler. Useful for testing."""
+    global _preloaded_handler, _initialization_attempted
+    _preloaded_handler = None
+    _initialization_attempted = False

backend/services/nltk_preloader.py

@@ -0,0 +1,122 @@
+"""NLTK resource preloader for container startup.
+
+Loads NLTK data at container startup to avoid slow downloads during request processing.
+Cloud Run's ephemeral filesystem means NLTK data must be re-downloaded on each cold start,
+which can take 30-100+ seconds for cmudict.
+
+See docs/archive/2026-01-08-performance-investigation.md for background.
+"""
+
+import logging
+import time
+from typing import Optional, Dict, Any
+
+logger = logging.getLogger(__name__)
+
+# Singleton storage for preloaded resources
+_preloaded_resources: Dict[str, Any] = {}
+
+
+def preload_nltk_cmudict() -> None:
+    """Preload NLTK's CMU Pronouncing Dictionary at startup.
+
+    The cmudict is used by SyllablesMatchHandler for syllable counting.
+    Without preloading, each SyllablesMatchHandler init downloads ~30MB,
+    which took 50-100+ seconds in Cloud Run.
+    """
+    global _preloaded_resources
+
+    if "cmudict" in _preloaded_resources:
+        logger.info("NLTK cmudict already preloaded")
+        return
+
+    logger.info("Preloading NLTK cmudict...")
+    start_time = time.time()
+
+    try:
+        import nltk
+
+        # Ensure the data is downloaded
+        try:
+            from nltk.corpus import cmudict
+
+            # Try to access it - will raise LookupError if not downloaded
+            _ = cmudict.dict()
+        except LookupError:
+            logger.info("Downloading NLTK cmudict data...")
+            nltk.download("cmudict", quiet=True)
+            from nltk.corpus import cmudict
+
+        # Load into memory
+        cmu_dict = cmudict.dict()
+        _preloaded_resources["cmudict"] = cmu_dict
+
+        elapsed = time.time() - start_time
+        logger.info(f"NLTK cmudict preloaded in {elapsed:.2f}s ({len(cmu_dict)} entries)")
+
+    except Exception as e:
+        logger.error(f"Failed to preload NLTK cmudict: {e}")
+        raise
+
+
+def get_preloaded_cmudict() -> Optional[Dict]:
+    """Get the preloaded CMU dictionary if available.
+
+    Returns:
+        The preloaded cmudict dictionary, or None if not preloaded
+    """
+    return _preloaded_resources.get("cmudict")
+
+
+def is_cmudict_preloaded() -> bool:
+    """Check if cmudict has been preloaded."""
+    return "cmudict" in _preloaded_resources
+
+
+def preload_nltk_punkt() -> None:
+    """Preload NLTK's punkt tokenizer (optional, used for sentence tokenization)."""
+    global _preloaded_resources
+
+    if "punkt" in _preloaded_resources:
+        logger.info("NLTK punkt already preloaded")
+        return
+
+    logger.info("Preloading NLTK punkt tokenizer...")
+    start_time = time.time()
+
+    try:
+        import nltk
+
+        try:
+            from nltk.tokenize import word_tokenize
+
+            # Test it works
+            _ = word_tokenize("test")
+        except LookupError:
+            logger.info("Downloading NLTK punkt data...")
+            nltk.download("punkt", quiet=True)
+            nltk.download("punkt_tab", quiet=True)
+
+        _preloaded_resources["punkt"] = True
+
+        elapsed = time.time() - start_time
+        logger.info(f"NLTK punkt preloaded in {elapsed:.2f}s")
+
+    except Exception as e:
+        logger.warning(f"Failed to preload NLTK punkt (non-critical): {e}")
+
+
+def preload_all_nltk_resources() -> None:
+    """Preload all NLTK resources used by the application."""
+    preload_nltk_cmudict()
+    # punkt is optional and less critical
+    try:
+        preload_nltk_punkt()
+    except Exception:
+        pass  # Non-critical
+
+
+def clear_preloaded_resources() -> None:
+    """Clear all preloaded resources. Useful for testing."""
+    global _preloaded_resources
+    _preloaded_resources.clear()

backend/services/spacy_preloader.py

@@ -0,0 +1,65 @@
+"""SpaCy model preloader for container startup.
+
+Loads SpaCy models at container startup to avoid slow loading during request processing.
+Cloud Run filesystem I/O can cause 60+ second delays when loading SpaCy models lazily.
+"""
+
+import logging
+import time
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+# Singleton storage for preloaded models
+_preloaded_models: dict = {}
+
+
+def preload_spacy_model(model_name: str = "en_core_web_sm") -> None:
+    """Preload a SpaCy model at startup.
+
+    Args:
+        model_name: The SpaCy model to load (default: en_core_web_sm)
+    """
+    global _preloaded_models
+
+    if model_name in _preloaded_models:
+        logger.info(f"SpaCy model '{model_name}' already preloaded")
+        return
+
+    logger.info(f"Preloading SpaCy model '{model_name}'...")
+    start_time = time.time()
+
+    try:
+        import spacy
+
+        nlp = spacy.load(model_name)
+        _preloaded_models[model_name] = nlp
+
+        elapsed = time.time() - start_time
+        logger.info(f"SpaCy model '{model_name}' preloaded in {elapsed:.2f}s")
+    except Exception as e:
+        logger.error(f"Failed to preload SpaCy model '{model_name}': {e}")
+        raise
+
+
+def get_preloaded_model(model_name: str = "en_core_web_sm") -> Optional[object]:
+    """Get a preloaded SpaCy model if available.
+
+    Args:
+        model_name: The SpaCy model name
+
+    Returns:
+        The preloaded SpaCy Language object, or None if not preloaded
+    """
+    return _preloaded_models.get(model_name)
+
+
+def is_model_preloaded(model_name: str = "en_core_web_sm") -> bool:
+    """Check if a SpaCy model has been preloaded."""
+    return model_name in _preloaded_models
+
+
+def clear_preloaded_models() -> None:
+    """Clear all preloaded models. Useful for testing."""
+    global _preloaded_models
+    _preloaded_models.clear()

backend/services/stripe_service.py

@@ -46,6 +46,13 @@ CREDIT_PACKAGES = {
     },
 }
 
+# Done-for-you service package (not a credit package - creates a job directly)
+DONE_FOR_YOU_PACKAGE = {
+    "price_cents": 1500,  # $15.00
+    "name": "Done For You Karaoke Video",
+    "description": "Full-service karaoke video creation with 24-hour delivery",
+}
+
 
 class StripeService:
     """Service for Stripe payment processing."""
@@ -143,6 +150,95 @@ class StripeService:
             logger.error(f"Error creating checkout session: {e}")
             return False, None, "Failed to create checkout session"
 
+    def create_done_for_you_checkout_session(
+        self,
+        customer_email: str,
+        artist: str,
+        title: str,
+        source_type: str = "search",
+        youtube_url: Optional[str] = None,
+        notes: Optional[str] = None,
+        success_url: Optional[str] = None,
+        cancel_url: Optional[str] = None,
+    ) -> Tuple[bool, Optional[str], str]:
+        """
+        Create a Stripe Checkout session for a done-for-you order.
+
+        This is for the full-service karaoke video creation where Nomad Karaoke
+        handles all the work (lyrics review, instrumental selection, etc.).
+
+        Args:
+            customer_email: Customer's email for delivery
+            artist: Song artist
+            title: Song title
+            source_type: Audio source type (search, youtube, upload)
+            youtube_url: YouTube URL if source_type is youtube
+            notes: Any special requests from customer
+            success_url: URL to redirect to on success (optional)
+            cancel_url: URL to redirect to on cancel (optional)
+
+        Returns:
+            (success, checkout_url, message)
+        """
+        if not self.is_configured():
+            return False, None, "Payment processing is not configured"
+
+        try:
+            # Default URLs - redirect to homepage success page
+            if not success_url:
+                success_url = "https://nomadkaraoke.com/order/success/?session_id={CHECKOUT_SESSION_ID}"
+            if not cancel_url:
+                cancel_url = "https://nomadkaraoke.com/#do-it-for-me"
+
+            # Build metadata for job creation after payment
+            metadata = {
+                'order_type': 'done_for_you',
+                'customer_email': customer_email,
+                'artist': artist,
+                'title': title,
+                'source_type': source_type,
+            }
+            if youtube_url:
+                metadata['youtube_url'] = youtube_url
+            if notes:
+                # Truncate notes to fit Stripe's 500 char limit per metadata value
+                metadata['notes'] = notes[:500] if len(notes) > 500 else notes
+
+            # Create checkout session
+            session = stripe.checkout.Session.create(
+                payment_method_types=['card'],
+                line_items=[{
+                    'price_data': {
+                        'currency': 'usd',
+                        'product_data': {
+                            'name': DONE_FOR_YOU_PACKAGE['name'],
+                            'description': f"{artist} - {title}",
+                        },
+                        'unit_amount': DONE_FOR_YOU_PACKAGE['price_cents'],
+                    },
+                    'quantity': 1,
+                }],
+                mode='payment',
+                success_url=success_url,
+                cancel_url=cancel_url,
+                customer_email=customer_email,
+                metadata=metadata,
+                allow_promotion_codes=True,
+            )
+
+            logger.info(
+                f"Created done-for-you checkout session {session.id} for {customer_email}, "
+                f"song: {artist} - {title}"
+            )
+            return True, session.url, "Checkout session created"
+
+        except stripe.error.StripeError as e:
+            logger.error(f"Stripe error creating done-for-you checkout session: {e}")
+            return False, None, f"Payment error: {str(e)}"
+        except Exception as e:
+            logger.error(f"Error creating done-for-you checkout session: {e}")
+            return False, None, "Failed to create checkout session"
+
     def verify_webhook_signature(self, payload: bytes, signature: str) -> Tuple[bool, Optional[Dict], str]:
         """
         Verify a Stripe webhook signature.