karaoke-gen 0.90.1__py3-none-any.whl → 0.99.3__py3-none-any.whl

lyrics_transcriber/correction/agentic/providers/langchain_bridge.py

@@ -13,6 +13,7 @@ from __future__ import annotations
 
 import logging
 import os
+import threading
 import time
 from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
 from typing import List, Dict, Any, Optional
@@ -94,11 +95,80 @@ class LangChainBridge(BaseAIProvider):
             cache_dir=self._config.cache_dir,
             enabled=cache_enabled
         )
-        
-        # Lazy-initialized chat model
+
+        # Lazy-initialized chat model with thread-safe initialization
+        # Lock prevents race condition where multiple threads try to initialize simultaneously
         self._chat_model: Optional[Any] = None
-        self._warmed_up: bool = False
-    
+        self._model_init_lock = threading.Lock()
+
+    def warmup(self) -> bool:
+        """Eagerly initialize the chat model.
+
+        Call this after creating the bridge to avoid lazy initialization delays
+        when multiple threads call generate_correction_proposals() simultaneously.
+
+        Returns:
+            True if model was initialized successfully, False otherwise
+        """
+        if self._chat_model is not None:
+            logger.debug(f"🤖 Model {self._model} already initialized")
+            return True
+
+        logger.info(f"🤖 Warming up model {self._model}...")
+        # Trigger initialization by calling the initialization logic directly
+        try:
+            self._ensure_model_initialized()
+            return self._chat_model is not None
+        except Exception as e:
+            logger.error(f"🤖 Warmup failed for {self._model}: {e}")
+            return False
+
+    def _ensure_model_initialized(self) -> None:
+        """Ensure the chat model is initialized (thread-safe).
+
+        This method handles the lazy initialization with proper locking.
+        It's separated out so it can be called from both warmup() and
+        generate_correction_proposals().
+        """
+        if self._chat_model is not None:
+            return
+
+        with self._model_init_lock:
+            # Double-check after acquiring lock
+            if self._chat_model is not None:
+                return
+
+            timeout = self._config.initialization_timeout_seconds
+            logger.info(f"🤖 Initializing model {self._model} with {timeout}s timeout...")
+            init_start = time.time()
+
+            try:
+                # Use ThreadPoolExecutor for cross-platform timeout
+                with ThreadPoolExecutor(max_workers=1) as executor:
+                    future = executor.submit(
+                        self._factory.create_chat_model,
+                        self._model,
+                        self._config
+                    )
+                    try:
+                        self._chat_model = future.result(timeout=timeout)
+                    except FuturesTimeoutError:
+                        raise InitializationTimeoutError(
+                            f"Model initialization timed out after {timeout}s. "
+                            f"This may indicate network issues or service unavailability."
+                        ) from None
+
+                init_elapsed = time.time() - init_start
+                logger.info(f"🤖 Model initialized in {init_elapsed:.2f}s")
+
+            except InitializationTimeoutError:
+                self._circuit_breaker.record_failure(self._model)
+                raise
+            except Exception as e:
+                self._circuit_breaker.record_failure(self._model)
+                logger.error(f"🤖 Failed to initialize chat model: {e}")
+                raise
+
     def name(self) -> str:
         """Return provider name for logging."""
         return f"langchain:{self._model}"
@@ -141,52 +211,28 @@ class LangChainBridge(BaseAIProvider):
                 "until": open_until
             }]
         
-        # Step 2: Get or create chat model with initialization timeout
-        if not self._chat_model:
-            timeout = self._config.initialization_timeout_seconds
-            logger.info(f"🤖 Initializing model {self._model} with {timeout}s timeout...")
-            init_start = time.time()
-
-            try:
-                # Use ThreadPoolExecutor for cross-platform timeout
-                with ThreadPoolExecutor(max_workers=1) as executor:
-                    future = executor.submit(
-                        self._factory.create_chat_model,
-                        self._model,
-                        self._config
-                    )
-                    try:
-                        self._chat_model = future.result(timeout=timeout)
-                    except FuturesTimeoutError:
-                        raise InitializationTimeoutError(
-                            f"Model initialization timed out after {timeout}s. "
-                            f"This may indicate network issues or service unavailability."
-                        ) from None
-
-                init_elapsed = time.time() - init_start
-                logger.info(f"🤖 Model created in {init_elapsed:.2f}s, starting warm-up...")
-
-                # Warm up the model to establish connection before real work
-                self._warm_up_model()
-
-                total_elapsed = time.time() - init_start
-                logger.info(f"🤖 Model initialization complete in {total_elapsed:.2f}s")
-
-            except InitializationTimeoutError as e:
-                self._circuit_breaker.record_failure(self._model)
-                logger.exception("🤖 Model initialization timeout")
-                return [{
-                    "error": INIT_TIMEOUT_ERROR,
-                    "message": str(e),
-                    "timeout_seconds": timeout
-                }]
-            except Exception as e:
-                self._circuit_breaker.record_failure(self._model)
-                logger.error(f"🤖 Failed to initialize chat model: {e}")
-                return [{
-                    "error": MODEL_INIT_ERROR,
-                    "message": str(e)
-                }]
+        # Step 2: Get or create chat model with thread-safe initialization
+        # Use double-checked locking to avoid race condition where multiple threads
+        # all try to initialize the model simultaneously (which caused job 2ccbdf6b
+        # to have 5 concurrent model initializations and 6+ minute delays)
+        #
+        # NOTE: For best performance, call warmup() after creating the bridge to
+        # eagerly initialize the model before parallel processing begins.
+        try:
+            self._ensure_model_initialized()
+        except InitializationTimeoutError as e:
+            logger.exception("🤖 Model initialization timeout")
+            return [{
+                "error": INIT_TIMEOUT_ERROR,
+                "message": str(e),
+                "timeout_seconds": self._config.initialization_timeout_seconds
+            }]
+        except Exception as e:
+            logger.error(f"🤖 Failed to initialize chat model: {e}")
+            return [{
+                "error": MODEL_INIT_ERROR,
+                "message": str(e)
+            }]
         
         # Step 3: Execute with retry logic
         logger.info(
@@ -271,47 +317,4 @@ class LangChainBridge(BaseAIProvider):
 
         return content
 
-    def _warm_up_model(self) -> None:
-        """Send a lightweight request to warm up the model connection.
-
-        This helps establish the REST connection and potentially warm up any
-        server-side resources before processing real correction requests.
-        The warm-up uses a timeout to fail fast if the service is unresponsive.
-        """
-        if self._warmed_up:
-            return
-
-        timeout = self._config.warmup_timeout_seconds
-        # Use print with flush=True for visibility when output is redirected
-        print(f"🔥 Warming up {self._model} connection (timeout: {timeout}s)...", flush=True)
-        logger.info(f"🔥 Warming up {self._model} connection (timeout: {timeout}s)...")
-
-        warmup_start = time.time()
-        try:
-            from langchain_core.messages import HumanMessage
-
-            # Minimal prompt that requires almost no processing
-            warm_up_prompt = 'Respond with exactly: {"status":"ready"}'
-
-            # Use ThreadPoolExecutor for timeout on warm-up call
-            with ThreadPoolExecutor(max_workers=1) as executor:
-                future = executor.submit(
-                    self._chat_model.invoke,
-                    [HumanMessage(content=warm_up_prompt)]
-                )
-                try:
-                    future.result(timeout=timeout)
-                except FuturesTimeoutError:
-                    raise TimeoutError(f"Warm-up timed out after {timeout}s") from None
-
-            elapsed = time.time() - warmup_start
-            self._warmed_up = True
-            print(f"🔥 Warm-up complete for {self._model} in {elapsed:.2f}s", flush=True)
-            logger.info(f"🔥 Warm-up complete for {self._model} in {elapsed:.2f}s")
-        except Exception as e:
-            elapsed = time.time() - warmup_start
-            # Don't fail the actual request if warm-up fails
-            # Just log and continue - the real request might still work
-            print(f"🔥 Warm-up failed for {self._model} after {elapsed:.2f}s: {e} (continuing anyway)", flush=True)
-            logger.warning(f"🔥 Warm-up failed for {self._model} after {elapsed:.2f}s: {e} (continuing anyway)")
 

lyrics_transcriber/correction/agentic/providers/model_factory.py

@@ -10,6 +10,14 @@ from .config import ProviderConfig
 
 logger = logging.getLogger(__name__)
 
+# Try to import Langfuse preloader (may not exist in standalone library usage)
+try:
+    from backend.services.langfuse_preloader import get_preloaded_langfuse_handler
+
+    _HAS_LANGFUSE_PRELOADER = True
+except ImportError:
+    _HAS_LANGFUSE_PRELOADER = False
+
 # Error message constant for TRY003 compliance
 GOOGLE_API_KEY_MISSING_ERROR = (
     "GOOGLE_API_KEY environment variable is required for Google/Gemini models. "
@@ -87,25 +95,38 @@ class ModelFactory:
     
     def _initialize_langfuse(self, model_spec: str) -> None:
         """Initialize Langfuse callback handler if keys are present.
-        
+
+        First tries to use a preloaded handler (to avoid 200+ second init delay
+        on Cloud Run cold starts), then falls back to creating a new one.
+
         Langfuse reads credentials from environment variables automatically:
         - LANGFUSE_PUBLIC_KEY
-        - LANGFUSE_SECRET_KEY  
+        - LANGFUSE_SECRET_KEY
         - LANGFUSE_HOST (optional)
-        
+
         Args:
             model_spec: Model specification for logging
-            
+
         Raises:
             RuntimeError: If Langfuse keys are set but initialization fails
         """
         public_key = os.getenv("LANGFUSE_PUBLIC_KEY")
         secret_key = os.getenv("LANGFUSE_SECRET_KEY")
-        
+
         if not (public_key and secret_key):
             logger.debug("🤖 Langfuse keys not found, tracing disabled")
             return
-        
+
+        # Try to use preloaded handler first (avoids 200+ second delay on Cloud Run)
+        if _HAS_LANGFUSE_PRELOADER:
+            preloaded = get_preloaded_langfuse_handler()
+            if preloaded is not None:
+                logger.info(f"🤖 Using preloaded Langfuse handler for {model_spec}")
+                self._langfuse_handler = preloaded
+                return
+
+        # Fall back to creating new handler
+        logger.info(f"🤖 Initializing Langfuse handler (not preloaded) for {model_spec}...")
         try:
             from langfuse.langchain import CallbackHandler
 

lyrics_transcriber/correction/anchor_sequence.py

@@ -32,19 +32,24 @@ class AnchorSequenceFinder:
         progress_check_interval: int = 50,  # Check progress every N iterations
         logger: Optional[logging.Logger] = None,
     ):
+        init_start = time.time()
         self.min_sequence_length = min_sequence_length
         self.min_sources = min_sources
         self.timeout_seconds = timeout_seconds
         self.max_iterations_per_ngram = max_iterations_per_ngram
         self.progress_check_interval = progress_check_interval
         self.logger = logger or logging.getLogger(__name__)
+
+        self.logger.info("Initializing AnchorSequenceFinder...")
         self.phrase_analyzer = PhraseAnalyzer(logger=self.logger)
         self.used_positions = {}
 
         # Initialize cache directory
         self.cache_dir = Path(cache_dir)
         self.cache_dir.mkdir(parents=True, exist_ok=True)
-        self.logger.info(f"Initialized AnchorSequenceFinder with cache dir: {self.cache_dir}, timeout: {timeout_seconds}s")
+
+        init_elapsed = time.time() - init_start
+        self.logger.info(f"Initialized AnchorSequenceFinder in {init_elapsed:.2f}s (cache: {self.cache_dir}, timeout: {timeout_seconds}s)")
 
     def _check_timeout(self, start_time: float, operation_name: str = "operation"):
         """Check if timeout has occurred and raise exception if so."""
@@ -245,6 +250,65 @@ class AnchorSequenceFinder:
             self.logger.error(f"Unexpected error loading cache: {type(e).__name__}: {e}")
             return None
 
+    def _process_ngram_length_no_state(
+        self,
+        n: int,
+        trans_words: List[str],
+        all_words: List[Word],
+        ref_texts_clean: Dict[str, List[str]],
+        ref_words: Dict[str, List[Word]],
+        min_sources: int,
+    ) -> List[AnchorSequence]:
+        """Process a single n-gram length without modifying shared state (thread-safe).
+
+        This version doesn't track used positions - overlap filtering happens later
+        in _remove_overlapping_sequences. This allows parallel processing of different
+        n-gram lengths.
+        """
+        candidate_anchors = []
+
+        # Build hash-based index for O(1) lookups
+        ngram_index = self._build_ngram_index(ref_texts_clean, n)
+
+        # Generate n-grams from transcribed text
+        trans_ngrams = self._find_ngrams(trans_words, n)
+
+        # Single pass through all transcription n-grams
+        for ngram, trans_pos in trans_ngrams:
+            # Use indexed lookup (O(1) instead of O(n))
+            ngram_tuple = tuple(ngram)
+            if ngram_tuple not in ngram_index:
+                continue
+
+            # Find matches in all sources (no used_positions check - handled later)
+            matches = {}
+            source_positions = ngram_index[ngram_tuple]
+            for source, positions in source_positions.items():
+                if positions:
+                    matches[source] = positions[0]  # Take first position
+
+            if len(matches) >= min_sources:
+                # Get Word IDs for transcribed words
+                transcribed_word_ids = [w.id for w in all_words[trans_pos : trans_pos + n]]
+
+                # Get Word IDs for reference words
+                reference_word_ids = {
+                    source: [w.id for w in ref_words[source][pos : pos + n]]
+                    for source, pos in matches.items()
+                }
+
+                anchor = AnchorSequence(
+                    id=WordUtils.generate_id(),
+                    transcribed_word_ids=transcribed_word_ids,
+                    transcription_position=trans_pos,
+                    reference_positions=matches,
+                    reference_word_ids=reference_word_ids,
+                    confidence=len(matches) / len(ref_texts_clean),
+                )
+                candidate_anchors.append(anchor)
+
+        return candidate_anchors
+
     def _process_ngram_length(
         self,
         n: int,
@@ -408,45 +472,95 @@ class AnchorSequenceFinder:
                 min_sources=self.min_sources,
             )
 
-            # Process n-gram lengths sequentially (single-threaded for cloud compatibility)
+            # Process n-gram lengths in parallel for better performance
+            # The overlap filtering at the end handles deduplication, so we don't
+            # need to track used_positions during processing
             candidate_anchors = []
-            
+
             # Check timeout before processing
             self._check_timeout(start_time, "n-gram processing start")
-            self.logger.info(f"🔍 ANCHOR SEARCH: Starting sequential n-gram processing ({len(n_gram_lengths)} lengths)")
-            
-            batch_size = 10
-            batch_results = []
-            
-            for i, n in enumerate(n_gram_lengths):
-                try:
-                    # Check timeout periodically
-                    if self.timeout_seconds > 0:
-                        elapsed_time = time.time() - start_time
-                        if elapsed_time > self.timeout_seconds:
-                            self.logger.warning(f"🔍 ANCHOR SEARCH: ⏰ Timeout reached at n-gram {n}, stopping")
-                            break
-                    
-                    anchors = self._process_ngram_length(
-                        n, trans_words, all_words, ref_texts_clean, ref_words, self.min_sources
-                    )
-                    candidate_anchors.extend(anchors)
-                    
-                    # Batch logging
-                    batch_results.append((n, len(anchors)))
-                    
-                    # Log progress every batch_size results or on the last result
-                    if (i + 1) % batch_size == 0 or (i + 1) == len(n_gram_lengths):
-                        total_anchors_in_batch = sum(anchor_count for _, anchor_count in batch_results)
-                        n_gram_ranges = [str(ng) for ng, _ in batch_results]
-                        range_str = f"{n_gram_ranges[0]}-{n_gram_ranges[-1]}" if len(n_gram_ranges) > 1 else n_gram_ranges[0]
-                        self.logger.debug(f"🔍 ANCHOR SEARCH: Completed n-gram lengths {range_str} - found {total_anchors_in_batch} anchors")
-                        batch_results = []
-                        
-                except Exception as e:
-                    self.logger.warning(f"🔍 ANCHOR SEARCH: ⚠️ n-gram length {n} failed: {str(e)}")
-                    batch_results.append((n, 0))
-                    continue
+
+            # Determine parallelization strategy
+            import os
+            from concurrent.futures import ThreadPoolExecutor, as_completed
+
+            # Use parallel processing by default, can be disabled via env var
+            use_parallel = os.getenv("ANCHOR_SEARCH_SEQUENTIAL", "0").lower() not in {"1", "true", "yes"}
+            max_workers = int(os.getenv("ANCHOR_SEARCH_WORKERS", "4"))
+
+            if use_parallel and len(n_gram_lengths) > 1:
+                self.logger.info(f"🔍 ANCHOR SEARCH: Starting PARALLEL n-gram processing ({len(n_gram_lengths)} lengths, {max_workers} workers)")
+
+                # Process in parallel - each n-gram length is independent
+                # since we don't track used_positions during processing
+                with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                    # Submit all tasks
+                    future_to_n = {
+                        executor.submit(
+                            self._process_ngram_length_no_state,
+                            n, trans_words, all_words, ref_texts_clean, ref_words, self.min_sources
+                        ): n
+                        for n in n_gram_lengths
+                    }
+
+                    completed = 0
+                    for future in as_completed(future_to_n):
+                        n = future_to_n[future]
+                        completed += 1
+
+                        # Check timeout periodically
+                        if self.timeout_seconds > 0:
+                            elapsed_time = time.time() - start_time
+                            if elapsed_time > self.timeout_seconds:
+                                self.logger.warning(f"🔍 ANCHOR SEARCH: ⏰ Timeout reached, stopping ({completed}/{len(n_gram_lengths)} completed)")
+                                # Cancel remaining futures
+                                for f in future_to_n.keys():
+                                    f.cancel()
+                                break
+
+                        try:
+                            anchors = future.result()
+                            candidate_anchors.extend(anchors)
+                            if completed % 20 == 0:
+                                self.logger.debug(f"🔍 ANCHOR SEARCH: Progress {completed}/{len(n_gram_lengths)} lengths processed")
+                        except Exception as e:
+                            self.logger.warning(f"🔍 ANCHOR SEARCH: ⚠️ n-gram length {n} failed: {str(e)}")
+            else:
+                # Sequential fallback
+                self.logger.info(f"🔍 ANCHOR SEARCH: Starting sequential n-gram processing ({len(n_gram_lengths)} lengths)")
+
+                batch_size = 10
+                batch_results = []
+
+                for i, n in enumerate(n_gram_lengths):
+                    try:
+                        # Check timeout periodically
+                        if self.timeout_seconds > 0:
+                            elapsed_time = time.time() - start_time
+                            if elapsed_time > self.timeout_seconds:
+                                self.logger.warning(f"🔍 ANCHOR SEARCH: ⏰ Timeout reached at n-gram {n}, stopping")
+                                break
+
+                        anchors = self._process_ngram_length(
+                            n, trans_words, all_words, ref_texts_clean, ref_words, self.min_sources
+                        )
+                        candidate_anchors.extend(anchors)
+
+                        # Batch logging
+                        batch_results.append((n, len(anchors)))
+
+                        # Log progress every batch_size results or on the last result
+                        if (i + 1) % batch_size == 0 or (i + 1) == len(n_gram_lengths):
+                            total_anchors_in_batch = sum(anchor_count for _, anchor_count in batch_results)
+                            n_gram_ranges = [str(ng) for ng, _ in batch_results]
+                            range_str = f"{n_gram_ranges[0]}-{n_gram_ranges[-1]}" if len(n_gram_ranges) > 1 else n_gram_ranges[0]
+                            self.logger.debug(f"🔍 ANCHOR SEARCH: Completed n-gram lengths {range_str} - found {total_anchors_in_batch} anchors")
+                            batch_results = []
+
+                    except Exception as e:
+                        self.logger.warning(f"🔍 ANCHOR SEARCH: ⚠️ n-gram length {n} failed: {str(e)}")
+                        batch_results.append((n, 0))
+                        continue
 
             self.logger.info(f"🔍 ANCHOR SEARCH: ✅ Found {len(candidate_anchors)} candidate anchors in {time.time() - start_time:.1f}s")