lyrics-transcriber 0.43.1__py3-none-any.whl → 0.45.0__py3-none-any.whl

lyrics_transcriber/core/controller.py

@@ -1,5 +1,6 @@
 import os
 import logging
+import json
 from dataclasses import dataclass, field
 from typing import Dict, Optional, List
 from lyrics_transcriber.types import LyricsData, TranscriptionResult, CorrectionResult
@@ -224,7 +225,29 @@ class LyricsTranscriber:
 
         self.logger.info(f"LyricsTranscriber controller beginning processing for {self.artist} - {self.title}")
 
-        # Step 1: Fetch lyrics if enabled and artist/title are provided
+        # Check for existing corrections JSON
+        corrections_json_path = os.path.join(self.output_config.output_dir, f"{self.output_prefix} (Lyrics Corrections).json")
+
+        if os.path.exists(corrections_json_path):
+            self.logger.info(f"Found existing corrections JSON: {corrections_json_path}")
+            try:
+                with open(corrections_json_path, "r", encoding="utf-8") as f:
+                    corrections_data = json.load(f)
+
+                # Reconstruct CorrectionResult from JSON
+                self.results.transcription_corrected = CorrectionResult.from_dict(corrections_data)
+                self.logger.info("Successfully loaded existing corrections data")
+
+                # Skip to output generation
+                self.generate_outputs()
+                self.logger.info("Processing completed successfully using existing corrections")
+                return self.results
+
+            except Exception as e:
+                self.logger.error(f"Failed to load existing corrections JSON: {str(e)}")
+                # Continue with normal processing if loading fails
+
+        # Normal processing flow continues...
         if self.output_config.fetch_lyrics and self.artist and self.title:
             self.fetch_lyrics()
         else:
@@ -298,6 +321,9 @@ class LyricsTranscriber:
                 sorted_results = sorted(self.results.transcription_results, key=lambda x: x.priority)
                 best_transcription = sorted_results[0]
 
+                # Count total words in the transcription
+                total_words = sum(len(segment.words) for segment in best_transcription.result.segments)
+
                 # Create a CorrectionResult with no corrections
                 self.results.transcription_corrected = CorrectionResult(
                     original_segments=best_transcription.result.segments,
@@ -308,39 +334,47 @@ class LyricsTranscriber:
                     reference_lyrics={},
                     anchor_sequences=[],
                     gap_sequences=[],
-                    resized_segments=[],  # Will be populated later
+                    resized_segments=[],
+                    correction_steps=[],
+                    word_id_map={},
+                    segment_id_map={},
                     metadata={
                         "correction_type": "none",
                         "reason": "no_reference_lyrics",
-                        "audio_filepath": self.audio_filepath,  # Add audio filepath
+                        "audio_filepath": self.audio_filepath,
+                        "anchor_sequences_count": 0,
+                        "gap_sequences_count": 0,
+                        "total_words": total_words,
+                        "correction_ratio": 0.0,
+                        "available_handlers": [],
+                        "enabled_handlers": [],
                     },
                 )
-            return
-
-        # Create metadata dict with song info
-        metadata = {
-            "artist": self.artist,
-            "title": self.title,
-            "full_reference_texts": {source: lyrics.get_full_text() for source, lyrics in self.results.lyrics_results.items()},
-        }
+        else:
+            # Create metadata dict with song info
+            metadata = {
+                "artist": self.artist,
+                "title": self.title,
+                "full_reference_texts": {source: lyrics.get_full_text() for source, lyrics in self.results.lyrics_results.items()},
+            }
 
-        # Get enabled handlers from metadata if available
-        enabled_handlers = metadata.get("enabled_handlers", None)
+            # Get enabled handlers from metadata if available
+            enabled_handlers = metadata.get("enabled_handlers", None)
 
-        # Create corrector with enabled handlers
-        corrector = LyricsCorrector(cache_dir=self.output_config.cache_dir, enabled_handlers=enabled_handlers, logger=self.logger)
+            # Create corrector with enabled handlers
+            corrector = LyricsCorrector(cache_dir=self.output_config.cache_dir, enabled_handlers=enabled_handlers, logger=self.logger)
 
-        corrected_data = corrector.run(
-            transcription_results=self.results.transcription_results,
-            lyrics_results=self.results.lyrics_results,
-            metadata=metadata,
-        )
+            corrected_data = corrector.run(
+                transcription_results=self.results.transcription_results,
+                lyrics_results=self.results.lyrics_results,
+                metadata=metadata,
+            )
 
-        # Store corrected results
-        self.results.transcription_corrected = corrected_data
-        self.logger.info("Lyrics correction completed")
+            # Store corrected results
+            self.results.transcription_corrected = corrected_data
+            self.logger.info("Lyrics correction completed")
 
-        # Add human review step
+        # Add human review step (moved outside the else block)
         if self.output_config.enable_review:
             from lyrics_transcriber.review.server import ReviewServer
 

lyrics_transcriber/correction/anchor_sequence.py

@@ -96,14 +96,17 @@ class AnchorSequenceFinder:
 
     def _get_cache_key(self, transcribed: str, references: Dict[str, LyricsData], transcription_result: TranscriptionResult) -> str:
         """Generate a unique cache key for the input combination."""
-        # Create a string that uniquely identifies the inputs, but only using stable content
-        # Use only the text content, not IDs or other potentially varying metadata
+        # Create a string that uniquely identifies the inputs, including word IDs
         ref_texts = []
         for source, lyrics in sorted(references.items()):
-            text = " ".join(w.text for s in lyrics.segments for w in s.words)
-            ref_texts.append(f"{source}:{text}")
+            # Include both text and ID for each word to ensure cache uniqueness
+            words_with_ids = [f"{w.text}:{w.id}" for s in lyrics.segments for w in s.words]
+            ref_texts.append(f"{source}:{','.join(words_with_ids)}")
 
-        input_str = f"{transcribed}|" f"{','.join(ref_texts)}"
+        # Also include transcription word IDs to ensure complete matching
+        trans_words_with_ids = [f"{w.text}:{w.id}" for s in transcription_result.segments for w in s.words]
+
+        input_str = f"{transcribed}|" f"{','.join(trans_words_with_ids)}|" f"{','.join(ref_texts)}"
         return hashlib.md5(input_str.encode()).hexdigest()
 
     def _save_to_cache(self, cache_path: Path, anchors: List[ScoredAnchor]) -> None:
@@ -259,15 +262,26 @@ class AnchorSequenceFinder:
         for segment in transcription_result.segments:
             all_words.extend(segment.words)
 
-        # Clean and split texts - this should match all_words exactly
-        trans_words = [w.text.lower().strip('.,?!"\n') for w in all_words]  # Changed to derive directly from all_words
+        # Clean and split texts
+        trans_words = [w.text.lower().strip('.,?!"\n') for w in all_words]
         ref_texts_clean = {
             source: self._clean_text(" ".join(w.text for s in lyrics.segments for w in s.words)).split()
             for source, lyrics in references.items()
         }
         ref_words = {source: [w for s in lyrics.segments for w in s.words] for source, lyrics in references.items()}
 
-        max_length = min(len(trans_words), min(len(words) for words in ref_texts_clean.values()))
+        # Filter out very short reference sources for n-gram length calculation
+        valid_ref_lengths = [
+            len(words) for words in ref_texts_clean.values()
+            if len(words) >= self.min_sequence_length
+        ]
+
+        if not valid_ref_lengths:
+            self.logger.warning("No reference sources long enough for anchor detection")
+            return []
+
+        # Calculate max length using only valid reference sources
+        max_length = min(len(trans_words), min(valid_ref_lengths))
         n_gram_lengths = range(max_length, self.min_sequence_length - 1, -1)
 
         # Process n-gram lengths in parallel

lyrics_transcriber/correction/corrector.py

@@ -2,6 +2,7 @@ from typing import List, Optional, Tuple, Union, Dict, Any
 import logging
 from pathlib import Path
 from copy import deepcopy
+import os
 
 from lyrics_transcriber.correction.handlers.levenshtein import LevenshteinHandler
 from lyrics_transcriber.correction.handlers.llm import LLMHandler
@@ -25,6 +26,7 @@ from lyrics_transcriber.correction.anchor_sequence import AnchorSequenceFinder
 from lyrics_transcriber.correction.handlers.base import GapCorrectionHandler
 from lyrics_transcriber.correction.handlers.extend_anchor import ExtendAnchorHandler
 from lyrics_transcriber.utils.word_utils import WordUtils
+from lyrics_transcriber.correction.handlers.llm_providers import OllamaProvider, OpenAIProvider
 
 
 class LyricsCorrector:
@@ -60,12 +62,54 @@ class LyricsCorrector:
             ("SyllablesMatchHandler", SyllablesMatchHandler(logger=self.logger)),
             ("RelaxedWordCountMatchHandler", RelaxedWordCountMatchHandler(logger=self.logger)),
             ("NoSpacePunctuationMatchHandler", NoSpacePunctuationMatchHandler(logger=self.logger)),
-            ("LLMHandler", LLMHandler(logger=self.logger, cache_dir=self._cache_dir)),
+            (
+                "LLMHandler_Ollama_R17B",
+                LLMHandler(
+                    provider=OllamaProvider(model="deepseek-r1:7b", logger=self.logger),
+                    name="LLMHandler_Ollama_R17B",
+                    logger=self.logger,
+                    cache_dir=self._cache_dir,
+                ),
+            ),
             ("RepeatCorrectionHandler", RepeatCorrectionHandler(logger=self.logger)),
             ("SoundAlikeHandler", SoundAlikeHandler(logger=self.logger)),
             ("LevenshteinHandler", LevenshteinHandler(logger=self.logger)),
         ]
 
+        # Add OpenRouter handlers only if API key is available
+        if os.getenv("OPENROUTER_API_KEY"):
+            openrouter_handlers = [
+                (
+                    "LLMHandler_OpenRouter_Sonnet",
+                    LLMHandler(
+                        provider=OpenAIProvider(
+                            model="anthropic/claude-3-sonnet",
+                            api_key=os.getenv("OPENROUTER_API_KEY"),
+                            base_url="https://openrouter.ai/api/v1",
+                            logger=self.logger,
+                        ),
+                        name="LLMHandler_OpenRouter_Sonnet",
+                        logger=self.logger,
+                        cache_dir=self._cache_dir,
+                    ),
+                ),
+                (
+                    "LLMHandler_OpenRouter_R1",
+                    LLMHandler(
+                        provider=OpenAIProvider(
+                            model="deepseek/deepseek-r1",
+                            api_key=os.getenv("OPENROUTER_API_KEY"),
+                            base_url="https://openrouter.ai/api/v1",
+                            logger=self.logger,
+                        ),
+                        name="LLMHandler_OpenRouter_R1",
+                        logger=self.logger,
+                        cache_dir=self._cache_dir,
+                    ),
+                ),
+            ]
+            all_handlers.extend(openrouter_handlers)
+
         # Store all handler information
         self.all_handlers = [
             {
@@ -127,8 +171,8 @@ class LyricsCorrector:
         corrections_made = len(corrections)
         correction_ratio = 1 - (corrections_made / total_words if total_words > 0 else 0)
 
-        # Get the currently enabled handler IDs using full class names
-        enabled_handlers = [handler.__class__.__name__ for handler in self.handlers]
+        # Get the currently enabled handler IDs using the handler's name attribute if available
+        enabled_handlers = [getattr(handler, "name", handler.__class__.__name__) for handler in self.handlers]
 
         return CorrectionResult(
             original_segments=primary_transcription.segments,

lyrics_transcriber/correction/handlers/llm.py

@@ -1,22 +1,25 @@
 from typing import List, Optional, Tuple, Dict, Any, Union
 import logging
 import json
-from ollama import chat
 from datetime import datetime
 from pathlib import Path
 
 from lyrics_transcriber.types import GapSequence, WordCorrection
 from lyrics_transcriber.correction.handlers.base import GapCorrectionHandler
 from lyrics_transcriber.correction.handlers.word_operations import WordOperations
+from lyrics_transcriber.correction.handlers.llm_providers import LLMProvider
 
 
 class LLMHandler(GapCorrectionHandler):
     """Uses an LLM to analyze and correct gaps by comparing with reference lyrics."""
 
-    def __init__(self, logger: Optional[logging.Logger] = None, cache_dir: Optional[Union[str, Path]] = None):
+    def __init__(
+        self, provider: LLMProvider, name: str, logger: Optional[logging.Logger] = None, cache_dir: Optional[Union[str, Path]] = None
+    ):
         super().__init__(logger)
         self.logger = logger or logging.getLogger(__name__)
-        self.model = "deepseek-r1:7b"
+        self.provider = provider
+        self.name = name
         self.cache_dir = Path(cache_dir) if cache_dir else None
 
     def _format_prompt(self, gap: GapSequence, data: Optional[Dict[str, Any]] = None) -> str:
@@ -160,16 +163,16 @@ class LLMHandler(GapCorrectionHandler):
             self.logger.debug(f"Processing gap words: {transcribed_words}")
             self.logger.debug(f"Reference word IDs: {gap.reference_word_ids}")
 
-            response = chat(model=self.model, messages=[{"role": "user", "content": prompt}], format="json")
+            response = self.provider.generate_response(prompt)
 
             # Write debug info to files
-            self._write_debug_info(prompt, response.message.content, gap_index, audio_file_hash=data.get("audio_file_hash"))
+            self._write_debug_info(prompt, response, gap_index, audio_file_hash=data.get("audio_file_hash"))
 
             try:
-                corrections_data = json.loads(response.message.content)
+                corrections_data = json.loads(response)
             except json.JSONDecodeError as e:
                 self.logger.error(f"Failed to parse LLM response as JSON: {e}")
-                self.logger.error(f"Raw response content: {response.message.content}")
+                self.logger.error(f"Raw response content: {response}")
                 return []
 
             # Check if corrections exist and are non-empty
@@ -202,7 +205,7 @@ class LLMHandler(GapCorrectionHandler):
                             source="LLM",
                             confidence=correction["confidence"],
                             reason=correction["reason"],
-                            handler="LLMHandler",
+                            handler=self.name,
                             reference_positions=reference_positions,
                             original_word_id=correction["word_id"],
                             corrected_word_id=correction.get("reference_word_id"),
@@ -223,7 +226,7 @@ class LLMHandler(GapCorrectionHandler):
                             source="LLM",
                             confidence=correction["confidence"],
                             reason=correction["reason"],
-                            handler="LLMHandler",
+                            handler=self.name,
                             reference_positions=reference_positions,
                             original_word_id=correction["word_id"],
                             corrected_word_ids=reference_word_ids,
@@ -256,7 +259,7 @@ class LLMHandler(GapCorrectionHandler):
                             confidence=correction["confidence"],
                             combine_reason=correction["reason"],
                             delete_reason=f"Part of combining words: {correction['reason']}",
-                            handler="LLMHandler",
+                            handler=self.name,
                             reference_positions=reference_positions,
                             original_word_ids=word_ids_to_combine,
                             corrected_word_id=correction.get("reference_word_id"),
@@ -275,10 +278,10 @@ class LLMHandler(GapCorrectionHandler):
                             reason=correction["reason"],
                             alternatives={},
                             is_deletion=True,
-                            handler="LLMHandler",
+                            handler=self.name,
                             reference_positions=reference_positions,
                             word_id=correction["word_id"],
-                            corrected_word_id=None,  # Deleted words don't need a corrected ID
+                            corrected_word_id=None,
                         )
                     )
 

lyrics_transcriber/correction/handlers/llm_providers.py

@@ -0,0 +1,60 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+import logging
+from ollama import chat as ollama_chat
+import openai
+
+
+class LLMProvider(ABC):
+    """Abstract base class for LLM providers."""
+
+    def __init__(self, logger: Optional[logging.Logger] = None):
+        self.logger = logger or logging.getLogger(__name__)
+
+    @abstractmethod
+    def generate_response(self, prompt: str, **kwargs) -> str:
+        """Generate a response from the LLM.
+
+        Args:
+            prompt: The prompt to send to the LLM
+            **kwargs: Additional provider-specific parameters
+
+        Returns:
+            str: The LLM's response
+        """
+        pass
+
+
+class OllamaProvider(LLMProvider):
+    """Provider for local Ollama models."""
+
+    def __init__(self, model: str, logger: Optional[logging.Logger] = None):
+        super().__init__(logger)
+        self.model = model
+
+    def generate_response(self, prompt: str, **kwargs) -> str:
+        try:
+            response = ollama_chat(model=self.model, messages=[{"role": "user", "content": prompt}], format="json")
+            return response.message.content
+        except Exception as e:
+            self.logger.error(f"Error generating Ollama response: {e}")
+            raise
+
+
+class OpenAIProvider(LLMProvider):
+    """Provider for OpenAI-compatible APIs (including OpenRouter)."""
+
+    def __init__(self, model: str, api_key: str, base_url: Optional[str] = None, logger: Optional[logging.Logger] = None):
+        super().__init__(logger)
+        self.model = model
+        self.client = openai.OpenAI(api_key=api_key, base_url=base_url)
+
+    def generate_response(self, prompt: str, **kwargs) -> str:
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model, messages=[{"role": "user", "content": prompt}], response_format={"type": "json_object"}, **kwargs
+            )
+            return response.choices[0].message.content
+        except Exception as e:
+            self.logger.error(f"Error generating OpenAI response: {e}")
+            raise