lyrics-transcriber 0.30.0__py3-none-any.whl → 0.32.1__py3-none-any.whl

lyrics_transcriber/correction/corrector.py

@@ -0,0 +1,256 @@
+from typing import List, Optional, Tuple, Union
+import logging
+from pathlib import Path
+
+from lyrics_transcriber.correction.handlers.no_space_punct_match import NoSpacePunctuationMatchHandler
+from lyrics_transcriber.correction.handlers.relaxed_word_count_match import RelaxedWordCountMatchHandler
+from lyrics_transcriber.correction.handlers.syllables_match import SyllablesMatchHandler
+from lyrics_transcriber.types import GapSequence, LyricsData, TranscriptionResult, CorrectionResult, LyricsSegment, WordCorrection, Word
+from lyrics_transcriber.correction.anchor_sequence import AnchorSequenceFinder
+from lyrics_transcriber.correction.handlers.base import GapCorrectionHandler
+from lyrics_transcriber.correction.handlers.word_count_match import WordCountMatchHandler
+from lyrics_transcriber.correction.handlers.extend_anchor import ExtendAnchorHandler
+from lyrics_transcriber.correction.handlers.sound_alike import SoundAlikeHandler
+from lyrics_transcriber.correction.handlers.levenshtein import LevenshteinHandler
+from lyrics_transcriber.correction.handlers.repeat import RepeatCorrectionHandler
+
+
+class LyricsCorrector:
+    """
+    Coordinates lyrics correction process using multiple correction handlers.
+    """
+
+    def __init__(
+        self,
+        cache_dir: Union[str, Path],
+        handlers: Optional[List[GapCorrectionHandler]] = None,
+        anchor_finder: Optional[AnchorSequenceFinder] = None,
+        logger: Optional[logging.Logger] = None,
+    ):
+        self.logger = logger or logging.getLogger(__name__)
+        self.anchor_finder = anchor_finder or AnchorSequenceFinder(cache_dir=cache_dir, logger=self.logger)
+
+        # Default handlers in order of preference
+        self.handlers = handlers or [
+            WordCountMatchHandler(),
+            RelaxedWordCountMatchHandler(),
+            NoSpacePunctuationMatchHandler(),
+            SyllablesMatchHandler(),
+            ExtendAnchorHandler(),
+            RepeatCorrectionHandler(),
+            SoundAlikeHandler(),
+            LevenshteinHandler(),
+        ]
+
+    def run(self, transcription_results: List[TranscriptionResult], lyrics_results: List[LyricsData]) -> CorrectionResult:
+        """Execute the correction process."""
+        if not transcription_results:
+            self.logger.error("No transcription results available")
+            raise ValueError("No primary transcription data available")
+
+        # Get primary transcription
+        primary_transcription = sorted(transcription_results, key=lambda x: x.priority)[0].result
+        transcribed_text = " ".join(" ".join(w.text for w in segment.words) for segment in primary_transcription.segments)
+        reference_texts = {lyrics.source: lyrics.lyrics for lyrics in lyrics_results}
+
+        # Find anchor sequences and gaps
+        self.logger.debug("Finding anchor sequences and gaps")
+        anchor_sequences = self.anchor_finder.find_anchors(transcribed_text, reference_texts)
+        gap_sequences = self.anchor_finder.find_gaps(transcribed_text, anchor_sequences, reference_texts)
+
+        # Process corrections
+        corrections, corrected_segments = self._process_corrections(primary_transcription.segments, gap_sequences)
+
+        # Calculate correction ratio
+        total_words = sum(len(segment.words) for segment in corrected_segments)
+        corrections_made = len(corrections)
+        correction_ratio = 1 - (corrections_made / total_words if total_words > 0 else 0)
+
+        return CorrectionResult(
+            original_segments=primary_transcription.segments,
+            corrected_segments=corrected_segments,
+            corrected_text="\n".join(segment.text for segment in corrected_segments) + "\n",
+            corrections=corrections,
+            corrections_made=corrections_made,
+            confidence=correction_ratio,
+            transcribed_text=transcribed_text,
+            reference_texts=reference_texts,
+            anchor_sequences=anchor_sequences,
+            resized_segments=[],
+            gap_sequences=gap_sequences,
+            metadata={
+                "anchor_sequences_count": len(anchor_sequences),
+                "gap_sequences_count": len(gap_sequences),
+                "total_words": total_words,
+                "correction_ratio": correction_ratio,
+            },
+        )
+
+    def _preserve_formatting(self, original: str, new_word: str) -> str:
+        """Preserve original word's formatting when applying correction."""
+        # Find leading/trailing whitespace
+        leading_space = " " if original != original.lstrip() else ""
+        trailing_space = " " if original != original.rstrip() else ""
+        return leading_space + new_word.strip() + trailing_space
+
+    def _process_corrections(
+        self, segments: List[LyricsSegment], gap_sequences: List[GapSequence]
+    ) -> Tuple[List[WordCorrection], List[LyricsSegment]]:
+        """Process corrections using handlers.
+
+        The correction flow works as follows:
+        1. First pass: Process all gaps
+           - Iterate through each gap sequence
+           - Try handlers until one can handle the gap
+           - Store all corrections in the gap
+        2. Second pass: Apply corrections to segments
+           - Iterate through segments and words
+           - Look up any corrections that apply to each word
+           - Create new segments with corrected words
+
+        This two-pass approach separates the concerns of:
+        a) Finding and making corrections (gap-centric)
+        b) Applying those corrections to the original text (segment-centric)
+        """
+        self.logger.info(f"Starting correction process with {len(gap_sequences)} gaps")
+
+        # First pass: Process all gaps
+        all_corrections = self._process_gaps(gap_sequences)
+
+        # Second pass: Apply corrections to segments
+        corrected_segments = self._apply_corrections_to_segments(segments, all_corrections)
+
+        self.logger.info(f"Correction process complete. Made {len(all_corrections)} corrections")
+        return all_corrections, corrected_segments
+
+    def _process_gaps(self, gap_sequences: List[GapSequence]) -> List[WordCorrection]:
+        """Process each gap using available handlers until all words are corrected or no handlers remain."""
+        all_corrections = []
+
+        for gap in gap_sequences:
+            self.logger.debug(f"Processing gap: {gap.text}")
+            high_confidence_positions = set()  # Track positions that have high confidence corrections
+            corrected_positions = set()  # Track all corrected positions regardless of confidence
+
+            # Try each handler until gap is fully corrected
+            for handler in self.handlers:
+                # Skip if all words have high confidence corrections
+                uncorrected_positions = set(range(gap.transcription_position, gap.transcription_position + gap.length))
+                uncorrected_positions -= corrected_positions  # Skip any corrected positions
+
+                if not uncorrected_positions:
+                    self.logger.debug("All words have been corrected, skipping remaining handlers")
+                    break
+
+                self.logger.debug(f"Trying handler {handler.__class__.__name__}")
+
+                # Pass previous corrections to RepeatCorrectionHandler
+                if isinstance(handler, RepeatCorrectionHandler):
+                    handler.set_previous_corrections(all_corrections)
+
+                can_handle, handler_data = handler.can_handle(gap)
+                if can_handle:
+                    self.logger.debug(f"{handler.__class__.__name__} can handle gap")
+                    # Only pass handler_data if it's not empty
+                    corrections = handler.handle(gap, handler_data if handler_data else None)
+                    if corrections:
+                        # Add corrections to gap and track corrected positions
+                        for correction in corrections:
+                            # Skip if this position was already corrected
+                            if correction.original_position in corrected_positions:
+                                continue
+
+                            gap.add_correction(correction)
+                            corrected_positions.add(correction.original_position)
+                            # Track positions with high confidence corrections (>= 0.9)
+                            if correction.confidence >= 0.9:
+                                high_confidence_positions.add(correction.original_position)
+
+                        # Filter out corrections for already corrected positions
+                        new_corrections = [c for c in corrections if c.original_position in corrected_positions]
+                        if new_corrections:
+                            self.logger.debug(
+                                f"{handler.__class__.__name__} made {len(new_corrections)} corrections: "
+                                f"{[f'{c.original_word}->{c.corrected_word}' for c in new_corrections]}"
+                            )
+                            all_corrections.extend(new_corrections)
+
+                        # Log remaining uncorrected words
+                        if not gap.is_fully_corrected:
+                            uncorrected = [word for pos, word in gap.uncorrected_words if pos not in corrected_positions]
+                            if uncorrected:
+                                self.logger.debug(f"Uncorrected words remaining: {', '.join(uncorrected)}")
+
+            if not gap.corrections:
+                self.logger.warning("No handler could handle the gap")
+
+        return all_corrections
+
+    def _apply_corrections_to_segments(self, segments: List[LyricsSegment], corrections: List[WordCorrection]) -> List[LyricsSegment]:
+        """Apply corrections to create new segments."""
+        correction_map = {}
+        # Group corrections by original_position to handle splits
+        for c in corrections:
+            if c.original_position not in correction_map:
+                correction_map[c.original_position] = []
+            correction_map[c.original_position].append(c)
+
+        corrected_segments = []
+        current_word_idx = 0
+
+        for segment_idx, segment in enumerate(segments):
+            corrected_words = []
+            for word in segment.words:
+                if current_word_idx in correction_map:
+                    word_corrections = sorted(correction_map[current_word_idx], key=lambda x: x.split_index or 0)
+
+                    # Check if any correction has a valid split_total
+                    total_splits = next((c.split_total for c in word_corrections if c.split_total is not None), None)
+
+                    if total_splits:
+                        # Handle word split
+                        split_duration = (word.end_time - word.start_time) / total_splits
+
+                        for i, correction in enumerate(word_corrections):
+                            start_time = word.start_time + (i * split_duration)
+                            end_time = start_time + split_duration
+
+                            # Update corrected_position as we create new words
+                            correction.corrected_position = len(corrected_words)
+                            corrected_words.append(
+                                Word(
+                                    text=self._preserve_formatting(correction.original_word, correction.corrected_word),
+                                    start_time=start_time,
+                                    end_time=end_time,
+                                    confidence=correction.confidence,
+                                )
+                            )
+                    else:
+                        # Handle single word replacement
+                        correction = word_corrections[0]
+                        if not correction.is_deletion:
+                            # Update corrected_position
+                            correction.corrected_position = len(corrected_words)
+                            corrected_words.append(
+                                Word(
+                                    text=self._preserve_formatting(correction.original_word, correction.corrected_word),
+                                    start_time=word.start_time,
+                                    end_time=word.end_time,
+                                    confidence=correction.confidence,
+                                )
+                            )
+                else:
+                    corrected_words.append(word)
+                current_word_idx += 1
+
+            if corrected_words:
+                corrected_segments.append(
+                    LyricsSegment(
+                        text=" ".join(w.text for w in corrected_words),
+                        words=corrected_words,
+                        start_time=segment.start_time,
+                        end_time=segment.end_time,
+                    )
+                )
+
+        return corrected_segments

lyrics_transcriber/correction/handlers/base.py

@@ -0,0 +1,30 @@
+from abc import ABC, abstractmethod
+from typing import List, Optional, Tuple, Dict, Any
+
+from lyrics_transcriber.types import GapSequence, WordCorrection
+
+
+class GapCorrectionHandler(ABC):
+    """Base class for gap correction handlers."""
+
+    @abstractmethod
+    def can_handle(self, gap: GapSequence) -> Tuple[bool, Dict[str, Any]]:
+        """Determine if this handler can process the given gap.
+
+        Returns:
+            Tuple containing:
+            - bool: Whether this handler can process the gap
+            - dict: Data computed during can_handle that will be needed by handle().
+                   Empty dict if no data needs to be passed.
+        """
+        pass
+
+    @abstractmethod
+    def handle(self, gap: GapSequence, data: Optional[Dict[str, Any]] = None) -> List[WordCorrection]:
+        """Process a gap and return any corrections.
+
+        Args:
+            gap: The gap sequence to process
+            data: Optional data dictionary returned by can_handle()
+        """
+        pass

lyrics_transcriber/correction/handlers/extend_anchor.py

@@ -0,0 +1,91 @@
+from typing import List, Optional, Tuple, Dict, Any
+
+from lyrics_transcriber.types import GapSequence, WordCorrection
+from lyrics_transcriber.correction.handlers.base import GapCorrectionHandler
+from lyrics_transcriber.correction.handlers.word_operations import WordOperations
+
+
+class ExtendAnchorHandler(GapCorrectionHandler):
+    """Handles gaps where some words match reference text but there are extra words.
+
+    This handler looks for cases where:
+    1. One or more words in the gap match words in the same position in at least one reference source
+    2. The gap may contain additional words that aren't in the reference
+
+    When such matches are found, it:
+    1. Validates all matching words (creates corrections that keep the same words)
+    2. Leaves all non-matching words unchanged for other handlers to process
+
+    The confidence of validations is based on the ratio of reference sources that agree.
+    For example, if 2 out of 4 sources have the matching word, confidence will be 0.5.
+
+    Examples:
+        Gap: "hello world extra words"
+        References:
+            genius: ["hello", "world"]
+            spotify: ["hello", "world"]
+        Result:
+            - Validate "hello" (confidence=1.0)
+            - Validate "world" (confidence=1.0)
+            - Leave "extra" and "words" unchanged
+
+        Gap: "martyr youre a"
+        References:
+            genius: ["martyr"]
+            spotify: ["mother"]
+        Result:
+            - Validate "martyr" (confidence=0.5, source="genius")
+            - Leave "youre" and "a" unchanged
+    """
+
+    def can_handle(self, gap: GapSequence) -> Tuple[bool, Dict[str, Any]]:
+        # Must have reference words
+        if not gap.reference_words:
+            return False, {}
+
+        # Gap must have words
+        if not gap.words:
+            return False, {}
+
+        # At least one word must match between gap and any reference source
+        # in the same position
+        has_match = any(
+            i < len(ref_words) and gap.words[i].lower() == ref_words[i].lower()
+            for ref_words in gap.reference_words.values()
+            for i in range(min(len(gap.words), len(ref_words)))
+        )
+
+        return has_match, {}
+
+    def handle(self, gap: GapSequence, data: Optional[Dict[str, Any]] = None) -> List[WordCorrection]:
+        corrections = []
+
+        # Process each word in the gap that has a corresponding reference position
+        for i, word in enumerate(gap.words):
+            # Find reference sources that have a matching word at this position
+            matching_sources = [
+                source for source, ref_words in gap.reference_words.items() if i < len(ref_words) and word.lower() == ref_words[i].lower()
+            ]
+
+            if matching_sources:
+                # Word matches reference(s) at this position - validate it
+                confidence = len(matching_sources) / len(gap.reference_words)
+                sources = ", ".join(matching_sources)
+
+                # Calculate reference positions for matching sources
+                reference_positions = WordOperations.calculate_reference_positions(gap, matching_sources)
+
+                corrections.append(
+                    WordOperations.create_word_replacement_correction(
+                        original_word=word,
+                        corrected_word=word,  # Same word, just validating
+                        original_position=gap.transcription_position + i,
+                        source=sources,
+                        confidence=confidence,
+                        reason="ExtendAnchorHandler: Matched reference source(s)",
+                        reference_positions=reference_positions,
+                    )
+                )
+            # No else clause - non-matching words are left unchanged
+
+        return corrections

lyrics_transcriber/correction/handlers/levenshtein.py

@@ -0,0 +1,147 @@
+from typing import List, Optional, Tuple, Dict, Any
+import string
+import Levenshtein
+import logging
+
+from lyrics_transcriber.types import GapSequence, WordCorrection
+from lyrics_transcriber.correction.handlers.base import GapCorrectionHandler
+from lyrics_transcriber.correction.handlers.word_operations import WordOperations
+
+
+class LevenshteinHandler(GapCorrectionHandler):
+    """Handles corrections based on Levenshtein (edit distance) similarity between words.
+
+    This handler looks for words that are similar in spelling to reference words in the same position.
+    The similarity calculation includes:
+    1. Basic Levenshtein ratio
+    2. Bonus for words starting with the same letter
+    3. Penalty for words starting with different letters
+    4. Bonus for similar length words
+
+    Examples:
+        Gap: "wold" (misspelling)
+        References:
+            genius: ["world"]
+            spotify: ["world"]
+        Result:
+            - Correct "wold" to "world" (high confidence due to small edit distance)
+
+        Gap: "worde" (misspelling)
+        References:
+            genius: ["world"]
+            spotify: ["words"]
+        Result:
+            - Correct "worde" to "world" (lower confidence due to disagreeing sources)
+    """
+
+    def __init__(self, similarity_threshold: float = 0.65, logger: Optional[logging.Logger] = None):
+        self.similarity_threshold = similarity_threshold
+        self.logger = logger or logging.getLogger(__name__)
+
+    def can_handle(self, gap: GapSequence) -> Tuple[bool, Dict[str, Any]]:
+        """Check if we can handle this gap - we'll try if there are reference words."""
+        if not gap.reference_words:
+            self.logger.debug("No reference words available")
+            return False, {}
+
+        if not gap.words:
+            self.logger.debug("No gap words available")
+            return False, {}
+
+        # Check if any word has sufficient similarity to reference
+        for i, word in enumerate(gap.words):
+            for ref_words in gap.reference_words.values():
+                if i < len(ref_words):
+                    similarity = self._get_string_similarity(word, ref_words[i])
+                    if similarity >= self.similarity_threshold:
+                        self.logger.debug(f"Found similar word: '{word}' -> '{ref_words[i]}' ({similarity:.2f})")
+                        return True, {}
+
+        self.logger.debug("No words meet similarity threshold")
+        return False, {}
+
+    def handle(self, gap: GapSequence, data: Optional[Dict[str, Any]] = None) -> List[WordCorrection]:
+        """Try to correct words based on string similarity."""
+        corrections = []
+
+        # Process each word in the gap
+        for i, word in enumerate(gap.words):
+            # Skip if word is empty or just punctuation
+            if not word.strip():
+                continue
+
+            # Skip exact matches
+            if any(i < len(ref_words) and word.lower() == ref_words[i].lower() for ref_words in gap.reference_words.values()):
+                self.logger.debug(f"Skipping exact match: '{word}'")
+                continue
+
+            # Find matching reference words at this position
+            matches = {}  # word -> (sources, similarity)
+            for source, ref_words in gap.reference_words.items():
+                ref_words_original = gap.reference_words_original[source]  # Get original formatted words
+                if i >= len(ref_words):
+                    continue
+
+                ref_word = ref_words[i]
+                ref_word_original = ref_words_original[i]  # Get original formatted word
+                similarity = self._get_string_similarity(word, ref_word)
+
+                if similarity >= self.similarity_threshold:
+                    self.logger.debug(f"Found match: '{word}' -> '{ref_word}' ({similarity:.2f})")
+                    if ref_word_original not in matches:  # Use original formatted word as key
+                        matches[ref_word_original] = ([], similarity)
+                    matches[ref_word_original][0].append(source)
+
+            # Create correction for best match if any found
+            if matches:
+                best_match, (sources, similarity) = max(
+                    matches.items(), key=lambda x: (len(x[1][0]), x[1][1])  # Sort by number of sources, then similarity
+                )
+
+                source_confidence = len(sources) / len(gap.reference_words)
+                final_confidence = similarity * source_confidence
+
+                # Calculate reference positions for matching sources
+                reference_positions = WordOperations.calculate_reference_positions(gap, sources)
+
+                self.logger.debug(f"Creating correction: {word} -> {best_match} (confidence: {final_confidence})")
+                corrections.append(
+                    WordOperations.create_word_replacement_correction(
+                        original_word=word,
+                        corrected_word=best_match,  # Using original formatted word
+                        original_position=gap.transcription_position + i,
+                        source=", ".join(sources),
+                        confidence=final_confidence,
+                        reason=f"LevenshteinHandler: String similarity ({final_confidence:.2f})",
+                        reference_positions=reference_positions,
+                    )
+                )
+
+        return corrections
+
+    def _clean_word(self, word: str) -> str:
+        """Remove punctuation and standardize for comparison."""
+        return word.strip().lower().strip(string.punctuation)
+
+    def _get_string_similarity(self, word1: str, word2: str) -> float:
+        """Calculate string similarity using Levenshtein ratio with adjustments."""
+        # Clean words
+        w1, w2 = self._clean_word(word1), self._clean_word(word2)
+        if not w1 or not w2:
+            return 0.0
+
+        # Calculate Levenshtein ratio
+        similarity = Levenshtein.ratio(w1, w2)
+
+        # Boost similarity for words starting with the same letter
+        if w1[0] == w2[0]:
+            similarity = (similarity + 1) / 2
+        else:
+            # Penalize words starting with different letters
+            similarity = similarity * 0.9
+
+        # Boost for similar length words
+        length_ratio = min(len(w1), len(w2)) / max(len(w1), len(w2))
+        similarity = (similarity + length_ratio) / 2
+
+        return similarity

lyrics_transcriber/correction/handlers/no_space_punct_match.py

@@ -0,0 +1,98 @@
+from typing import List, Optional, Tuple, Dict, Any
+import re
+
+from lyrics_transcriber.types import GapSequence, WordCorrection
+from lyrics_transcriber.correction.handlers.base import GapCorrectionHandler
+from lyrics_transcriber.correction.handlers.word_operations import WordOperations
+
+
+class NoSpacePunctuationMatchHandler(GapCorrectionHandler):
+    """Handles gaps where reference text matches when spaces and punctuation are removed."""
+
+    def _remove_spaces_and_punct(self, words: List[str]) -> str:
+        """Join words and remove all whitespace and punctuation."""
+        text = "".join(words).lower()
+        # Remove all punctuation including apostrophes
+        return re.sub(r"[^\w\s]", "", text)
+
+    def can_handle(self, gap: GapSequence) -> Tuple[bool, Dict[str, Any]]:
+        # Must have reference words
+        if not gap.reference_words:
+            return False, {}
+
+        # Get the gap text without spaces and punctuation
+        gap_text = self._remove_spaces_and_punct(gap.words)
+
+        # Check if any reference source matches when spaces and punctuation are removed
+        for words in gap.reference_words.values():
+            ref_text = self._remove_spaces_and_punct(words)
+            if gap_text == ref_text:
+                return True, {}
+
+        return False, {}
+
+    def handle(self, gap: GapSequence, data: Optional[Dict[str, Any]] = None) -> List[WordCorrection]:
+        corrections = []
+
+        # Find the matching source (we know there is at least one from can_handle)
+        gap_text = self._remove_spaces_and_punct(gap.words)
+        matching_source = None
+        reference_words = None
+        reference_words_original = None
+        for source, words in gap.reference_words.items():
+            if self._remove_spaces_and_punct(words) == gap_text:
+                matching_source = source
+                reference_words = words
+                reference_words_original = gap.reference_words_original[source]
+                break
+
+        # Calculate reference positions for the matching source
+        reference_positions = WordOperations.calculate_reference_positions(gap, [matching_source])
+
+        # Handle cases where number of words differ
+        if len(gap.words) > len(reference_words):
+            # Multiple transcribed words -> fewer reference words
+            corrections.extend(
+                WordOperations.create_word_combine_corrections(
+                    original_words=gap.words,
+                    reference_word=reference_words_original[0],
+                    original_position=gap.transcription_position,
+                    source=matching_source,
+                    confidence=1.0,
+                    combine_reason="NoSpacePunctuationMatchHandler: Words combined based on text match",
+                    delete_reason="NoSpacePunctuationMatchHandler: Word removed as part of text match combination",
+                    reference_positions=reference_positions,
+                )
+            )
+
+        elif len(gap.words) < len(reference_words):
+            # Single transcribed word -> multiple reference words
+            corrections.extend(
+                WordOperations.create_word_split_corrections(
+                    original_word=gap.words[0],
+                    reference_words=reference_words_original,
+                    original_position=gap.transcription_position,
+                    source=matching_source,
+                    confidence=1.0,
+                    reason="NoSpacePunctuationMatchHandler: Split word based on text match",
+                    reference_positions=reference_positions,
+                )
+            )
+
+        else:
+            # One-to-one replacement
+            for i, (orig_word, ref_word, ref_word_original) in enumerate(zip(gap.words, reference_words, reference_words_original)):
+                if orig_word.lower() != ref_word.lower():
+                    corrections.append(
+                        WordOperations.create_word_replacement_correction(
+                            original_word=orig_word,
+                            corrected_word=ref_word_original,
+                            original_position=gap.transcription_position + i,
+                            source=matching_source,
+                            confidence=1.0,
+                            reason=f"NoSpacePunctuationMatchHandler: Source '{matching_source}' matched when spaces and punctuation removed",
+                            reference_positions=reference_positions,
+                        )
+                    )
+
+        return corrections