karaoke-gen 0.75.54__py3-none-any.whl

lyrics_transcriber/correction/feedback/__init__.py

@@ -0,0 +1,2 @@
+"""Human feedback collection system for continuous improvement."""
+

lyrics_transcriber/correction/feedback/schemas.py

@@ -0,0 +1,107 @@
+"""Schemas for correction annotations and human feedback."""
+
+from __future__ import annotations
+
+from typing import Optional, List, Dict, Any
+from pydantic import BaseModel, Field
+from enum import Enum
+from datetime import datetime
+import uuid
+
+
+class CorrectionAnnotationType(str, Enum):
+    """Types of corrections that can be annotated."""
+    PUNCTUATION_ONLY = "PUNCTUATION_ONLY"
+    SOUND_ALIKE = "SOUND_ALIKE"
+    BACKGROUND_VOCALS = "BACKGROUND_VOCALS"
+    EXTRA_WORDS = "EXTRA_WORDS"
+    REPEATED_SECTION = "REPEATED_SECTION"
+    COMPLEX_MULTI_ERROR = "COMPLEX_MULTI_ERROR"
+    AMBIGUOUS = "AMBIGUOUS"
+    NO_ERROR = "NO_ERROR"
+    MANUAL_EDIT = "MANUAL_EDIT"  # Human-initiated edit not from gap
+
+
+class CorrectionAction(str, Enum):
+    """Actions that can be taken for corrections."""
+    NO_ACTION = "NO_ACTION"
+    REPLACE = "REPLACE"
+    DELETE = "DELETE"
+    INSERT = "INSERT"
+    MERGE = "MERGE"
+    SPLIT = "SPLIT"
+    FLAG = "FLAG"
+
+
+class CorrectionAnnotation(BaseModel):
+    """Annotation for a manual correction made by a human."""
+    
+    annotation_id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique identifier")
+    audio_hash: str = Field(..., description="Hash of the audio file")
+    gap_id: Optional[str] = Field(None, description="Gap ID if this correction is for a gap")
+    
+    # Classification
+    annotation_type: CorrectionAnnotationType = Field(..., description="Type of correction")
+    action_taken: CorrectionAction = Field(..., description="Action that was taken")
+    
+    # Content
+    original_text: str = Field(..., description="Original transcribed text")
+    corrected_text: str = Field(..., description="Corrected text after human edit")
+    
+    # Metadata
+    confidence: float = Field(..., ge=1.0, le=5.0, description="Human confidence rating (1-5)")
+    reasoning: str = Field(..., min_length=10, description="Human explanation for the correction")
+    word_ids_affected: List[str] = Field(default_factory=list, description="Word IDs involved in correction")
+    
+    # Agentic AI comparison
+    agentic_proposal: Optional[Dict[str, Any]] = Field(None, description="What the AI suggested (if applicable)")
+    agentic_category: Optional[str] = Field(None, description="Category the AI classified this as")
+    agentic_agreed: bool = Field(False, description="Whether human agreed with AI proposal")
+    
+    # Reference lyrics
+    reference_sources_consulted: List[str] = Field(default_factory=list, description="Which reference sources were used")
+    
+    # Song metadata
+    artist: str = Field(..., description="Song artist")
+    title: str = Field(..., description="Song title")
+    session_id: str = Field(..., description="Correction session ID")
+    
+    # Timestamp
+    timestamp: datetime = Field(default_factory=datetime.utcnow, description="When annotation was created")
+    
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "annotation_id": "550e8400-e29b-41d4-a716-446655440000",
+                "audio_hash": "abc123",
+                "gap_id": "gap_1",
+                "annotation_type": "sound_alike",
+                "action_taken": "REPLACE",
+                "original_text": "out I'm starting over",
+                "corrected_text": "now I'm starting over",
+                "confidence": 5.0,
+                "reasoning": "The word 'out' sounds like 'now' but the reference lyrics and context make it clear it should be 'now'",
+                "word_ids_affected": ["word_123"],
+                "agentic_proposal": {"action": "ReplaceWord", "replacement_text": "now"},
+                "agentic_category": "sound_alike",
+                "agentic_agreed": True,
+                "reference_sources_consulted": ["genius", "spotify"],
+                "artist": "Rancid",
+                "title": "Time Bomb",
+                "session_id": "session_abc",
+                "timestamp": "2025-01-01T12:00:00"
+            }
+        }
+
+
+class AnnotationStatistics(BaseModel):
+    """Aggregated statistics from annotations."""
+    
+    total_annotations: int = 0
+    annotations_by_type: Dict[str, int] = Field(default_factory=dict)
+    annotations_by_action: Dict[str, int] = Field(default_factory=dict)
+    average_confidence: float = 0.0
+    agentic_agreement_rate: float = 0.0
+    most_common_errors: List[Dict[str, Any]] = Field(default_factory=list)
+    songs_annotated: int = 0
+

lyrics_transcriber/correction/feedback/store.py

@@ -0,0 +1,236 @@
+"""Storage backend for correction annotations."""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+from datetime import datetime
+from collections import Counter, defaultdict
+
+from .schemas import CorrectionAnnotation, AnnotationStatistics
+
+logger = logging.getLogger(__name__)
+
+
+class FeedbackStore:
+    """Stores correction annotations in JSONL format."""
+    
+    def __init__(self, storage_dir: str = "cache"):
+        """Initialize feedback store.
+        
+        Args:
+            storage_dir: Directory to store annotations file
+        """
+        self.storage_dir = Path(storage_dir)
+        self.storage_dir.mkdir(parents=True, exist_ok=True)
+        self.annotations_file = self.storage_dir / "correction_annotations.jsonl"
+        
+        # Ensure file exists
+        if not self.annotations_file.exists():
+            self.annotations_file.touch()
+            logger.info(f"Created annotations file: {self.annotations_file}")
+    
+    def save_annotation(self, annotation: CorrectionAnnotation) -> bool:
+        """Save a single annotation to the JSONL file.
+        
+        Args:
+            annotation: CorrectionAnnotation to save
+        
+        Returns:
+            True if successful, False otherwise
+        """
+        try:
+            # Convert to dict and handle datetime serialization
+            data = annotation.model_dump()
+            data['timestamp'] = data['timestamp'].isoformat()
+            
+            # Append to JSONL file
+            with open(self.annotations_file, 'a', encoding='utf-8') as f:
+                f.write(json.dumps(data, ensure_ascii=False) + '\n')
+            
+            logger.debug(f"Saved annotation {annotation.annotation_id}")
+            return True
+            
+        except Exception as e:
+            logger.error(f"Failed to save annotation: {e}")
+            return False
+    
+    def save_annotations(self, annotations: List[CorrectionAnnotation]) -> int:
+        """Save multiple annotations.
+        
+        Args:
+            annotations: List of annotations to save
+        
+        Returns:
+            Number of annotations successfully saved
+        """
+        saved = 0
+        for annotation in annotations:
+            if self.save_annotation(annotation):
+                saved += 1
+        return saved
+    
+    def get_all_annotations(self) -> List[CorrectionAnnotation]:
+        """Load all annotations from the JSONL file.
+        
+        Returns:
+            List of CorrectionAnnotation objects
+        """
+        annotations = []
+        
+        if not self.annotations_file.exists():
+            return annotations
+        
+        try:
+            with open(self.annotations_file, 'r', encoding='utf-8') as f:
+                for line_num, line in enumerate(f, 1):
+                    line = line.strip()
+                    if not line:
+                        continue
+                    
+                    try:
+                        data = json.loads(line)
+                        # Parse timestamp if string
+                        if isinstance(data.get('timestamp'), str):
+                            data['timestamp'] = datetime.fromisoformat(data['timestamp'])
+                        
+                        annotation = CorrectionAnnotation.model_validate(data)
+                        annotations.append(annotation)
+                        
+                    except Exception as e:
+                        logger.warning(f"Failed to parse annotation on line {line_num}: {e}")
+                        continue
+            
+            logger.debug(f"Loaded {len(annotations)} annotations")
+            return annotations
+            
+        except Exception as e:
+            logger.error(f"Failed to load annotations: {e}")
+            return []
+    
+    def get_annotations_by_song(self, audio_hash: str) -> List[CorrectionAnnotation]:
+        """Get all annotations for a specific song.
+        
+        Args:
+            audio_hash: Hash of the audio file
+        
+        Returns:
+            List of annotations for that song
+        """
+        all_annotations = self.get_all_annotations()
+        return [a for a in all_annotations if a.audio_hash == audio_hash]
+    
+    def get_annotations_by_category(self, category: str) -> List[CorrectionAnnotation]:
+        """Get all annotations of a specific type.
+        
+        Args:
+            category: Annotation type category
+        
+        Returns:
+            List of annotations of that type
+        """
+        all_annotations = self.get_all_annotations()
+        return [a for a in all_annotations if a.annotation_type == category]
+    
+    def get_statistics(self) -> AnnotationStatistics:
+        """Generate aggregated statistics from all annotations.
+        
+        Returns:
+            AnnotationStatistics object with aggregated data
+        """
+        annotations = self.get_all_annotations()
+        
+        if not annotations:
+            return AnnotationStatistics()
+        
+        # Count by type
+        type_counts = Counter(a.annotation_type for a in annotations)
+        
+        # Count by action
+        action_counts = Counter(a.action_taken for a in annotations)
+        
+        # Average confidence
+        avg_confidence = sum(a.confidence for a in annotations) / len(annotations)
+        
+        # Agentic agreement rate
+        agentic_proposals = [a for a in annotations if a.agentic_proposal is not None]
+        if agentic_proposals:
+            agentic_agreement_rate = sum(1 for a in agentic_proposals if a.agentic_agreed) / len(agentic_proposals)
+        else:
+            agentic_agreement_rate = 0.0
+        
+        # Most common error patterns
+        error_patterns = defaultdict(list)
+        for a in annotations:
+            if a.action_taken != "NO_ACTION":
+                pattern = f"{a.original_text} -> {a.corrected_text}"
+                error_patterns[pattern].append(a)
+        
+        most_common = [
+            {
+                "pattern": pattern,
+                "count": len(anns),
+                "annotation_type": anns[0].annotation_type
+            }
+            for pattern, anns in sorted(error_patterns.items(), key=lambda x: len(x[1]), reverse=True)[:10]
+        ]
+        
+        # Unique songs
+        unique_hashes = set(a.audio_hash for a in annotations)
+        
+        return AnnotationStatistics(
+            total_annotations=len(annotations),
+            annotations_by_type={k: v for k, v in type_counts.items()},
+            annotations_by_action={k: v for k, v in action_counts.items()},
+            average_confidence=avg_confidence,
+            agentic_agreement_rate=agentic_agreement_rate,
+            most_common_errors=most_common,
+            songs_annotated=len(unique_hashes)
+        )
+    
+    def export_to_training_data(self, output_file: Optional[Path] = None) -> Path:
+        """Export annotations in a format suitable for model fine-tuning.
+        
+        Args:
+            output_file: Optional path for output file
+        
+        Returns:
+            Path to the exported file
+        """
+        if output_file is None:
+            output_file = self.storage_dir / "training_data.jsonl"
+        
+        annotations = self.get_all_annotations()
+        
+        # Filter to high-confidence annotations (4-5 rating)
+        high_confidence = [a for a in annotations if a.confidence >= 4.0]
+        
+        with open(output_file, 'w', encoding='utf-8') as f:
+            for annotation in high_confidence:
+                # Create a training example with input/output format
+                training_example = {
+                    "input": {
+                        "original_text": annotation.original_text,
+                        "annotation_type": annotation.annotation_type,
+                        "artist": annotation.artist,
+                        "title": annotation.title,
+                        "reference_sources": annotation.reference_sources_consulted
+                    },
+                    "output": {
+                        "action": annotation.action_taken,
+                        "corrected_text": annotation.corrected_text,
+                        "reasoning": annotation.reasoning
+                    },
+                    "metadata": {
+                        "confidence": annotation.confidence,
+                        "annotation_id": annotation.annotation_id,
+                        "timestamp": annotation.timestamp.isoformat()
+                    }
+                }
+                f.write(json.dumps(training_example, ensure_ascii=False) + '\n')
+        
+        logger.info(f"Exported {len(high_confidence)} training examples to {output_file}")
+        return output_file
+

lyrics_transcriber/correction/handlers/base.py

@@ -0,0 +1,52 @@
+from abc import ABC, abstractmethod
+from typing import List, Optional, Tuple, Dict, Any
+import logging
+
+from lyrics_transcriber.types import GapSequence, WordCorrection
+
+
+class GapCorrectionHandler(ABC):
+    """Base class for gap correction handlers."""
+
+    def __init__(self, logger: Optional[logging.Logger] = None):
+        self.logger = logger or logging.getLogger(__name__)
+
+    @abstractmethod
+    def can_handle(self, gap: GapSequence, data: Optional[Dict[str, Any]] = None) -> Tuple[bool, Dict[str, Any]]:
+        """Check if this handler can process the given gap.
+        
+        Args:
+            gap: The gap sequence to check
+            data: Optional dictionary containing additional data like word_map
+            
+        Returns:
+            Tuple of (can_handle, handler_data)
+        """
+        pass
+
+    @abstractmethod
+    def handle(self, gap: GapSequence, data: Optional[Dict[str, Any]] = None) -> List[WordCorrection]:
+        """Process the gap and return any corrections.
+        
+        Args:
+            gap: The gap sequence to process
+            data: Optional dictionary containing additional data like word_map
+            
+        Returns:
+            List of corrections to apply
+        """
+        pass
+
+    def _validate_data(self, data: Optional[Dict[str, Any]]) -> bool:
+        """Validate that required data is present.
+        
+        Args:
+            data: The data dictionary to validate
+            
+        Returns:
+            True if data is valid, False otherwise
+        """
+        if not data or "word_map" not in data:
+            self.logger.error("No word_map provided in data")
+            return False
+        return True

lyrics_transcriber/correction/handlers/extend_anchor.py

@@ -0,0 +1,149 @@
+from typing import List, Optional, Tuple, Dict, Any
+import logging
+
+from lyrics_transcriber.types import GapSequence, WordCorrection, Word
+from lyrics_transcriber.correction.handlers.base import GapCorrectionHandler
+from lyrics_transcriber.correction.handlers.word_operations import WordOperations
+
+
+class ExtendAnchorHandler(GapCorrectionHandler):
+    """Handles gaps where some words match reference text but there are extra words.
+
+    This handler looks for cases where:
+    1. One or more words in the gap match words in the same position in at least one reference source
+    2. The gap may contain additional words that aren't in the reference
+
+    When such matches are found, it:
+    1. Validates all matching words (creates corrections that keep the same words)
+    2. Leaves all non-matching words unchanged for other handlers to process
+
+    The confidence of validations is based on the ratio of reference sources that agree.
+    For example, if 2 out of 4 sources have the matching word, confidence will be 0.5.
+
+    Examples:
+        Gap: "hello world extra words"
+        References:
+            genius: ["hello", "world"]
+            spotify: ["hello", "world"]
+        Result:
+            - Validate "hello" (confidence=1.0)
+            - Validate "world" (confidence=1.0)
+            - Leave "extra" and "words" unchanged
+
+        Gap: "martyr youre a"
+        References:
+            genius: ["martyr"]
+            spotify: ["mother"]
+        Result:
+            - Validate "martyr" (confidence=0.5, source="genius")
+            - Leave "youre" and "a" unchanged
+    """
+
+    def __init__(self, logger: Optional[logging.Logger] = None):
+        self.logger = logger or logging.getLogger(__name__)
+
+    def can_handle(self, gap: GapSequence, data: Optional[Dict[str, Any]] = None) -> Tuple[bool, Dict[str, Any]]:
+        """Check if this gap can be handled by extending anchor sequences."""
+        # Must have reference word IDs
+        if not gap.reference_word_ids:
+            self.logger.debug("No reference word IDs available.")
+            return False, {}
+
+        # Gap must have word IDs
+        if not gap.transcribed_word_ids:
+            self.logger.debug("No word IDs in the gap to process.")
+            return False, {}
+
+        # Must have word map to resolve IDs to actual words
+        if not self._validate_data(data):
+            return False, {}
+            
+        word_map = data["word_map"]
+
+        # At least one word must match between gap and any reference source by text content
+        has_match = False
+        for i, trans_word_id in enumerate(gap.transcribed_word_ids):
+            if trans_word_id not in word_map:
+                continue
+            trans_word = word_map[trans_word_id]
+            
+            # Check if this word matches any reference word at the same position
+            for ref_word_ids in gap.reference_word_ids.values():
+                if i < len(ref_word_ids):
+                    ref_word_id = ref_word_ids[i]
+                    if ref_word_id in word_map:
+                        ref_word = word_map[ref_word_id]
+                        if trans_word.text.lower() == ref_word.text.lower():
+                            has_match = True
+                            break
+            if has_match:
+                break
+
+        self.logger.debug(f"Can handle gap: {has_match}")
+        return has_match, {"word_map": word_map}
+
+    def handle(self, gap: GapSequence, data: Optional[Dict[str, Any]] = None) -> List[WordCorrection]:
+        corrections = []
+
+        # Get word lookup map from data
+        if not self._validate_data(data):
+            return []
+            
+        word_map = data["word_map"]
+
+        # Process each word in the gap that has a corresponding reference position
+        for i, word_id in enumerate(gap.transcribed_word_ids):
+            # Get the actual word object
+            if word_id not in word_map:
+                self.logger.error(f"Word ID {word_id} not found in word_map")
+                continue
+            word = word_map[word_id]
+
+            # Find reference sources that have a matching word (by text) at this position
+            matching_sources = []
+            corrected_word_id = None
+            
+            for source, ref_word_ids in gap.reference_word_ids.items():
+                if i < len(ref_word_ids):
+                    ref_word_id = ref_word_ids[i]
+                    if ref_word_id in word_map:
+                        ref_word = word_map[ref_word_id]
+                        if word.text.lower() == ref_word.text.lower():
+                            matching_sources.append(source)
+                            if corrected_word_id is None:
+                                corrected_word_id = ref_word_id
+
+            if not matching_sources:
+                self.logger.debug(f"Skipping word '{word.text}' at position {i} - no matching references")
+                continue
+
+            # Word matches reference(s) at this position - validate it
+            confidence = len(matching_sources) / len(gap.reference_word_ids)
+            sources = ", ".join(matching_sources)
+
+            # Get base reference positions
+            base_reference_positions = WordOperations.calculate_reference_positions(gap, matching_sources)
+
+            # Adjust reference positions based on the word's position in the reference text
+            reference_positions = {}
+            for source in matching_sources:
+                if source in base_reference_positions:
+                    reference_positions[source] = base_reference_positions[source] + i
+
+            corrections.append(
+                WordOperations.create_word_replacement_correction(
+                    original_word=word.text,
+                    corrected_word=word.text,
+                    original_position=gap.transcription_position + i,
+                    source=sources,
+                    confidence=confidence,
+                    reason="Matched reference source(s)",
+                    reference_positions=reference_positions,
+                    handler="ExtendAnchorHandler",
+                    original_word_id=word_id,
+                    corrected_word_id=corrected_word_id,
+                )
+            )
+            self.logger.debug(f"Validated word '{word.text}' with confidence {confidence} from sources: {sources}")
+
+        return corrections