karaoke-gen 0.75.54__py3-none-any.whl

lyrics_transcriber/storage/dropbox.py

@@ -0,0 +1,225 @@
+from dataclasses import dataclass
+from typing import Protocol, BinaryIO, Optional, List, Any
+import os
+import time
+import logging
+import requests
+from dropbox import Dropbox
+from dropbox.files import WriteMode, FileMetadata
+from dropbox.sharing import RequestedVisibility, SharedLinkSettings
+from dropbox.exceptions import AuthError, ApiError
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class DropboxConfig:
+    """Configuration for Dropbox client."""
+
+    app_key: Optional[str] = None
+    app_secret: Optional[str] = None
+    refresh_token: Optional[str] = None
+
+    @classmethod
+    def from_env(cls) -> "DropboxConfig":
+        """Create config from environment variables."""
+        return cls(
+            app_key=os.environ.get("WHISPER_DROPBOX_APP_KEY"),
+            app_secret=os.environ.get("WHISPER_DROPBOX_APP_SECRET"),
+            refresh_token=os.environ.get("WHISPER_DROPBOX_REFRESH_TOKEN"),
+        )
+
+
+class DropboxAPI(Protocol):
+    """Protocol for Dropbox API operations."""
+
+    def files_upload(self, f: bytes, path: str, mode: WriteMode) -> Any: ...
+    def files_list_folder(self, path: str, recursive: bool = False) -> Any: ...
+    def files_list_folder_continue(self, cursor: str) -> Any: ...
+    def files_download(self, path: str) -> tuple[Any, Any]: ...
+    def files_download_to_file(self, download_path: str, path: str) -> None: ...
+    def files_get_metadata(self, path: str) -> Any: ...
+    def sharing_create_shared_link_with_settings(self, path: str, settings: SharedLinkSettings) -> Any: ...
+    def sharing_list_shared_links(self, path: str) -> Any: ...
+
+
+class DropboxHandler:
+    """Handles Dropbox storage operations with automatic token refresh."""
+
+    def __init__(
+        self,
+        config: Optional[DropboxConfig] = None,
+        client: Optional[DropboxAPI] = None,
+    ):
+        """Initialize the Dropbox handler."""
+        self.config = config or DropboxConfig.from_env()
+        self._validate_config()
+
+        self.client = client or Dropbox(
+            app_key=self.config.app_key,
+            app_secret=self.config.app_secret,
+            oauth2_refresh_token=self.config.refresh_token,
+        )
+
+    def _validate_config(self) -> None:
+        """Validate the configuration."""
+        logger.debug("Validating DropboxConfig with values:")
+        logger.debug(f"app_key: {self.config.app_key[:4] + '...' if self.config.app_key else 'None'}")
+        logger.debug(f"app_secret: {self.config.app_secret[:4] + '...' if self.config.app_secret else 'None'}")
+        logger.debug(f"refresh_token: {self.config.refresh_token[:4] + '...' if self.config.refresh_token else 'None'}")
+
+        missing = []
+        if not self.config.app_key:
+            missing.append("app_key")
+        if not self.config.app_secret:
+            missing.append("app_secret")
+        if not self.config.refresh_token:
+            missing.append("refresh_token")
+
+        if missing:
+            error_msg = f"Missing required Dropbox configuration: {', '.join(missing)}"
+            logger.error(error_msg)
+            raise ValueError(error_msg)
+
+    def upload_with_retry(self, file: BinaryIO, path: str, max_retries: int = 3) -> None:
+        """Upload a file to Dropbox with retries."""
+        for attempt in range(max_retries):
+            try:
+                logger.debug(f"Attempting file upload to {path} (attempt {attempt + 1}/{max_retries})")
+                file.seek(0)
+                self.client.files_upload(file.read(), path, mode=WriteMode.overwrite)
+                logger.debug(f"Successfully uploaded file to {path}")
+                return
+            except ApiError as e:
+                logger.warning(f"Upload attempt {attempt + 1} failed: {str(e)}")
+                if attempt == max_retries - 1:
+                    logger.error(f"All upload attempts failed for {path}")
+                    raise
+                time.sleep(1 * (attempt + 1))
+
+    def upload_string_with_retry(self, content: str, path: str, max_retries: int = 3) -> None:
+        """Upload a string content to Dropbox with retries."""
+        for attempt in range(max_retries):
+            try:
+                logger.debug(f"Attempting string upload to {path} (attempt {attempt + 1}/{max_retries})")
+                self.client.files_upload(content.encode(), path, mode=WriteMode.overwrite)
+                logger.debug(f"Successfully uploaded string content to {path}")
+                return
+            except ApiError as e:
+                logger.warning(f"Upload attempt {attempt + 1} failed: {str(e)}")
+                if attempt == max_retries - 1:
+                    logger.error(f"All upload attempts failed for {path}")
+                    raise
+                time.sleep(1 * (attempt + 1))
+
+    def list_folder_recursive(self, path: str = "") -> List[FileMetadata]:
+        """List all files in a folder recursively."""
+        try:
+            logger.debug(f"Listing files recursively from {path}")
+            entries = []
+            result = self.client.files_list_folder(path, recursive=True)
+
+            while True:
+                entries.extend(result.entries)
+                if not result.has_more:
+                    break
+                result = self.client.files_list_folder_continue(result.cursor)
+
+            return entries
+        except Exception as e:
+            logger.error(f"Error listing files: {str(e)}", exc_info=True)
+            raise
+
+    def download_file_content(self, path: str) -> bytes:
+        """Download and return the content of a file."""
+        try:
+            logger.debug(f"Downloading file content from {path}")
+            return self.client.files_download(path)[1].content
+        except Exception as e:
+            logger.error(f"Error downloading file: {str(e)}", exc_info=True)
+            raise
+
+    def download_folder(self, dropbox_path: str, local_path: str) -> None:
+        """Download all files from a Dropbox folder to a local path."""
+        try:
+            logger.debug(f"Downloading folder {dropbox_path} to {local_path}")
+            entries = self.list_folder_recursive(dropbox_path)
+
+            for entry in entries:
+                if isinstance(entry, FileMetadata):
+                    rel_path = entry.path_display[len(dropbox_path) :].lstrip("/")
+                    local_file_path = os.path.join(local_path, rel_path)
+
+                    os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+                    logger.debug(f"Downloading {entry.path_display} to {local_file_path}")
+                    self.client.files_download_to_file(local_file_path, entry.path_display)
+
+            logger.debug(f"Successfully downloaded folder {dropbox_path}")
+        except Exception as e:
+            logger.error(f"Error downloading folder: {str(e)}", exc_info=True)
+            raise
+
+    def upload_folder(self, local_path: str, dropbox_path: str) -> None:
+        """Upload all files from a local folder to a Dropbox path."""
+        try:
+            logger.debug(f"Uploading folder {local_path} to {dropbox_path}")
+            for root, _, files in os.walk(local_path):
+                for filename in files:
+                    local_file_path = os.path.join(root, filename)
+                    rel_path = os.path.relpath(local_file_path, local_path)
+                    target_path = f"{dropbox_path}/{rel_path}"
+
+                    logger.debug(f"Uploading {rel_path} to {target_path}")
+                    with open(local_file_path, "rb") as f:
+                        self.client.files_upload(f.read(), target_path, mode=WriteMode.overwrite)
+
+            logger.debug(f"Successfully uploaded folder {local_path}")
+        except Exception as e:
+            logger.error(f"Error uploading folder: {str(e)}", exc_info=True)
+            raise
+
+    def create_shared_link(self, path: str) -> str:
+        """Create a shared link for a file that's accessible without login."""
+        try:
+            logger.debug(f"Creating shared link for {path}")
+            shared_link = self.client.sharing_create_shared_link_with_settings(
+                path, settings=SharedLinkSettings(requested_visibility=RequestedVisibility.public)
+            )
+            return shared_link.url.replace("www.dropbox.com", "dl.dropboxusercontent.com")
+        except Exception as e:
+            logger.error(f"Error creating shared link: {str(e)}", exc_info=True)
+            raise
+
+    def get_existing_shared_link(self, path: str) -> Optional[str]:
+        """Get existing shared link for a file if it exists."""
+        try:
+            logger.debug(f"Getting existing shared link for {path}")
+            shared_links = self.client.sharing_list_shared_links(path=path).links
+            if shared_links:
+                return shared_links[0].url.replace("www.dropbox.com", "dl.dropboxusercontent.com")
+            return None
+        except Exception as e:
+            logger.error(f"Error getting existing shared link: {str(e)}", exc_info=True)
+            return None
+
+    def create_or_get_shared_link(self, path: str) -> str:
+        """Create a shared link or get existing one."""
+        try:
+            existing_link = self.get_existing_shared_link(path)
+            if existing_link:
+                logger.debug(f"Found existing shared link for {path}")
+                return existing_link
+
+            logger.debug(f"Creating new shared link for {path}")
+            return self.create_shared_link(path)
+        except Exception as e:
+            logger.error(f"Error creating/getting shared link: {str(e)}", exc_info=True)
+            raise
+
+    def file_exists(self, path: str) -> bool:
+        """Check if a file exists in Dropbox."""
+        try:
+            self.client.files_get_metadata(path)
+            return True
+        except:
+            return False

lyrics_transcriber/transcribers/audioshake.py

@@ -0,0 +1,379 @@
+from dataclasses import dataclass
+import requests
+import time
+import os
+import tempfile
+from typing import Dict, Optional, Any, Union, Tuple
+from pathlib import Path
+from pydub import AudioSegment
+from lyrics_transcriber.types import TranscriptionData, LyricsSegment, Word
+from lyrics_transcriber.transcribers.base_transcriber import BaseTranscriber, TranscriptionError
+from lyrics_transcriber.utils.word_utils import WordUtils
+
+# Lossy formats that should be uploaded directly (transcoding would cause quality loss)
+LOSSY_FORMATS = {'.mp3', '.aac', '.ogg', '.m4a', '.wma', '.opus'}
+# Lossless formats that are already compressed and can be uploaded directly
+LOSSLESS_COMPRESSED_FORMATS = {'.flac', '.alac'}
+# Uncompressed formats that should be converted to FLAC for efficient upload
+UNCOMPRESSED_FORMATS = {'.wav', '.aiff', '.aif', '.pcm'}
+
+
+class AudioUploadOptimizer:
+    """Optimizes audio files for upload by converting uncompressed formats to FLAC."""
+
+    def __init__(self, logger):
+        self.logger = logger
+
+    def prepare_for_upload(self, filepath: str) -> Tuple[str, Optional[str]]:
+        """
+        Prepare audio file for optimal upload.
+        
+        Returns:
+            Tuple of (filepath_to_upload, temp_file_to_cleanup)
+            - If no conversion needed, returns (original_filepath, None)
+            - If converted, returns (temp_flac_filepath, temp_flac_filepath)
+        """
+        ext = os.path.splitext(filepath)[1].lower()
+        
+        # Lossy formats: upload directly (transcoding would lose quality)
+        if ext in LOSSY_FORMATS:
+            self.logger.info(f"Uploading lossy format ({ext}) directly to preserve quality")
+            return filepath, None
+        
+        # Already compressed lossless: upload directly
+        if ext in LOSSLESS_COMPRESSED_FORMATS:
+            self.logger.info(f"Uploading lossless compressed format ({ext}) directly")
+            return filepath, None
+        
+        # Uncompressed formats: convert to FLAC for smaller upload
+        if ext in UNCOMPRESSED_FORMATS:
+            self.logger.info(f"Converting uncompressed format ({ext}) to FLAC for efficient upload")
+            return self._convert_to_flac(filepath)
+        
+        # Unknown format: try to upload directly
+        self.logger.warning(f"Unknown audio format ({ext}), uploading directly")
+        return filepath, None
+
+    def _convert_to_flac(self, filepath: str) -> Tuple[str, str]:
+        """Convert audio file to FLAC format."""
+        ext = os.path.splitext(filepath)[1].lower()
+        
+        # Load audio based on format
+        if ext == '.wav':
+            audio = AudioSegment.from_wav(filepath)
+        elif ext in {'.aiff', '.aif'}:
+            audio = AudioSegment.from_file(filepath, format='aiff')
+        else:
+            audio = AudioSegment.from_file(filepath)
+        
+        # Create temp file for FLAC output
+        with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as temp_flac:
+            flac_path = temp_flac.name
+            audio.export(flac_path, format="flac")
+        
+        # Log size reduction
+        original_size = os.path.getsize(filepath)
+        flac_size = os.path.getsize(flac_path)
+        reduction_pct = (1 - flac_size / original_size) * 100
+        self.logger.info(f"Converted to FLAC: {original_size / 1024 / 1024:.1f}MB → {flac_size / 1024 / 1024:.1f}MB ({reduction_pct:.0f}% smaller)")
+        
+        return flac_path, flac_path
+
+    def cleanup(self, temp_filepath: Optional[str]) -> None:
+        """Clean up temporary file if it exists."""
+        if temp_filepath and os.path.exists(temp_filepath):
+            try:
+                os.unlink(temp_filepath)
+                self.logger.debug(f"Cleaned up temporary file: {temp_filepath}")
+            except OSError as e:
+                self.logger.warning(f"Failed to clean up temporary file {temp_filepath}: {e}")
+
+
+@dataclass
+class AudioShakeConfig:
+    """Configuration for AudioShake transcription service."""
+
+    api_token: Optional[str] = None
+    base_url: str = "https://api.audioshake.ai"
+    output_prefix: Optional[str] = None
+    timeout_minutes: int = 20  # Added timeout configuration
+
+
+class AudioShakeAPI:
+    """Handles direct API interactions with AudioShake."""
+
+    def __init__(self, config: AudioShakeConfig, logger):
+        self.config = config
+        self.logger = logger
+
+    def _validate_config(self) -> None:
+        """Validate API configuration."""
+        if not self.config.api_token:
+            raise ValueError("AudioShake API token must be provided")
+
+    def _get_headers(self) -> Dict[str, str]:
+        """Get headers for API requests."""
+        self._validate_config()  # Validate before making any API calls
+        return {"x-api-key": self.config.api_token, "Content-Type": "application/json"}
+
+    def upload_file(self, filepath: str) -> str:
+        """Upload audio file and return file URL."""
+        self.logger.info(f"Uploading {filepath} to AudioShake")
+        self._validate_config()  # Validate before making API call
+
+        url = f"{self.config.base_url}/upload/"
+        with open(filepath, "rb") as file:
+            files = {"file": (os.path.basename(filepath), file)}
+            response = requests.post(url, headers={"x-api-key": self.config.api_token}, files=files)
+
+        self.logger.debug(f"Upload response: {response.status_code} - {response.text}")
+        response.raise_for_status()
+        return response.json()["link"]
+
+    def create_task(self, file_url: str) -> str:
+        """Create transcription task and return task ID."""
+        self.logger.info(f"Creating task for file {file_url}")
+
+        url = f"{self.config.base_url}/tasks"
+        data = {
+            "url": file_url,
+            "targets": [
+                {
+                    "model": "alignment",
+                    "formats": ["json"],
+                    "language": "en"
+                }
+            ],
+        }
+        response = requests.post(url, headers=self._get_headers(), json=data)
+        response.raise_for_status()
+        return response.json()["id"]
+
+    def wait_for_task_result(self, task_id: str) -> Dict[str, Any]:
+        """Poll for task completion and return results."""
+        self.logger.info(f"Getting task result for task {task_id}")
+
+        # Use the list endpoint which has fresh data, not the individual task endpoint which caches
+        url = f"{self.config.base_url}/tasks"
+        start_time = time.time()
+        last_status_log = start_time
+        timeout_seconds = self.config.timeout_minutes * 60
+        
+        # Add initial retry logic for when task is not found yet
+        initial_retry_count = 0
+        max_initial_retries = 5
+        initial_retry_delay = 2  # seconds
+
+        while True:
+            current_time = time.time()
+            elapsed_time = current_time - start_time
+
+            # Check for timeout
+            if elapsed_time > timeout_seconds:
+                raise TranscriptionError(f"Transcription timed out after {self.config.timeout_minutes} minutes")
+
+            # Log status every minute
+            if current_time - last_status_log >= 60:
+                self.logger.info(f"Still waiting for transcription... " f"Elapsed time: {int(elapsed_time/60)} minutes")
+                last_status_log = current_time
+
+            try:
+                response = requests.get(url, headers=self._get_headers())
+                response.raise_for_status()
+                tasks_list = response.json()
+                
+                # Find our specific task in the list
+                task_data = None
+                for task in tasks_list:
+                    if task.get("id") == task_id:
+                        task_data = task
+                        break
+                
+                if not task_data:
+                    # Task not found in list yet
+                    if initial_retry_count < max_initial_retries:
+                        initial_retry_count += 1
+                        self.logger.info(f"Task not found in list yet (attempt {initial_retry_count}/{max_initial_retries}), retrying in {initial_retry_delay} seconds...")
+                        time.sleep(initial_retry_delay)
+                        continue
+                    else:
+                        raise TranscriptionError(f"Task {task_id} not found in task list after {max_initial_retries} retries")
+                
+                # Log the full response for debugging
+                self.logger.debug(f"Task status response: {task_data}")
+
+                # Check status of targets (not the task itself)
+                targets = task_data.get("targets", [])
+                if not targets:
+                    raise TranscriptionError("No targets found in task response")
+                
+                # Check if all targets are completed or if any failed
+                all_completed = True
+                for target in targets:
+                    target_status = target.get("status")
+                    target_model = target.get("model")
+                    self.logger.debug(f"Target {target_model} status: {target_status}")
+                    
+                    if target_status == "failed":
+                        error_msg = target.get("error", "Unknown error")
+                        raise TranscriptionError(f"Target {target_model} failed: {error_msg}")
+                    elif target_status != "completed":
+                        all_completed = False
+                
+                if all_completed:
+                    self.logger.info("All targets completed successfully")
+                    return task_data
+
+                # Reset retry count on successful response
+                initial_retry_count = 0
+                
+            except requests.exceptions.HTTPError as e:
+                raise
+
+            time.sleep(30)  # Wait before next poll
+
+
+class AudioShakeTranscriber(BaseTranscriber):
+    """Transcription service using AudioShake's API."""
+
+    def __init__(
+        self,
+        cache_dir: Union[str, Path],
+        config: Optional[AudioShakeConfig] = None,
+        logger: Optional[Any] = None,
+        api_client: Optional[AudioShakeAPI] = None,
+        upload_optimizer: Optional[AudioUploadOptimizer] = None,
+    ):
+        """Initialize AudioShake transcriber."""
+        super().__init__(cache_dir=cache_dir, logger=logger)
+        self.config = config or AudioShakeConfig(api_token=os.getenv("AUDIOSHAKE_API_TOKEN"))
+        self.api = api_client or AudioShakeAPI(self.config, self.logger)
+        self.upload_optimizer = upload_optimizer or AudioUploadOptimizer(self.logger)
+
+    def get_name(self) -> str:
+        return "AudioShake"
+
+    def _perform_transcription(self, audio_filepath: str) -> TranscriptionData:
+        """Actually perform the transcription using AudioShake API."""
+        self.logger.debug(f"Entering _perform_transcription() for {audio_filepath}")
+        self.logger.info(f"Starting transcription for {audio_filepath}")
+
+        try:
+            # Start task and get results
+            self.logger.debug("Calling start_transcription()")
+            task_id = self.start_transcription(audio_filepath)
+            self.logger.debug(f"Got task_id: {task_id}")
+
+            self.logger.debug("Calling get_transcription_result()")
+            result = self.get_transcription_result(task_id)
+            self.logger.debug("Got transcription result")
+
+            return result
+        except Exception as e:
+            self.logger.error(f"Error in _perform_transcription: {str(e)}")
+            raise
+
+    def start_transcription(self, audio_filepath: str) -> str:
+        """Starts the transcription task and returns the task ID."""
+        self.logger.debug(f"Entering start_transcription() for {audio_filepath}")
+
+        # Optimize file format for upload (convert WAV to FLAC, etc.)
+        upload_filepath, temp_filepath = self.upload_optimizer.prepare_for_upload(audio_filepath)
+        
+        try:
+            # Upload file and create task
+            file_url = self.api.upload_file(upload_filepath)
+            self.logger.debug(f"File uploaded successfully. File URL: {file_url}")
+
+            task_id = self.api.create_task(file_url)
+            self.logger.debug(f"Task created successfully. Task ID: {task_id}")
+
+            return task_id
+        finally:
+            # Clean up any temporary file created during optimization
+            self.upload_optimizer.cleanup(temp_filepath)
+
+    def get_transcription_result(self, task_id: str) -> Dict[str, Any]:
+        """Gets the raw results for a previously started task."""
+        self.logger.debug(f"Entering get_transcription_result() for task ID: {task_id}")
+
+        # Wait for task completion
+        task_data = self.api.wait_for_task_result(task_id)
+        self.logger.debug("Task completed. Getting results...")
+
+        # Find the alignment target output
+        alignment_target = None
+        for target in task_data.get("targets", []):
+            if target.get("model") == "alignment":
+                alignment_target = target
+                break
+        
+        if not alignment_target:
+            raise TranscriptionError("Required output not found in task results")
+        
+        # Get the output file URL
+        output = alignment_target.get("output", [])
+        if not output:
+            raise TranscriptionError("No output found in alignment target")
+        
+        output_url = output[0].get("link")
+        if not output_url:
+            raise TranscriptionError("Output link not found in alignment target")
+
+        # Fetch transcription data
+        response = requests.get(output_url)
+        response.raise_for_status()
+
+        # Return combined raw data
+        raw_data = {"task_data": task_data, "transcription": response.json()}
+
+        self.logger.debug("Raw results retrieved successfully")
+        return raw_data
+
+    def _convert_result_format(self, raw_data: Dict[str, Any]) -> TranscriptionData:
+        """Process raw Audioshake API response into standard format."""
+        self.logger.debug(f"Processing result for task {raw_data['task_data']['id']}")
+
+        transcription_data = raw_data["transcription"]
+        task_data = raw_data["task_data"]
+
+        segments = []
+        all_words = []  # Collect all words across segments
+
+        for line in transcription_data.get("lines", []):
+            words = [
+                Word(
+                    id=WordUtils.generate_id(),  # Generate unique ID for each word
+                    text=word["text"].strip(" "),
+                    start_time=word.get("start", 0.0),
+                    end_time=word.get("end", 0.0),
+                )
+                for word in line.get("words", [])
+            ]
+            all_words.extend(words)  # Add words to flat list
+
+            segments.append(
+                LyricsSegment(
+                    id=WordUtils.generate_id(),  # Generate unique ID for each segment
+                    text=line.get("text", " ".join(w.text for w in words)),
+                    words=words,
+                    start_time=min((w.start_time for w in words), default=0.0),
+                    end_time=max((w.end_time for w in words), default=0.0),
+                )
+            )
+
+        return TranscriptionData(
+            text=transcription_data.get("text", ""),
+            words=all_words,
+            segments=segments,
+            source=self.get_name(),
+            metadata={
+                "language": transcription_data.get("metadata", {}).get("language"),
+                "task_id": task_data["id"],
+                "duration": task_data.get("duration"),
+            },
+        )
+
+    def get_output_filename(self, suffix: str) -> str:
+        """Generate consistent filename with (Purpose) suffix pattern."""
+        return f"{self.config.output_prefix}{suffix}"