karaoke-gen 0.90.1__py3-none-any.whl → 0.99.3__py3-none-any.whl

backend/services/packaging_service.py

@@ -0,0 +1,287 @@
+"""
+Packaging Service.
+
+Provides CDG and TXT package generation functionality, extracted from KaraokeFinalise
+for use by both the cloud backend (video_worker) and local CLI.
+
+This service handles:
+- CDG (CD+G) file generation from LRC files
+- TXT lyric file generation from LRC files
+- ZIP packaging of CDG/MP3 and TXT/MP3 pairs
+"""
+
+import logging
+import os
+import zipfile
+from typing import Optional, Dict, Any, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+class PackagingService:
+    """
+    Service for creating CDG and TXT karaoke packages.
+
+    CDG (CD+Graphics) is a format used by karaoke machines.
+    TXT packages are used by software karaoke players.
+    Both formats are packaged as ZIP files with an MP3 audio track.
+    """
+
+    def __init__(
+        self,
+        cdg_styles: Optional[Dict[str, Any]] = None,
+        dry_run: bool = False,
+        non_interactive: bool = False,
+        logger: Optional[logging.Logger] = None,
+    ):
+        """
+        Initialize the packaging service.
+
+        Args:
+            cdg_styles: CDG style configuration for CDG generation
+            dry_run: If True, log actions without performing them
+            non_interactive: If True, skip interactive prompts
+            logger: Optional logger instance
+        """
+        self.cdg_styles = cdg_styles
+        self.dry_run = dry_run
+        self.non_interactive = non_interactive
+        self.logger = logger or logging.getLogger(__name__)
+
+    def create_cdg_package(
+        self,
+        lrc_file: str,
+        audio_file: str,
+        output_zip_path: str,
+        artist: str,
+        title: str,
+        output_mp3_path: Optional[str] = None,
+        output_cdg_path: Optional[str] = None,
+    ) -> Tuple[str, Optional[str], Optional[str]]:
+        """
+        Create a CDG package (ZIP containing CDG and MP3 files).
+
+        Args:
+            lrc_file: Path to the LRC lyrics file
+            audio_file: Path to the instrumental audio file
+            output_zip_path: Path for the output ZIP file
+            artist: Artist name for metadata
+            title: Song title for metadata
+            output_mp3_path: Optional path for the extracted MP3 file
+            output_cdg_path: Optional path for the extracted CDG file
+
+        Returns:
+            Tuple of (zip_path, mp3_path, cdg_path)
+
+        Raises:
+            ValueError: If CDG styles are not configured
+            FileNotFoundError: If input files are missing
+            Exception: If CDG generation fails
+        """
+        self.logger.info(f"Creating CDG package for {artist} - {title}")
+
+        # Validate inputs
+        if not os.path.isfile(lrc_file):
+            raise FileNotFoundError(f"LRC file not found: {lrc_file}")
+        if not os.path.isfile(audio_file):
+            raise FileNotFoundError(f"Audio file not found: {audio_file}")
+
+        # Check if ZIP already exists
+        if os.path.isfile(output_zip_path):
+            if self.non_interactive:
+                self.logger.info(
+                    f"CDG ZIP exists, will be overwritten: {output_zip_path}"
+                )
+            else:
+                self.logger.info(f"CDG ZIP already exists: {output_zip_path}")
+
+        # Check if individual files exist (allows skipping generation)
+        if output_mp3_path and output_cdg_path:
+            if os.path.isfile(output_mp3_path) and os.path.isfile(output_cdg_path):
+                self.logger.info("Found existing MP3 and CDG files, creating ZIP directly")
+                if not self.dry_run:
+                    self._create_zip_from_files(
+                        output_zip_path,
+                        [(output_mp3_path, os.path.basename(output_mp3_path)),
+                         (output_cdg_path, os.path.basename(output_cdg_path))]
+                    )
+                return output_zip_path, output_mp3_path, output_cdg_path
+
+        if self.dry_run:
+            self.logger.info(
+                f"DRY RUN: Would generate CDG package: {output_zip_path}"
+            )
+            return output_zip_path, output_mp3_path, output_cdg_path
+
+        # Generate CDG files
+        if self.cdg_styles is None:
+            raise ValueError(
+                "CDG styles configuration is required for CDG generation"
+            )
+
+        self.logger.info("Generating CDG and MP3 files")
+        from lyrics_transcriber.output.cdg import CDGGenerator
+
+        output_dir = os.path.dirname(output_zip_path) or os.getcwd()
+        generator = CDGGenerator(output_dir=output_dir, logger=self.logger)
+
+        cdg_file, mp3_file, zip_file = generator.generate_cdg_from_lrc(
+            lrc_file=lrc_file,
+            audio_file=audio_file,
+            title=title,
+            artist=artist,
+            cdg_styles=self.cdg_styles,
+        )
+
+        # Rename ZIP to expected output path if different
+        if os.path.isfile(zip_file) and zip_file != output_zip_path:
+            os.rename(zip_file, output_zip_path)
+            self.logger.info(f"Renamed CDG ZIP: {zip_file} -> {output_zip_path}")
+
+        if not os.path.isfile(output_zip_path):
+            raise Exception(f"Failed to create CDG ZIP file: {output_zip_path}")
+
+        # Extract the ZIP to get individual files if paths provided
+        extracted_mp3 = None
+        extracted_cdg = None
+        if output_mp3_path or output_cdg_path:
+            self.logger.info(f"Extracting CDG ZIP file: {output_zip_path}")
+            with zipfile.ZipFile(output_zip_path, "r") as zip_ref:
+                zip_ref.extractall(output_dir)
+
+            # Find extracted files
+            if output_mp3_path and os.path.isfile(output_mp3_path):
+                extracted_mp3 = output_mp3_path
+                self.logger.info(f"Extracted MP3: {extracted_mp3}")
+            if output_cdg_path and os.path.isfile(output_cdg_path):
+                extracted_cdg = output_cdg_path
+                self.logger.info(f"Extracted CDG: {extracted_cdg}")
+
+        self.logger.info(f"CDG package created: {output_zip_path}")
+        return output_zip_path, extracted_mp3, extracted_cdg
+
+    def create_txt_package(
+        self,
+        lrc_file: str,
+        mp3_file: str,
+        output_zip_path: str,
+        output_txt_path: Optional[str] = None,
+    ) -> Tuple[str, Optional[str]]:
+        """
+        Create a TXT package (ZIP containing TXT lyrics and MP3 files).
+
+        Args:
+            lrc_file: Path to the LRC lyrics file
+            mp3_file: Path to the MP3 audio file
+            output_zip_path: Path for the output ZIP file
+            output_txt_path: Optional path for the generated TXT file
+
+        Returns:
+            Tuple of (zip_path, txt_path)
+
+        Raises:
+            FileNotFoundError: If input files are missing
+            Exception: If TXT generation fails
+        """
+        self.logger.info(f"Creating TXT package from {lrc_file}")
+
+        # Validate inputs
+        if not os.path.isfile(lrc_file):
+            raise FileNotFoundError(f"LRC file not found: {lrc_file}")
+        if not os.path.isfile(mp3_file):
+            raise FileNotFoundError(f"MP3 file not found: {mp3_file}")
+
+        # Check if ZIP already exists
+        if os.path.isfile(output_zip_path):
+            if self.non_interactive:
+                self.logger.info(
+                    f"TXT ZIP exists, will be overwritten: {output_zip_path}"
+                )
+            else:
+                self.logger.info(f"TXT ZIP already exists: {output_zip_path}")
+
+        if self.dry_run:
+            self.logger.info(
+                f"DRY RUN: Would create TXT package: {output_zip_path}"
+            )
+            return output_zip_path, output_txt_path
+
+        # Generate TXT from LRC
+        self.logger.info(f"Converting LRC to TXT format: {lrc_file}")
+        from lyrics_converter import LyricsConverter
+
+        txt_converter = LyricsConverter(output_format="txt", filepath=lrc_file)
+        converted_txt = txt_converter.convert_file()
+
+        # Write TXT file
+        if output_txt_path is None:
+            # Default to same name as ZIP but with .txt extension
+            output_txt_path = output_zip_path.replace(".zip", ".txt")
+
+        with open(output_txt_path, "w") as txt_file:
+            txt_file.write(converted_txt)
+            self.logger.info(f"TXT file written: {output_txt_path}")
+
+        # Create ZIP containing MP3 and TXT
+        self.logger.info(f"Creating TXT ZIP: {output_zip_path}")
+        self._create_zip_from_files(
+            output_zip_path,
+            [(mp3_file, os.path.basename(mp3_file)),
+             (output_txt_path, os.path.basename(output_txt_path))]
+        )
+
+        if not os.path.isfile(output_zip_path):
+            raise Exception(f"Failed to create TXT ZIP file: {output_zip_path}")
+
+        self.logger.info(f"TXT package created: {output_zip_path}")
+        return output_zip_path, output_txt_path
+
+    def _create_zip_from_files(
+        self,
+        zip_path: str,
+        files: list,
+    ) -> None:
+        """
+        Create a ZIP file from a list of files.
+
+        Args:
+            zip_path: Path for the output ZIP file
+            files: List of (file_path, archive_name) tuples
+        """
+        with zipfile.ZipFile(zip_path, "w") as zipf:
+            for file_path, archive_name in files:
+                if os.path.isfile(file_path):
+                    zipf.write(file_path, archive_name)
+                    self.logger.debug(f"Added to ZIP: {archive_name}")
+                else:
+                    self.logger.warning(f"File not found for ZIP: {file_path}")
+
+
+# Singleton instance and factory function (following existing service pattern)
+_packaging_service: Optional[PackagingService] = None
+
+
+def get_packaging_service(
+    cdg_styles: Optional[Dict[str, Any]] = None,
+    **kwargs
+) -> PackagingService:
+    """
+    Get a packaging service instance.
+
+    Args:
+        cdg_styles: CDG style configuration
+        **kwargs: Additional arguments passed to PackagingService
+
+    Returns:
+        PackagingService instance
+    """
+    global _packaging_service
+
+    # Create new instance if settings changed
+    if _packaging_service is None or cdg_styles:
+        _packaging_service = PackagingService(
+            cdg_styles=cdg_styles,
+            **kwargs
+        )
+
+    return _packaging_service

backend/services/rclone_service.py

@@ -0,0 +1,106 @@
+"""
+Rclone configuration service for cloud storage integration.
+
+This service manages the rclone configuration needed for Dropbox
+and other cloud storage uploads from the backend workers.
+"""
+import logging
+import os
+import tempfile
+from typing import Optional
+
+from backend.config import get_settings
+
+logger = logging.getLogger(__name__)
+
+
+class RcloneService:
+    """Service for managing rclone configuration."""
+    
+    # Secret Manager secret name for rclone config
+    RCLONE_CONFIG_SECRET = "rclone-config"
+    
+    def __init__(self):
+        self.settings = get_settings()
+        self._config_file: Optional[str] = None
+        self._config_loaded = False
+    
+    def setup_rclone_config(self) -> bool:
+        """
+        Load rclone config from Secret Manager and set up environment.
+        
+        Writes the config to a temp file and sets RCLONE_CONFIG env var.
+        
+        Returns:
+            True if successful, False otherwise
+        """
+        if self._config_loaded:
+            logger.debug("Rclone config already loaded")
+            return True
+        
+        try:
+            # Get rclone config from Secret Manager
+            config_content = self.settings.get_secret(self.RCLONE_CONFIG_SECRET)
+            
+            if not config_content:
+                logger.warning("Rclone config not found in Secret Manager")
+                return False
+            
+            # Write to a temp file
+            fd, config_path = tempfile.mkstemp(prefix="rclone_", suffix=".conf")
+            try:
+                with os.fdopen(fd, 'w') as f:
+                    f.write(config_content)
+                
+                self._config_file = config_path
+                
+                # Set environment variable for rclone to find the config
+                os.environ["RCLONE_CONFIG"] = config_path
+                
+                logger.info(f"Rclone config loaded and written to {config_path}")
+                self._config_loaded = True
+                return True
+                
+            except Exception:
+                # Clean up the temp file on error
+                # Note: os.fdopen() takes ownership of fd, so it's already closed
+                # We only need to remove the temp file if it exists
+                if os.path.exists(config_path):
+                    os.unlink(config_path)
+                raise
+                
+        except Exception as e:
+            logger.error(f"Failed to setup rclone config: {e}")
+            return False
+    
+    def cleanup(self) -> None:
+        """Remove the temporary config file."""
+        if self._config_file and os.path.exists(self._config_file):
+            try:
+                os.unlink(self._config_file)
+                logger.debug(f"Cleaned up rclone config file: {self._config_file}")
+            except Exception as e:
+                logger.warning(f"Failed to cleanup rclone config: {e}")
+        
+        # Always reset internal state and environment, even if the file was missing
+        if self._config_file is not None:
+            os.environ.pop("RCLONE_CONFIG", None)
+        self._config_file = None
+        self._config_loaded = False
+    
+    @property
+    def is_configured(self) -> bool:
+        """Check if rclone is configured and ready to use."""
+        return self._config_loaded and self._config_file is not None
+
+
+# Singleton instance
+_rclone_service: Optional[RcloneService] = None
+
+
+def get_rclone_service() -> RcloneService:
+    """Get the singleton rclone service instance."""
+    global _rclone_service
+    if _rclone_service is None:
+        _rclone_service = RcloneService()
+    return _rclone_service

backend/services/spacy_preloader.py

@@ -0,0 +1,65 @@
+"""SpaCy model preloader for container startup.
+
+Loads SpaCy models at container startup to avoid slow loading during request processing.
+Cloud Run filesystem I/O can cause 60+ second delays when loading SpaCy models lazily.
+"""
+
+import logging
+import time
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+# Singleton storage for preloaded models
+_preloaded_models: dict = {}
+
+
+def preload_spacy_model(model_name: str = "en_core_web_sm") -> None:
+    """Preload a SpaCy model at startup.
+
+    Args:
+        model_name: The SpaCy model to load (default: en_core_web_sm)
+    """
+    global _preloaded_models
+
+    if model_name in _preloaded_models:
+        logger.info(f"SpaCy model '{model_name}' already preloaded")
+        return
+
+    logger.info(f"Preloading SpaCy model '{model_name}'...")
+    start_time = time.time()
+
+    try:
+        import spacy
+
+        nlp = spacy.load(model_name)
+        _preloaded_models[model_name] = nlp
+
+        elapsed = time.time() - start_time
+        logger.info(f"SpaCy model '{model_name}' preloaded in {elapsed:.2f}s")
+    except Exception as e:
+        logger.error(f"Failed to preload SpaCy model '{model_name}': {e}")
+        raise
+
+
+def get_preloaded_model(model_name: str = "en_core_web_sm") -> Optional[object]:
+    """Get a preloaded SpaCy model if available.
+
+    Args:
+        model_name: The SpaCy model name
+
+    Returns:
+        The preloaded SpaCy Language object, or None if not preloaded
+    """
+    return _preloaded_models.get(model_name)
+
+
+def is_model_preloaded(model_name: str = "en_core_web_sm") -> bool:
+    """Check if a SpaCy model has been preloaded."""
+    return model_name in _preloaded_models
+
+
+def clear_preloaded_models() -> None:
+    """Clear all preloaded models. Useful for testing."""
+    global _preloaded_models
+    _preloaded_models.clear()

backend/services/storage_service.py

@@ -0,0 +1,209 @@
+"""
+Google Cloud Storage operations for file management.
+"""
+import logging
+import os
+import json
+from typing import Optional, BinaryIO, Any, Dict
+from pathlib import Path
+from google.cloud import storage
+from datetime import timedelta
+
+from backend.config import settings
+
+
+logger = logging.getLogger(__name__)
+
+
+class StorageService:
+    """Service for Google Cloud Storage operations."""
+    
+    def __init__(self):
+        """Initialize GCS client."""
+        self.client = storage.Client(project=settings.google_cloud_project)
+        self.bucket = self.client.bucket(settings.gcs_bucket_name)
+    
+    def upload_file(self, local_path: str, destination_path: str) -> str:
+        """Upload a file to GCS."""
+        try:
+            blob = self.bucket.blob(destination_path)
+            blob.upload_from_filename(local_path)
+            logger.info(f"Uploaded {local_path} to gs://{settings.gcs_bucket_name}/{destination_path}")
+            return destination_path
+        except Exception as e:
+            logger.error(f"Error uploading file {local_path}: {e}")
+            raise
+    
+    def upload_fileobj(self, file_obj: BinaryIO, destination_path: str, content_type: Optional[str] = None) -> str:
+        """Upload a file object to GCS."""
+        try:
+            blob = self.bucket.blob(destination_path)
+            if content_type:
+                blob.content_type = content_type
+            blob.upload_from_file(file_obj, rewind=True)
+            logger.info(f"Uploaded file object to gs://{settings.gcs_bucket_name}/{destination_path}")
+            return destination_path
+        except Exception as e:
+            logger.error(f"Error uploading file object: {e}")
+            raise
+    
+    def download_file(self, source_path: str, destination_path: str) -> str:
+        """Download a file from GCS."""
+        try:
+            blob = self.bucket.blob(source_path)
+            blob.download_to_filename(destination_path)
+            logger.info(f"Downloaded gs://{settings.gcs_bucket_name}/{source_path} to {destination_path}")
+            return destination_path
+        except Exception as e:
+            logger.error(f"Error downloading file {source_path}: {e}")
+            raise
+    
+    def generate_signed_url(self, blob_path: str, expiration_minutes: int = 60) -> str:
+        """Generate a signed URL for downloading a file.
+        
+        In Cloud Run, this uses the IAM signBlob API since we don't have
+        a private key available. Requires the service account to have
+        roles/iam.serviceAccountTokenCreator on itself.
+        """
+        return self._generate_signed_url_internal(blob_path, "GET", expiration_minutes)
+    
+    def generate_signed_upload_url(self, blob_path: str, content_type: str = "application/octet-stream", expiration_minutes: int = 60) -> str:
+        """Generate a signed URL for uploading a file directly to GCS.
+        
+        This allows clients to upload files directly to GCS without going through
+        the backend, bypassing any request body size limits.
+        
+        Args:
+            blob_path: The destination path in GCS
+            content_type: The expected content type of the upload
+            expiration_minutes: How long the URL is valid for
+            
+        Returns:
+            A signed URL that accepts PUT requests with the file content
+        """
+        return self._generate_signed_url_internal(blob_path, "PUT", expiration_minutes, content_type)
+    
+    def _generate_signed_url_internal(self, blob_path: str, method: str, expiration_minutes: int = 60, content_type: Optional[str] = None) -> str:
+        """Internal method to generate signed URLs for GET or PUT operations."""
+        import google.auth
+        from google.auth.transport import requests
+        
+        try:
+            blob = self.bucket.blob(blob_path)
+            
+            # Get default credentials and refresh to ensure we have a valid token
+            credentials, project = google.auth.default()
+            
+            # Common kwargs for signed URL generation
+            kwargs = {
+                "version": "v4",
+                "expiration": timedelta(minutes=expiration_minutes),
+                "method": method,
+            }
+            
+            # For PUT requests, we need to specify the content type in headers
+            if method == "PUT" and content_type:
+                kwargs["headers"] = {"Content-Type": content_type}
+            
+            # Check if we're using compute credentials (Cloud Run/GCE)
+            # These need to use IAM signBlob via service_account_email + access_token
+            if hasattr(credentials, 'service_account_email'):
+                # Refresh credentials to get a valid access token
+                auth_request = requests.Request()
+                credentials.refresh(auth_request)
+                
+                kwargs["service_account_email"] = credentials.service_account_email
+                kwargs["access_token"] = credentials.token
+            
+            url = blob.generate_signed_url(**kwargs)
+            
+            logger.info(f"Generated signed {method} URL for {blob_path}")
+            return url
+        except Exception as e:
+            logger.error(f"Error generating signed {method} URL for {blob_path}: {e}")
+            raise
+    
+    def delete_file(self, blob_path: str) -> None:
+        """Delete a file from GCS."""
+        try:
+            blob = self.bucket.blob(blob_path)
+            blob.delete()
+            logger.info(f"Deleted gs://{settings.gcs_bucket_name}/{blob_path}")
+        except Exception as e:
+            logger.error(f"Error deleting file {blob_path}: {e}")
+            raise
+    
+    def delete_folder(self, prefix: str) -> int:
+        """
+        Delete all files in GCS with a given prefix (folder).
+        
+        Args:
+            prefix: The folder prefix to delete (e.g., "uploads/abc123/")
+            
+        Returns:
+            Number of files deleted
+        """
+        try:
+            blobs = list(self.bucket.list_blobs(prefix=prefix))
+            deleted_count = 0
+            
+            for blob in blobs:
+                try:
+                    blob.delete()
+                    deleted_count += 1
+                except Exception as e:
+                    logger.warning(f"Error deleting blob {blob.name}: {e}")
+            
+            if deleted_count > 0:
+                logger.info(f"Deleted {deleted_count} files from gs://{settings.gcs_bucket_name}/{prefix}")
+            
+            return deleted_count
+        except Exception as e:
+            logger.error(f"Error deleting folder {prefix}: {e}")
+            return 0  # Don't raise - folder deletion shouldn't break operations
+    
+    def list_files(self, prefix: str) -> list:
+        """List files in GCS with a given prefix."""
+        try:
+            blobs = self.bucket.list_blobs(prefix=prefix)
+            return [blob.name for blob in blobs]
+        except Exception as e:
+            logger.error(f"Error listing files with prefix {prefix}: {e}")
+            raise
+    
+    def file_exists(self, blob_path: str) -> bool:
+        """Check if a file exists in GCS."""
+        try:
+            blob = self.bucket.blob(blob_path)
+            return blob.exists()
+        except Exception as e:
+            logger.error(f"Error checking file existence {blob_path}: {e}")
+            raise
+    
+    def upload_json(self, destination_path: str, data: Dict[str, Any]) -> str:
+        """Upload a JSON object to GCS."""
+        try:
+            blob = self.bucket.blob(destination_path)
+            blob.content_type = "application/json"
+            blob.upload_from_string(
+                json.dumps(data, indent=2, ensure_ascii=False),
+                content_type="application/json"
+            )
+            logger.info(f"Uploaded JSON to gs://{settings.gcs_bucket_name}/{destination_path}")
+            return destination_path
+        except Exception as e:
+            logger.error(f"Error uploading JSON to {destination_path}: {e}")
+            raise
+    
+    def download_json(self, source_path: str) -> Dict[str, Any]:
+        """Download and parse a JSON file from GCS."""
+        try:
+            blob = self.bucket.blob(source_path)
+            content = blob.download_as_text()
+            data = json.loads(content)
+            logger.info(f"Downloaded JSON from gs://{settings.gcs_bucket_name}/{source_path}")
+            return data
+        except Exception as e:
+            logger.error(f"Error downloading JSON from {source_path}: {e}")
+            raise
+