karaoke-gen 0.86.7__py3-none-any.whl → 0.96.0__py3-none-any.whl

backend/services/gdrive_service.py

@@ -0,0 +1,356 @@
+"""Native Google Drive API service for cloud backend.
+
+This service provides Google Drive operations using the native API,
+for uploading files to public share folders. It handles:
+- Folder creation and lookup
+- File uploads with resumable upload support
+- Uploading to organized folder structure (MP4/, MP4-720p/, CDG/)
+
+Credentials are loaded from Google Cloud Secret Manager and can be
+shared with YouTube credentials if scopes include drive.file.
+"""
+import json
+import logging
+import os
+from typing import Any, Dict, Optional
+
+from backend.config import get_settings
+from karaoke_gen.utils import sanitize_filename
+
+logger = logging.getLogger(__name__)
+
+
+class GoogleDriveService:
+    """Google Drive operations using native API."""
+
+    # Secret Manager secret name for Google Drive credentials
+    # Can be same as YouTube if scopes include drive.file
+    GDRIVE_CREDENTIALS_SECRET = "gdrive-oauth-credentials"
+
+    def __init__(self):
+        self.settings = get_settings()
+        self._service = None
+        self._credentials_data: Optional[Dict[str, Any]] = None
+        self._loaded = False
+
+    def _load_credentials(self) -> Optional[Dict[str, Any]]:
+        """Load OAuth credentials from Secret Manager."""
+        if self._loaded:
+            return self._credentials_data
+
+        try:
+            creds_json = self.settings.get_secret(self.GDRIVE_CREDENTIALS_SECRET)
+
+            if not creds_json:
+                # Try falling back to YouTube credentials (may have drive scope)
+                logger.info(
+                    "Google Drive credentials not found, trying YouTube credentials"
+                )
+                creds_json = self.settings.get_secret("youtube-oauth-credentials")
+
+            if not creds_json:
+                logger.warning("Google Drive credentials not found in Secret Manager")
+                self._loaded = True
+                return None
+
+            self._credentials_data = json.loads(creds_json)
+
+            # Validate required fields
+            required_fields = ["refresh_token", "client_id", "client_secret"]
+            missing = [f for f in required_fields if not self._credentials_data.get(f)]
+
+            if missing:
+                logger.error(f"Google Drive credentials missing required fields: {missing}")
+                self._credentials_data = None
+                self._loaded = True
+                return None
+
+            logger.info("Google Drive credentials loaded successfully from Secret Manager")
+            self._loaded = True
+            return self._credentials_data
+
+        except json.JSONDecodeError as e:
+            logger.error(f"Failed to parse Google Drive credentials JSON: {e}")
+            self._loaded = True
+            return None
+        except Exception as e:
+            logger.error(f"Failed to load Google Drive credentials: {e}")
+            self._loaded = True
+            return None
+
+    @property
+    def is_configured(self) -> bool:
+        """Check if Google Drive credentials are available."""
+        creds = self._load_credentials()
+        return creds is not None
+
+    @property
+    def service(self):
+        """Get or create Google Drive service."""
+        if self._service is None:
+            # Import here to avoid import errors if google packages not installed
+            try:
+                from google.oauth2.credentials import Credentials
+                from googleapiclient.discovery import build
+            except ImportError:
+                raise ImportError(
+                    "google-api-python-client and google-auth packages are required. "
+                    "Install them with: pip install google-api-python-client google-auth"
+                )
+
+            creds_data = self._load_credentials()
+            if not creds_data:
+                raise RuntimeError(
+                    "Google Drive credentials not configured in Secret Manager"
+                )
+
+            # Create credentials object
+            credentials = Credentials(
+                token=creds_data.get("token"),
+                refresh_token=creds_data.get("refresh_token"),
+                token_uri=creds_data.get(
+                    "token_uri", "https://oauth2.googleapis.com/token"
+                ),
+                client_id=creds_data.get("client_id"),
+                client_secret=creds_data.get("client_secret"),
+                scopes=creds_data.get(
+                    "scopes", ["https://www.googleapis.com/auth/drive.file"]
+                ),
+            )
+
+            self._service = build("drive", "v3", credentials=credentials)
+            logger.info("Google Drive service initialized successfully")
+
+        return self._service
+
+    def get_or_create_folder(self, parent_id: str, folder_name: str) -> str:
+        """
+        Get existing folder or create new one, return folder ID.
+
+        Args:
+            parent_id: Parent folder ID
+            folder_name: Name of folder to find or create
+
+        Returns:
+            Folder ID
+        """
+        logger.info(f"Looking for folder '{folder_name}' in parent {parent_id}")
+
+        # Search for existing folder
+        # Escape single quotes in folder name for Google Drive API query syntax
+        escaped_folder_name = folder_name.replace("'", "\\'")
+        query = (
+            f"name='{escaped_folder_name}' and '{parent_id}' in parents "
+            f"and mimeType='application/vnd.google-apps.folder' and trashed=false"
+        )
+        results = self.service.files().list(q=query, fields="files(id, name)").execute()
+
+        if results.get("files"):
+            folder_id = results["files"][0]["id"]
+            logger.info(f"Found existing folder '{folder_name}': {folder_id}")
+            return folder_id
+
+        # Create folder
+        logger.info(f"Creating new folder '{folder_name}'")
+        metadata = {
+            "name": folder_name,
+            "mimeType": "application/vnd.google-apps.folder",
+            "parents": [parent_id],
+        }
+        folder = self.service.files().create(body=metadata, fields="id").execute()
+        folder_id = folder["id"]
+        logger.info(f"Created folder '{folder_name}': {folder_id}")
+        return folder_id
+
+    def upload_file(
+        self,
+        local_path: str,
+        parent_id: str,
+        filename: str,
+        replace_existing: bool = True,
+    ) -> str:
+        """
+        Upload a file to a specific Drive folder.
+
+        Args:
+            local_path: Local file path
+            parent_id: Parent folder ID in Google Drive
+            filename: Name for the file in Drive
+            replace_existing: If True, delete existing file with same name first
+
+        Returns:
+            File ID of uploaded file
+        """
+        from googleapiclient.http import MediaFileUpload
+
+        file_size = os.path.getsize(local_path)
+        logger.info(
+            f"Uploading {local_path} ({file_size / 1024 / 1024:.1f} MB) "
+            f"as '{filename}' to folder {parent_id}"
+        )
+
+        # Determine MIME type
+        ext = os.path.splitext(local_path)[1].lower()
+        mime_types = {
+            ".mp4": "video/mp4",
+            ".mkv": "video/x-matroska",
+            ".zip": "application/zip",
+            ".flac": "audio/flac",
+            ".mp3": "audio/mpeg",
+            ".wav": "audio/wav",
+            ".png": "image/png",
+            ".jpg": "image/jpeg",
+            ".jpeg": "image/jpeg",
+        }
+        mime_type = mime_types.get(ext, "application/octet-stream")
+
+        # Check for existing file with same name
+        if replace_existing:
+            # Escape single quotes in filename for Google Drive API query syntax
+            escaped_filename = filename.replace("'", "\\'")
+            query = (
+                f"name='{escaped_filename}' and '{parent_id}' in parents and trashed=false"
+            )
+            results = self.service.files().list(q=query, fields="files(id)").execute()
+            for existing_file in results.get("files", []):
+                logger.info(f"Deleting existing file: {existing_file['id']}")
+                self.service.files().delete(fileId=existing_file["id"]).execute()
+
+        # Upload file
+        metadata = {"name": filename, "parents": [parent_id]}
+        media = MediaFileUpload(local_path, mimetype=mime_type, resumable=True)
+
+        file_result = (
+            self.service.files().create(body=metadata, media_body=media, fields="id").execute()
+        )
+        file_id = file_result["id"]
+        logger.info(f"Successfully uploaded '{filename}': {file_id}")
+        return file_id
+
+    def upload_to_public_share(
+        self,
+        root_folder_id: str,
+        brand_code: str,
+        base_name: str,
+        output_files: dict,
+    ) -> Dict[str, str]:
+        """
+        Upload final files to public share folder structure.
+
+        Creates/uses subfolders:
+        - MP4/{brand_code} - {base_name}.mp4 (lossy 4k)
+        - MP4-720p/{brand_code} - {base_name}.mp4
+        - CDG/{brand_code} - {base_name}.zip
+
+        Args:
+            root_folder_id: Google Drive folder ID for public share root
+            brand_code: Brand code (e.g., "NOMAD-1163")
+            base_name: Base filename (e.g., "Artist - Title")
+            output_files: Dictionary with output file paths:
+                - final_karaoke_lossy_mp4: 4K lossy MP4
+                - final_karaoke_lossy_720p_mp4: 720p lossy MP4
+                - final_karaoke_cdg_zip: CDG package ZIP
+
+        Returns:
+            Dictionary mapping category to uploaded file ID
+        """
+        # Sanitize base_name to handle Unicode characters (curly quotes, em dashes, etc.)
+        # that could cause issues with Google Drive API queries and file naming
+        safe_base_name = sanitize_filename(base_name) if base_name else base_name
+        filename_base = f"{brand_code} - {safe_base_name}"
+        uploaded_files = {}
+
+        logger.info(
+            f"Uploading public share files to Google Drive folder {root_folder_id}"
+        )
+        logger.info(f"Filename base: {filename_base}")
+
+        # Upload lossy 4k to MP4/
+        lossy_mp4_path = output_files.get("final_karaoke_lossy_mp4")
+        if lossy_mp4_path and os.path.exists(lossy_mp4_path):
+            mp4_folder_id = self.get_or_create_folder(root_folder_id, "MP4")
+            file_id = self.upload_file(
+                lossy_mp4_path,
+                mp4_folder_id,
+                f"{filename_base}.mp4",
+            )
+            uploaded_files["mp4"] = file_id
+            logger.info(f"Uploaded 4K MP4 to MP4/ folder")
+
+        # Upload 720p to MP4-720p/
+        mp4_720p_path = output_files.get("final_karaoke_lossy_720p_mp4")
+        if mp4_720p_path and os.path.exists(mp4_720p_path):
+            mp4_720_folder_id = self.get_or_create_folder(root_folder_id, "MP4-720p")
+            file_id = self.upload_file(
+                mp4_720p_path,
+                mp4_720_folder_id,
+                f"{filename_base}.mp4",
+            )
+            uploaded_files["mp4_720p"] = file_id
+            logger.info(f"Uploaded 720p MP4 to MP4-720p/ folder")
+
+        # Upload CDG ZIP to CDG/
+        cdg_zip_path = output_files.get("final_karaoke_cdg_zip")
+        if cdg_zip_path and os.path.exists(cdg_zip_path):
+            cdg_folder_id = self.get_or_create_folder(root_folder_id, "CDG")
+            file_id = self.upload_file(
+                cdg_zip_path,
+                cdg_folder_id,
+                f"{filename_base}.zip",
+            )
+            uploaded_files["cdg"] = file_id
+            logger.info(f"Uploaded CDG ZIP to CDG/ folder")
+
+        logger.info(f"Public share upload complete: {len(uploaded_files)} files uploaded")
+        return uploaded_files
+
+    def delete_file(self, file_id: str) -> bool:
+        """
+        Delete a file from Google Drive.
+
+        Args:
+            file_id: Google Drive file ID to delete
+
+        Returns:
+            True if deleted successfully, False otherwise
+        """
+        logger.info(f"Deleting Google Drive file: {file_id}")
+
+        try:
+            self.service.files().delete(fileId=file_id).execute()
+            logger.info(f"Successfully deleted file: {file_id}")
+            return True
+        except Exception as e:
+            # Check if it's a 404 (already deleted)
+            if hasattr(e, 'resp') and e.resp.status == 404:
+                logger.warning(f"File not found (already deleted?): {file_id}")
+                return True
+            logger.error(f"Failed to delete Google Drive file: {e}")
+            return False
+
+    def delete_files(self, file_ids: list[str]) -> dict[str, bool]:
+        """
+        Delete multiple files from Google Drive.
+
+        Args:
+            file_ids: List of Google Drive file IDs to delete
+
+        Returns:
+            Dictionary mapping file_id to success status
+        """
+        results = {}
+        for file_id in file_ids:
+            results[file_id] = self.delete_file(file_id)
+        return results
+
+
+# Singleton instance
+_gdrive_service: Optional[GoogleDriveService] = None
+
+
+def get_gdrive_service() -> GoogleDriveService:
+    """Get the singleton Google Drive service instance."""
+    global _gdrive_service
+    if _gdrive_service is None:
+        _gdrive_service = GoogleDriveService()
+    return _gdrive_service

backend/services/job_logging.py

@@ -0,0 +1,258 @@
+"""
+Job-aware logging utilities.
+
+This module provides a context manager and logging handler that captures
+Python logging output and forwards it to Firestore for real-time streaming
+to the CLI.
+
+Usage:
+    with JobLogContext(job_id, worker="review") as log_context:
+        # Any logging within this block will be captured and sent to Firestore
+        logger.info("Processing...")  # This will be stored and streamed to CLI
+"""
+import logging
+import threading
+from typing import Optional
+from contextlib import contextmanager
+from datetime import datetime
+
+from backend.services.firestore_service import FirestoreService
+
+
+class JobLogHandler(logging.Handler):
+    """
+    A logging handler that forwards log records to Firestore for job tracking.
+    
+    This enables real-time log streaming to the CLI during long-running operations
+    like lyrics correction, preview generation, etc.
+    """
+    
+    def __init__(
+        self,
+        job_id: str,
+        worker: str = "review",
+        firestore: Optional[FirestoreService] = None,
+        min_level: int = logging.INFO,
+        batch_size: int = 1,  # Send immediately for real-time experience
+    ):
+        """
+        Initialize the job log handler.
+        
+        Args:
+            job_id: The job ID to associate logs with
+            worker: Worker name (e.g., "review", "add-lyrics", "preview")
+            firestore: FirestoreService instance (creates one if not provided)
+            min_level: Minimum log level to capture (default INFO)
+            batch_size: Number of logs to batch before sending (1 = immediate)
+        """
+        super().__init__()
+        self.job_id = job_id
+        self.worker = worker
+        self.firestore = firestore or FirestoreService()
+        self.min_level = min_level
+        self.batch_size = batch_size
+        self._log_buffer = []
+        self._lock = threading.Lock()
+        
+        # Set the handler's level
+        self.setLevel(min_level)
+        
+        # Create a formatter that extracts the useful parts
+        self.setFormatter(logging.Formatter('%(message)s'))
+    
+    def emit(self, record: logging.LogRecord) -> None:
+        """
+        Process a log record and send it to Firestore.
+        
+        This method is called by the logging framework for each log message.
+        """
+        try:
+            # Skip if below minimum level
+            if record.levelno < self.min_level:
+                return
+            
+            # Format the message
+            message = self.format(record)
+            
+            # Truncate very long messages
+            if len(message) > 1000:
+                message = message[:997] + "..."
+            
+            # Create log entry
+            log_entry = {
+                'timestamp': datetime.utcnow().isoformat() + 'Z',
+                'level': record.levelname,
+                'worker': self.worker,
+                'message': message,
+            }
+            
+            # Add to buffer
+            with self._lock:
+                self._log_buffer.append(log_entry)
+                
+                # Flush if batch size reached
+                if len(self._log_buffer) >= self.batch_size:
+                    self._flush_buffer()
+                    
+        except Exception:
+            # Never let logging errors propagate
+            self.handleError(record)
+    
+    def _flush_buffer(self) -> None:
+        """Flush buffered logs to Firestore."""
+        if not self._log_buffer:
+            return
+        
+        try:
+            for log_entry in self._log_buffer:
+                self.firestore.append_worker_log(self.job_id, log_entry)
+            self._log_buffer = []
+        except Exception:
+            # If Firestore fails, just clear buffer and continue
+            self._log_buffer = []
+    
+    def flush(self) -> None:
+        """Flush any remaining buffered logs."""
+        with self._lock:
+            self._flush_buffer()
+    
+    def close(self) -> None:
+        """Clean up the handler."""
+        self.flush()
+        super().close()
+
+
+@contextmanager
+def job_log_context(
+    job_id: str,
+    worker: str = "review",
+    logger_names: Optional[list] = None,
+    min_level: int = logging.INFO,
+):
+    """
+    Context manager that captures logs and forwards them to Firestore.
+    
+    This allows the CLI to see real-time logs from operations like
+    add-lyrics, preview generation, etc.
+    
+    Args:
+        job_id: Job ID to associate logs with
+        worker: Worker name for log categorization
+        logger_names: List of logger names to capture (None = capture common loggers)
+        min_level: Minimum log level to capture
+        
+    Usage:
+        with job_log_context(job_id, worker="add-lyrics"):
+            # All logging in this block will be captured
+            correction_operations.add_lyrics_source(...)
+    """
+    # Default loggers to capture - these cover most of our processing
+    # Note: We intentionally DON'T include the root logger ('') to avoid
+    # duplicate log entries since logs propagate up the logger hierarchy.
+    # Only capture at the top-level package loggers.
+    if logger_names is None:
+        logger_names = [
+            'backend.api.routes.review',
+            'lyrics_transcriber',
+            'karaoke_gen',
+        ]
+    
+    # Create the handler
+    handler = JobLogHandler(
+        job_id=job_id,
+        worker=worker,
+        min_level=min_level,
+    )
+    
+    # Add handler to all specified loggers
+    loggers = []
+    for name in logger_names:
+        log = logging.getLogger(name) if name else logging.getLogger()
+        log.addHandler(handler)
+        loggers.append(log)
+    
+    try:
+        # Log start
+        handler.emit(logging.LogRecord(
+            name='job_log_context',
+            level=logging.INFO,
+            pathname='',
+            lineno=0,
+            msg=f"🚀 Starting {worker} operation for job {job_id}",
+            args=(),
+            exc_info=None,
+        ))
+        
+        yield handler
+        
+    finally:
+        # Log completion
+        handler.emit(logging.LogRecord(
+            name='job_log_context',
+            level=logging.INFO,
+            pathname='',
+            lineno=0,
+            msg=f"✅ Completed {worker} operation for job {job_id}",
+            args=(),
+            exc_info=None,
+        ))
+        
+        # Flush remaining logs
+        handler.flush()
+        
+        # Remove handler from all loggers
+        for log in loggers:
+            log.removeHandler(handler)
+        
+        # Close handler
+        handler.close()
+
+
+class JobLogger:
+    """
+    A convenience class for direct logging to a job's log stream.
+    
+    Use this when you want to log specific messages without capturing
+    all logging output.
+    """
+    
+    def __init__(self, job_id: str, worker: str = "review"):
+        """
+        Initialize job logger.
+        
+        Args:
+            job_id: Job ID to log to
+            worker: Worker name for log categorization
+        """
+        self.job_id = job_id
+        self.worker = worker
+        self.firestore = FirestoreService()
+    
+    def _log(self, level: str, message: str) -> None:
+        """Internal logging method."""
+        log_entry = {
+            'timestamp': datetime.utcnow().isoformat() + 'Z',
+            'level': level,
+            'worker': self.worker,
+            'message': message[:1000],
+        }
+        try:
+            self.firestore.append_worker_log(self.job_id, log_entry)
+        except Exception:
+            pass  # Don't let logging errors propagate
+    
+    def info(self, message: str) -> None:
+        """Log an INFO message."""
+        self._log('INFO', message)
+    
+    def warning(self, message: str) -> None:
+        """Log a WARNING message."""
+        self._log('WARNING', message)
+    
+    def error(self, message: str) -> None:
+        """Log an ERROR message."""
+        self._log('ERROR', message)
+    
+    def debug(self, message: str) -> None:
+        """Log a DEBUG message."""
+        self._log('DEBUG', message)