karaoke-gen 0.90.1__py3-none-any.whl → 0.96.0__py3-none-any.whl

backend/services/metrics.py

@@ -0,0 +1,413 @@
+"""
+Custom metrics for Cloud Monitoring.
+
+This module provides application-level metrics for tracking job processing,
+worker performance, and external API usage. Metrics can be viewed in:
+1. Cloud Logging (via structured log entries)
+2. Cloud Monitoring (via log-based metrics or OpenTelemetry)
+
+The metrics service uses a pragmatic approach:
+- Always emits metrics as structured log entries (works immediately)
+- Uses the same JSON format as Cloud Logging
+- Can be enhanced with OpenTelemetry metrics exporters when available
+
+Usage:
+    from backend.services.metrics import metrics
+    
+    # Record a job completion
+    metrics.record_job_completed("abc123", source="upload")
+    
+    # Record worker duration
+    metrics.record_worker_duration("audio", 45.2, success=True)
+    
+    # Record external API call
+    with metrics.time_external_api("modal"):
+        response = await modal_client.separate_audio(...)
+"""
+import logging
+import time
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from typing import Any, Dict, Optional
+
+from backend.services.tracing import get_current_trace_id, get_current_span_id
+
+
+logger = logging.getLogger("metrics")
+
+
+@dataclass
+class MetricLabels:
+    """Common metric labels."""
+    job_id: Optional[str] = None
+    worker: Optional[str] = None
+    status: Optional[str] = None
+    source: Optional[str] = None
+    api: Optional[str] = None
+    operation: Optional[str] = None
+    bucket: Optional[str] = None
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dict, excluding None values."""
+        return {k: v for k, v in self.__dict__.items() if v is not None}
+
+
+class MetricsService:
+    """
+    Application metrics service.
+    
+    Emits metrics as structured log entries that can be:
+    1. Queried directly in Cloud Logging
+    2. Converted to Cloud Monitoring metrics via log-based metrics
+    3. Exported via OpenTelemetry (when configured)
+    """
+    
+    def __init__(self):
+        """Initialize the metrics service."""
+        self._logger = logging.getLogger("metrics")
+        # Ensure metrics logger outputs at INFO level
+        self._logger.setLevel(logging.INFO)
+    
+    def _emit_metric(self, metric_name: str, metric_type: str, value: float, labels: Dict[str, Any]) -> None:
+        """
+        Emit a metric as a structured log entry.
+        
+        The log format is designed to be easily parsed by Cloud Logging
+        and converted to log-based metrics.
+        
+        Args:
+            metric_name: Name of the metric (e.g., "jobs_total")
+            metric_type: Type of metric (counter, histogram, gauge)
+            value: Metric value
+            labels: Metric labels/dimensions
+        """
+        # Add trace context if available
+        trace_id = get_current_trace_id()
+        span_id = get_current_span_id()
+        
+        # Build metric entry
+        metric_entry = {
+            "metric_name": metric_name,
+            "metric_type": metric_type,
+            "metric_value": value,
+            **labels,
+        }
+        
+        if trace_id:
+            metric_entry["trace_id"] = trace_id
+        if span_id:
+            metric_entry["span_id"] = span_id
+        
+        # Emit as structured log entry
+        # Use INFO level so metrics always show up
+        self._logger.info(
+            f"METRIC {metric_name}={value}",
+            extra=metric_entry
+        )
+    
+    # =========================================
+    # Job Metrics
+    # =========================================
+    
+    def record_job_created(self, job_id: str, source: str = "unknown") -> None:
+        """
+        Record a new job creation.
+        
+        Args:
+            job_id: Job ID
+            source: Job source (upload, url, search)
+        """
+        self._emit_metric(
+            metric_name="jobs_total",
+            metric_type="counter",
+            value=1,
+            labels={"job_id": job_id, "status": "created", "source": source}
+        )
+    
+    def record_job_completed(self, job_id: str, source: str = "unknown") -> None:
+        """
+        Record a job completion.
+        
+        Args:
+            job_id: Job ID
+            source: Job source (upload, url, search)
+        """
+        self._emit_metric(
+            metric_name="jobs_total",
+            metric_type="counter",
+            value=1,
+            labels={"job_id": job_id, "status": "completed", "source": source}
+        )
+    
+    def record_job_failed(self, job_id: str, source: str = "unknown", error: Optional[str] = None) -> None:
+        """
+        Record a job failure.
+        
+        Args:
+            job_id: Job ID
+            source: Job source (upload, url, search)
+            error: Optional error message
+        """
+        labels = {"job_id": job_id, "status": "failed", "source": source}
+        if error:
+            labels["error"] = error[:200]  # Truncate long errors
+        self._emit_metric(
+            metric_name="jobs_total",
+            metric_type="counter",
+            value=1,
+            labels=labels
+        )
+    
+    def record_job_duration(self, job_id: str, duration_seconds: float, source: str = "unknown") -> None:
+        """
+        Record total job processing duration.
+        
+        Args:
+            job_id: Job ID
+            duration_seconds: Total processing time in seconds
+            source: Job source (upload, url, search)
+        """
+        self._emit_metric(
+            metric_name="job_duration_seconds",
+            metric_type="histogram",
+            value=duration_seconds,
+            labels={"job_id": job_id, "source": source}
+        )
+    
+    # =========================================
+    # Worker Metrics
+    # =========================================
+    
+    def record_worker_started(self, worker: str, job_id: str) -> None:
+        """
+        Record a worker invocation start.
+        
+        Args:
+            worker: Worker name (audio, lyrics, screens, video, render_video)
+            job_id: Job ID
+        """
+        self._emit_metric(
+            metric_name="worker_invocations_total",
+            metric_type="counter",
+            value=1,
+            labels={"worker": worker, "job_id": job_id, "status": "started"}
+        )
+    
+    def record_worker_duration(self, worker: str, duration_seconds: float, success: bool, job_id: Optional[str] = None) -> None:
+        """
+        Record worker processing duration.
+        
+        Args:
+            worker: Worker name
+            duration_seconds: Processing time in seconds
+            success: Whether worker completed successfully
+            job_id: Optional job ID
+        """
+        labels = {
+            "worker": worker,
+            "success": str(success).lower(),
+        }
+        if job_id:
+            labels["job_id"] = job_id
+        
+        self._emit_metric(
+            metric_name="job_stage_duration_seconds",
+            metric_type="histogram",
+            value=duration_seconds,
+            labels=labels
+        )
+        
+        # Also emit a counter for success/failure tracking
+        self._emit_metric(
+            metric_name="worker_invocations_total",
+            metric_type="counter",
+            value=1,
+            labels={"worker": worker, "success": str(success).lower(), "job_id": job_id or "unknown"}
+        )
+    
+    # =========================================
+    # GCS Metrics
+    # =========================================
+    
+    def record_gcs_operation(
+        self,
+        operation: str,
+        bucket: str,
+        success: bool,
+        size_bytes: Optional[int] = None,
+        duration_seconds: Optional[float] = None,
+        job_id: Optional[str] = None,
+    ) -> None:
+        """
+        Record a GCS operation.
+        
+        Args:
+            operation: Operation type (upload, download, delete)
+            bucket: GCS bucket name
+            success: Whether operation succeeded
+            size_bytes: Optional file size in bytes
+            duration_seconds: Optional operation duration
+            job_id: Optional job ID
+        """
+        labels = {
+            "operation": operation,
+            "bucket": bucket,
+            "success": str(success).lower(),
+        }
+        if job_id:
+            labels["job_id"] = job_id
+        
+        self._emit_metric(
+            metric_name="gcs_operations_total",
+            metric_type="counter",
+            value=1,
+            labels=labels
+        )
+        
+        if size_bytes is not None:
+            self._emit_metric(
+                metric_name="gcs_operation_bytes",
+                metric_type="histogram",
+                value=size_bytes,
+                labels={**labels, "operation": operation}
+            )
+        
+        if duration_seconds is not None:
+            self._emit_metric(
+                metric_name="gcs_operation_duration_seconds",
+                metric_type="histogram",
+                value=duration_seconds,
+                labels={**labels, "operation": operation}
+            )
+    
+    # =========================================
+    # External API Metrics
+    # =========================================
+    
+    def record_external_api_call(
+        self,
+        api: str,
+        success: bool,
+        duration_seconds: float,
+        job_id: Optional[str] = None,
+        error: Optional[str] = None,
+    ) -> None:
+        """
+        Record an external API call.
+        
+        Args:
+            api: API name (modal, audioshake, genius, spotify)
+            success: Whether call succeeded
+            duration_seconds: API call duration
+            job_id: Optional job ID
+            error: Optional error message
+        """
+        labels = {
+            "api": api,
+            "success": str(success).lower(),
+        }
+        if job_id:
+            labels["job_id"] = job_id
+        if error:
+            labels["error"] = error[:100]  # Truncate
+        
+        self._emit_metric(
+            metric_name="external_api_calls_total",
+            metric_type="counter",
+            value=1,
+            labels=labels
+        )
+        
+        self._emit_metric(
+            metric_name="external_api_duration_seconds",
+            metric_type="histogram",
+            value=duration_seconds,
+            labels={"api": api, "success": str(success).lower()}
+        )
+    
+    @contextmanager
+    def time_external_api(self, api: str, job_id: Optional[str] = None):
+        """
+        Context manager to time an external API call.
+        
+        Usage:
+            with metrics.time_external_api("modal", job_id) as timer:
+                response = await client.call_api()
+                timer.set_success(True)
+        
+        Args:
+            api: API name
+            job_id: Optional job ID
+            
+        Yields:
+            Timer object with set_success() method
+        """
+        timer = _ApiTimer()
+        start_time = time.time()
+        
+        try:
+            yield timer
+        except Exception as e:
+            timer.set_success(False)
+            timer.error = str(e)
+            raise
+        finally:
+            duration = time.time() - start_time
+            self.record_external_api_call(
+                api=api,
+                success=timer.success,
+                duration_seconds=duration,
+                job_id=job_id,
+                error=timer.error,
+            )
+    
+    @contextmanager
+    def time_worker(self, worker: str, job_id: str):
+        """
+        Context manager to time a worker execution.
+        
+        Usage:
+            with metrics.time_worker("audio", job_id) as timer:
+                await process_audio()
+                timer.set_success(True)
+        
+        Args:
+            worker: Worker name
+            job_id: Job ID
+            
+        Yields:
+            Timer object with set_success() method
+        """
+        timer = _ApiTimer()
+        self.record_worker_started(worker, job_id)
+        start_time = time.time()
+        
+        try:
+            yield timer
+        except Exception as e:
+            timer.set_success(False)
+            raise
+        finally:
+            duration = time.time() - start_time
+            self.record_worker_duration(
+                worker=worker,
+                duration_seconds=duration,
+                success=timer.success,
+                job_id=job_id,
+            )
+
+
+class _ApiTimer:
+    """Helper class for tracking API call success state."""
+    
+    def __init__(self):
+        self.success = True  # Assume success unless set otherwise
+        self.error: Optional[str] = None
+    
+    def set_success(self, success: bool) -> None:
+        self.success = success
+
+
+# Global metrics instance
+metrics = MetricsService()
+

backend/services/packaging_service.py

@@ -0,0 +1,287 @@
+"""
+Packaging Service.
+
+Provides CDG and TXT package generation functionality, extracted from KaraokeFinalise
+for use by both the cloud backend (video_worker) and local CLI.
+
+This service handles:
+- CDG (CD+G) file generation from LRC files
+- TXT lyric file generation from LRC files
+- ZIP packaging of CDG/MP3 and TXT/MP3 pairs
+"""
+
+import logging
+import os
+import zipfile
+from typing import Optional, Dict, Any, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+class PackagingService:
+    """
+    Service for creating CDG and TXT karaoke packages.
+
+    CDG (CD+Graphics) is a format used by karaoke machines.
+    TXT packages are used by software karaoke players.
+    Both formats are packaged as ZIP files with an MP3 audio track.
+    """
+
+    def __init__(
+        self,
+        cdg_styles: Optional[Dict[str, Any]] = None,
+        dry_run: bool = False,
+        non_interactive: bool = False,
+        logger: Optional[logging.Logger] = None,
+    ):
+        """
+        Initialize the packaging service.
+
+        Args:
+            cdg_styles: CDG style configuration for CDG generation
+            dry_run: If True, log actions without performing them
+            non_interactive: If True, skip interactive prompts
+            logger: Optional logger instance
+        """
+        self.cdg_styles = cdg_styles
+        self.dry_run = dry_run
+        self.non_interactive = non_interactive
+        self.logger = logger or logging.getLogger(__name__)
+
+    def create_cdg_package(
+        self,
+        lrc_file: str,
+        audio_file: str,
+        output_zip_path: str,
+        artist: str,
+        title: str,
+        output_mp3_path: Optional[str] = None,
+        output_cdg_path: Optional[str] = None,
+    ) -> Tuple[str, Optional[str], Optional[str]]:
+        """
+        Create a CDG package (ZIP containing CDG and MP3 files).
+
+        Args:
+            lrc_file: Path to the LRC lyrics file
+            audio_file: Path to the instrumental audio file
+            output_zip_path: Path for the output ZIP file
+            artist: Artist name for metadata
+            title: Song title for metadata
+            output_mp3_path: Optional path for the extracted MP3 file
+            output_cdg_path: Optional path for the extracted CDG file
+
+        Returns:
+            Tuple of (zip_path, mp3_path, cdg_path)
+
+        Raises:
+            ValueError: If CDG styles are not configured
+            FileNotFoundError: If input files are missing
+            Exception: If CDG generation fails
+        """
+        self.logger.info(f"Creating CDG package for {artist} - {title}")
+
+        # Validate inputs
+        if not os.path.isfile(lrc_file):
+            raise FileNotFoundError(f"LRC file not found: {lrc_file}")
+        if not os.path.isfile(audio_file):
+            raise FileNotFoundError(f"Audio file not found: {audio_file}")
+
+        # Check if ZIP already exists
+        if os.path.isfile(output_zip_path):
+            if self.non_interactive:
+                self.logger.info(
+                    f"CDG ZIP exists, will be overwritten: {output_zip_path}"
+                )
+            else:
+                self.logger.info(f"CDG ZIP already exists: {output_zip_path}")
+
+        # Check if individual files exist (allows skipping generation)
+        if output_mp3_path and output_cdg_path:
+            if os.path.isfile(output_mp3_path) and os.path.isfile(output_cdg_path):
+                self.logger.info("Found existing MP3 and CDG files, creating ZIP directly")
+                if not self.dry_run:
+                    self._create_zip_from_files(
+                        output_zip_path,
+                        [(output_mp3_path, os.path.basename(output_mp3_path)),
+                         (output_cdg_path, os.path.basename(output_cdg_path))]
+                    )
+                return output_zip_path, output_mp3_path, output_cdg_path
+
+        if self.dry_run:
+            self.logger.info(
+                f"DRY RUN: Would generate CDG package: {output_zip_path}"
+            )
+            return output_zip_path, output_mp3_path, output_cdg_path
+
+        # Generate CDG files
+        if self.cdg_styles is None:
+            raise ValueError(
+                "CDG styles configuration is required for CDG generation"
+            )
+
+        self.logger.info("Generating CDG and MP3 files")
+        from lyrics_transcriber.output.cdg import CDGGenerator
+
+        output_dir = os.path.dirname(output_zip_path) or os.getcwd()
+        generator = CDGGenerator(output_dir=output_dir, logger=self.logger)
+
+        cdg_file, mp3_file, zip_file = generator.generate_cdg_from_lrc(
+            lrc_file=lrc_file,
+            audio_file=audio_file,
+            title=title,
+            artist=artist,
+            cdg_styles=self.cdg_styles,
+        )
+
+        # Rename ZIP to expected output path if different
+        if os.path.isfile(zip_file) and zip_file != output_zip_path:
+            os.rename(zip_file, output_zip_path)
+            self.logger.info(f"Renamed CDG ZIP: {zip_file} -> {output_zip_path}")
+
+        if not os.path.isfile(output_zip_path):
+            raise Exception(f"Failed to create CDG ZIP file: {output_zip_path}")
+
+        # Extract the ZIP to get individual files if paths provided
+        extracted_mp3 = None
+        extracted_cdg = None
+        if output_mp3_path or output_cdg_path:
+            self.logger.info(f"Extracting CDG ZIP file: {output_zip_path}")
+            with zipfile.ZipFile(output_zip_path, "r") as zip_ref:
+                zip_ref.extractall(output_dir)
+
+            # Find extracted files
+            if output_mp3_path and os.path.isfile(output_mp3_path):
+                extracted_mp3 = output_mp3_path
+                self.logger.info(f"Extracted MP3: {extracted_mp3}")
+            if output_cdg_path and os.path.isfile(output_cdg_path):
+                extracted_cdg = output_cdg_path
+                self.logger.info(f"Extracted CDG: {extracted_cdg}")
+
+        self.logger.info(f"CDG package created: {output_zip_path}")
+        return output_zip_path, extracted_mp3, extracted_cdg
+
+    def create_txt_package(
+        self,
+        lrc_file: str,
+        mp3_file: str,
+        output_zip_path: str,
+        output_txt_path: Optional[str] = None,
+    ) -> Tuple[str, Optional[str]]:
+        """
+        Create a TXT package (ZIP containing TXT lyrics and MP3 files).
+
+        Args:
+            lrc_file: Path to the LRC lyrics file
+            mp3_file: Path to the MP3 audio file
+            output_zip_path: Path for the output ZIP file
+            output_txt_path: Optional path for the generated TXT file
+
+        Returns:
+            Tuple of (zip_path, txt_path)
+
+        Raises:
+            FileNotFoundError: If input files are missing
+            Exception: If TXT generation fails
+        """
+        self.logger.info(f"Creating TXT package from {lrc_file}")
+
+        # Validate inputs
+        if not os.path.isfile(lrc_file):
+            raise FileNotFoundError(f"LRC file not found: {lrc_file}")
+        if not os.path.isfile(mp3_file):
+            raise FileNotFoundError(f"MP3 file not found: {mp3_file}")
+
+        # Check if ZIP already exists
+        if os.path.isfile(output_zip_path):
+            if self.non_interactive:
+                self.logger.info(
+                    f"TXT ZIP exists, will be overwritten: {output_zip_path}"
+                )
+            else:
+                self.logger.info(f"TXT ZIP already exists: {output_zip_path}")
+
+        if self.dry_run:
+            self.logger.info(
+                f"DRY RUN: Would create TXT package: {output_zip_path}"
+            )
+            return output_zip_path, output_txt_path
+
+        # Generate TXT from LRC
+        self.logger.info(f"Converting LRC to TXT format: {lrc_file}")
+        from lyrics_converter import LyricsConverter
+
+        txt_converter = LyricsConverter(output_format="txt", filepath=lrc_file)
+        converted_txt = txt_converter.convert_file()
+
+        # Write TXT file
+        if output_txt_path is None:
+            # Default to same name as ZIP but with .txt extension
+            output_txt_path = output_zip_path.replace(".zip", ".txt")
+
+        with open(output_txt_path, "w") as txt_file:
+            txt_file.write(converted_txt)
+            self.logger.info(f"TXT file written: {output_txt_path}")
+
+        # Create ZIP containing MP3 and TXT
+        self.logger.info(f"Creating TXT ZIP: {output_zip_path}")
+        self._create_zip_from_files(
+            output_zip_path,
+            [(mp3_file, os.path.basename(mp3_file)),
+             (output_txt_path, os.path.basename(output_txt_path))]
+        )
+
+        if not os.path.isfile(output_zip_path):
+            raise Exception(f"Failed to create TXT ZIP file: {output_zip_path}")
+
+        self.logger.info(f"TXT package created: {output_zip_path}")
+        return output_zip_path, output_txt_path
+
+    def _create_zip_from_files(
+        self,
+        zip_path: str,
+        files: list,
+    ) -> None:
+        """
+        Create a ZIP file from a list of files.
+
+        Args:
+            zip_path: Path for the output ZIP file
+            files: List of (file_path, archive_name) tuples
+        """
+        with zipfile.ZipFile(zip_path, "w") as zipf:
+            for file_path, archive_name in files:
+                if os.path.isfile(file_path):
+                    zipf.write(file_path, archive_name)
+                    self.logger.debug(f"Added to ZIP: {archive_name}")
+                else:
+                    self.logger.warning(f"File not found for ZIP: {file_path}")
+
+
+# Singleton instance and factory function (following existing service pattern)
+_packaging_service: Optional[PackagingService] = None
+
+
+def get_packaging_service(
+    cdg_styles: Optional[Dict[str, Any]] = None,
+    **kwargs
+) -> PackagingService:
+    """
+    Get a packaging service instance.
+
+    Args:
+        cdg_styles: CDG style configuration
+        **kwargs: Additional arguments passed to PackagingService
+
+    Returns:
+        PackagingService instance
+    """
+    global _packaging_service
+
+    # Create new instance if settings changed
+    if _packaging_service is None or cdg_styles:
+        _packaging_service = PackagingService(
+            cdg_styles=cdg_styles,
+            **kwargs
+        )
+
+    return _packaging_service