karaoke-gen 0.90.1__py3-none-any.whl → 0.96.0__py3-none-any.whl

backend/services/structured_logging.py

@@ -0,0 +1,254 @@
+"""
+Structured logging with trace correlation for Cloud Logging.
+
+This module provides JSON-formatted logging that integrates with Google Cloud Logging
+and correlates logs with OpenTelemetry traces in Cloud Trace.
+
+When running in Cloud Run:
+- Logs are output as JSON for Cloud Logging to parse
+- Each log entry includes trace ID and span ID for correlation
+- Custom fields (job_id, worker) are preserved in the log structure
+
+Usage:
+    # In main.py, before any logging:
+    from backend.services.structured_logging import setup_structured_logging
+    setup_structured_logging()
+    
+    # Then use standard logging:
+    logger = logging.getLogger(__name__)
+    logger.info("Processing started", extra={"job_id": "abc123"})
+"""
+import json
+import logging
+import os
+import sys
+from datetime import datetime
+from typing import Any, Dict, Optional
+
+
+# Cloud Logging severity mapping (compatible with GCP)
+# https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#LogSeverity
+SEVERITY_MAP = {
+    "DEBUG": "DEBUG",
+    "INFO": "INFO",
+    "WARNING": "WARNING",
+    "ERROR": "ERROR",
+    "CRITICAL": "CRITICAL",
+}
+
+
+class StructuredFormatter(logging.Formatter):
+    """
+    JSON formatter with trace correlation for Google Cloud Logging.
+    
+    Produces log entries in the format expected by Cloud Logging, including:
+    - logging.googleapis.com/trace: Links to Cloud Trace
+    - logging.googleapis.com/spanId: Current span ID
+    - severity: Cloud Logging severity level
+    - Custom fields passed via 'extra' dict
+    """
+    
+    def __init__(self, project_id: Optional[str] = None):
+        """
+        Initialize the structured formatter.
+        
+        Args:
+            project_id: GCP project ID for trace URLs (auto-detected if not provided)
+        """
+        super().__init__()
+        self.project_id = project_id or os.environ.get("GOOGLE_CLOUD_PROJECT") or os.environ.get("GCP_PROJECT")
+    
+    def format(self, record: logging.LogRecord) -> str:
+        """
+        Format a log record as JSON.
+        
+        Args:
+            record: The log record to format
+            
+        Returns:
+            JSON string for Cloud Logging
+        """
+        from backend.services.tracing import get_current_trace_id, get_current_span_id
+        
+        # Build base log entry
+        log_entry: Dict[str, Any] = {
+            "timestamp": datetime.utcfromtimestamp(record.created).isoformat() + "Z",
+            "severity": SEVERITY_MAP.get(record.levelname, record.levelname),
+            "message": record.getMessage(),
+            "logger": record.name,
+        }
+        
+        # Add trace correlation if available
+        trace_id = get_current_trace_id()
+        span_id = get_current_span_id()
+        
+        if trace_id and self.project_id:
+            # Format for Cloud Logging trace correlation
+            log_entry["logging.googleapis.com/trace"] = f"projects/{self.project_id}/traces/{trace_id}"
+        
+        if span_id:
+            log_entry["logging.googleapis.com/spanId"] = span_id
+        
+        # Add custom fields from 'extra' dict
+        # Common fields we want to extract from log records
+        custom_fields = [
+            # Job-related fields
+            "job_id", "worker", "operation", "duration", "status", "error",
+            # Audit logging fields (from middleware and auth)
+            "request_id", "user_email", "client_ip", "latency_ms",
+            "audit_type", "method", "path", "status_code", "query_string",
+            "user_agent", "user_type", "is_admin", "remaining_uses",
+            "auth_message", "token_provided", "token_length", "auth_header_present",
+        ]
+        for field in custom_fields:
+            value = getattr(record, field, None)
+            if value is not None:
+                log_entry[field] = value
+        
+        # Add source location for debugging
+        if record.pathname and record.lineno:
+            log_entry["logging.googleapis.com/sourceLocation"] = {
+                "file": record.pathname,
+                "line": record.lineno,
+                "function": record.funcName,
+            }
+        
+        # Add exception info if present
+        if record.exc_info:
+            log_entry["exception"] = self.formatException(record.exc_info)
+        
+        # Remove None values for cleaner output
+        return json.dumps({k: v for k, v in log_entry.items() if v is not None})
+
+
+class HumanReadableFormatter(logging.Formatter):
+    """
+    Human-readable formatter for local development.
+    
+    Includes trace context when available but outputs in traditional format.
+    """
+    
+    def format(self, record: logging.LogRecord) -> str:
+        """Format a log record for human readability."""
+        from backend.services.tracing import get_current_trace_id
+        
+        # Build base message
+        timestamp = datetime.utcfromtimestamp(record.created).strftime("%Y-%m-%d %H:%M:%S")
+        
+        # Add job_id prefix if available
+        job_id = getattr(record, "job_id", None)
+        job_prefix = f"[job:{job_id[:8]}] " if job_id else ""
+        
+        # Add trace ID suffix if available
+        trace_id = get_current_trace_id()
+        trace_suffix = f" [trace:{trace_id[:8]}]" if trace_id else ""
+        
+        message = f"{timestamp} - {record.name} - {record.levelname} - {job_prefix}{record.getMessage()}{trace_suffix}"
+        
+        # Add exception info if present
+        if record.exc_info:
+            message += "\n" + self.formatException(record.exc_info)
+        
+        return message
+
+
+def is_cloud_run() -> bool:
+    """Check if we're running in Cloud Run."""
+    return os.environ.get("K_SERVICE") is not None
+
+
+def setup_structured_logging(force_json: bool = False, log_level: Optional[str] = None) -> None:
+    """
+    Configure structured logging for the application.
+    
+    In Cloud Run: Uses JSON format with trace correlation for Cloud Logging
+    In development: Uses human-readable format with optional trace context
+    
+    Args:
+        force_json: Force JSON output even in development
+        log_level: Override log level (default from settings or INFO)
+    """
+    from backend.config import settings
+    
+    # Determine log level
+    level = log_level or getattr(settings, "log_level", "INFO")
+    level = getattr(logging, level.upper(), logging.INFO)
+    
+    # Choose formatter based on environment
+    if is_cloud_run() or force_json:
+        formatter = StructuredFormatter()
+    else:
+        formatter = HumanReadableFormatter()
+    
+    # Create handler
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(formatter)
+    handler.setLevel(level)
+    
+    # Configure root logger
+    root_logger = logging.getLogger()
+    root_logger.setLevel(level)
+    
+    # Remove existing handlers and add our handler
+    root_logger.handlers = [handler]
+    
+    # Also configure uvicorn loggers to use our format
+    for logger_name in ["uvicorn", "uvicorn.access", "uvicorn.error"]:
+        logger = logging.getLogger(logger_name)
+        logger.handlers = [handler]
+        logger.propagate = False
+
+
+class JobLogAdapter(logging.LoggerAdapter):
+    """
+    Logger adapter that automatically adds job context to log records.
+    
+    Usage:
+        logger = logging.getLogger(__name__)
+        job_logger = JobLogAdapter(logger, job_id="abc123", worker="audio")
+        job_logger.info("Processing started")  # Automatically includes job_id and worker
+    """
+    
+    def __init__(self, logger: logging.Logger, job_id: str, worker: Optional[str] = None, **extra):
+        """
+        Initialize the adapter.
+        
+        Args:
+            logger: Base logger instance
+            job_id: Job ID to include in all log records
+            worker: Optional worker name
+            **extra: Additional fields to include in all log records
+        """
+        super().__init__(logger, {"job_id": job_id, "worker": worker, **extra})
+    
+    def process(self, msg: str, kwargs: Dict[str, Any]) -> tuple:
+        """Process a logging call to add extra context."""
+        # Merge extra dict from adapter with any extra passed to the log call
+        extra = kwargs.get("extra", {})
+        extra.update(self.extra)
+        kwargs["extra"] = extra
+        return msg, kwargs
+
+
+def get_job_logger(job_id: str, worker: Optional[str] = None, name: Optional[str] = None) -> JobLogAdapter:
+    """
+    Get a logger adapter configured for a specific job.
+    
+    This is a convenience function for creating job-specific loggers.
+    
+    Args:
+        job_id: Job ID to include in all log records
+        worker: Optional worker name
+        name: Logger name (defaults to "job")
+        
+    Returns:
+        JobLogAdapter configured for the job
+        
+    Usage:
+        logger = get_job_logger("abc123", "audio")
+        logger.info("Starting audio separation")
+        logger.error("Failed to process", extra={"error": str(e)})
+    """
+    base_logger = logging.getLogger(name or "job")
+    return JobLogAdapter(base_logger, job_id=job_id, worker=worker)
+

backend/services/template_service.py

@@ -0,0 +1,330 @@
+"""
+Template service for managing email templates stored in GCS.
+
+Templates are stored in GCS bucket and rendered with job-specific variables.
+This allows updating email content without code deployment.
+
+Template locations:
+- gs://{bucket}/templates/job-completion.txt - Job completion email (plain text)
+- gs://{bucket}/templates/action-needed-lyrics.txt - Lyrics review reminder
+- gs://{bucket}/templates/action-needed-instrumental.txt - Instrumental selection reminder
+"""
+import logging
+import re
+from typing import Optional, Dict, Any
+
+from backend.config import get_settings
+
+
+logger = logging.getLogger(__name__)
+
+
+# Default templates (fallback if GCS fetch fails)
+DEFAULT_JOB_COMPLETION_TEMPLATE = """Hi {name},
+
+Thanks for your order!
+
+Here's the link for the karaoke video published to YouTube:
+{youtube_url}
+
+Here's the dropbox folder with all the finished files and source files, including:
+- "(Final Karaoke Lossless).mkv": combined karaoke video in 4k H264 with lossless FLAC audio
+- "(Final Karaoke).mp4": combined karaoke video with title/end screen in 4k H264/AAC
+- "(Final Karaoke 720p).mp4": combined karaoke video in 720p H264/AAC (smaller file for older systems)
+- "(With Vocals).mp4": sing along video in 4k H264/AAC with original vocals
+- "(Karaoke).mov": karaoke video output from MidiCo (no title/end screen)
+- "(Title).mov"/"(End).mov": title card and end screen videos
+- "(Final Karaoke CDG).zip": CDG+MP3 format for older/commercial karaoke systems
+- "(Final Karaoke TXT).zip": TXT+MP3 format for Power Karaoke
+- stems/*.flac: various separated instrumental and vocal audio stems in lossless format
+- lyrics/*.txt song lyrics from various sources in plain text format
+
+{dropbox_url}
+
+Let me know if anything isn't perfect and I'll happily tweak / fix, or if you need it in any other format I can probably convert it for you!
+
+If you have a moment, I'd really appreciate your feedback (takes 2 minutes):
+{feedback_url}
+
+Thanks again and have a great day!
+-Andrew
+"""
+
+DEFAULT_ACTION_NEEDED_LYRICS_TEMPLATE = """Hi {name},
+
+Your karaoke video for "{artist} - {title}" is ready for lyrics review!
+
+Our system has transcribed and synchronized the lyrics, but they may need some corrections. Please review and make any corrections necessary so we can finish generating your video.
+
+Review your lyrics here:
+{review_url}
+
+This usually takes just a few minutes.
+
+Thanks!
+-Andrew
+"""
+
+DEFAULT_ACTION_NEEDED_INSTRUMENTAL_TEMPLATE = """Hi {name},
+
+Your karaoke video for "{artist} - {title}" is almost done!
+
+We've separated the audio into different versions. Please select which instrumental track you'd like to use for the final video.
+
+Select your instrumental here:
+{instrumental_url}
+
+Thanks!
+-Andrew
+"""
+
+
+class TemplateService:
+    """
+    Service for fetching and rendering email templates from GCS.
+
+    Templates support the following variables:
+    - {name} - User's display name or "there" if unknown
+    - {youtube_url} - YouTube video URL
+    - {dropbox_url} - Dropbox folder URL
+    - {artist} - Artist name
+    - {title} - Song title
+    - {job_id} - Job ID
+    - {review_url} - Lyrics review URL
+    - {instrumental_url} - Instrumental selection URL
+    - {feedback_url} - Feedback form URL
+    """
+
+    TEMPLATE_PREFIX = "templates/"
+
+    def __init__(self):
+        """Initialize template service."""
+        self.settings = get_settings()
+        self._storage_client = None
+        self._bucket = None
+
+    @property
+    def storage_client(self):
+        """Lazy-initialize storage client."""
+        if self._storage_client is None:
+            from google.cloud import storage
+            self._storage_client = storage.Client(project=self.settings.google_cloud_project)
+        return self._storage_client
+
+    @property
+    def bucket(self):
+        """Get the GCS bucket for templates."""
+        if self._bucket is None:
+            self._bucket = self.storage_client.bucket(self.settings.gcs_bucket_name)
+        return self._bucket
+
+    def _fetch_template_from_gcs(self, template_name: str) -> Optional[str]:
+        """
+        Fetch a template from GCS.
+
+        Args:
+            template_name: Name of template file (e.g., "job-completion.txt")
+
+        Returns:
+            Template content as string, or None if not found
+        """
+        blob_path = f"{self.TEMPLATE_PREFIX}{template_name}"
+        try:
+            blob = self.bucket.blob(blob_path)
+            if blob.exists():
+                content = blob.download_as_text()
+                logger.debug(f"Fetched template from GCS: {blob_path}")
+                return content
+            else:
+                logger.warning(f"Template not found in GCS: {blob_path}")
+                return None
+        except Exception as e:
+            logger.error(f"Failed to fetch template {blob_path}: {e}")
+            return None
+
+    def get_job_completion_template(self) -> str:
+        """Get the job completion email template."""
+        template = self._fetch_template_from_gcs("job-completion.txt")
+        if template is None:
+            logger.info("Using default job completion template")
+            return DEFAULT_JOB_COMPLETION_TEMPLATE
+        return template
+
+    def get_action_needed_lyrics_template(self) -> str:
+        """Get the lyrics review reminder template."""
+        template = self._fetch_template_from_gcs("action-needed-lyrics.txt")
+        if template is None:
+            logger.info("Using default lyrics reminder template")
+            return DEFAULT_ACTION_NEEDED_LYRICS_TEMPLATE
+        return template
+
+    def get_action_needed_instrumental_template(self) -> str:
+        """Get the instrumental selection reminder template."""
+        template = self._fetch_template_from_gcs("action-needed-instrumental.txt")
+        if template is None:
+            logger.info("Using default instrumental reminder template")
+            return DEFAULT_ACTION_NEEDED_INSTRUMENTAL_TEMPLATE
+        return template
+
+    def render_template(self, template: str, variables: Dict[str, Any]) -> str:
+        """
+        Render a template with the given variables.
+
+        Missing variables are replaced with empty strings.
+        Handles conditional sections like feedback URL.
+
+        Args:
+            template: Template string with {variable} placeholders
+            variables: Dictionary of variable values
+
+        Returns:
+            Rendered template string
+        """
+        result = template
+
+        # Handle feedback URL section - remove if not provided
+        if not variables.get("feedback_url"):
+            # Remove feedback section (lines containing feedback_url placeholder and surrounding text)
+            result = re.sub(
+                r'\n*If you have a moment.*?\{feedback_url\}\n*',
+                '\n',
+                result,
+                flags=re.DOTALL
+            )
+            variables["feedback_url"] = ""
+
+        # Replace all variables
+        for key, value in variables.items():
+            placeholder = "{" + key + "}"
+            result = result.replace(placeholder, str(value) if value else "")
+
+        # Clean up any remaining unreplaced placeholders
+        result = re.sub(r'\{[a-z_]+\}', '', result)
+
+        return result.strip()
+
+    def render_job_completion(
+        self,
+        name: Optional[str] = None,
+        youtube_url: Optional[str] = None,
+        dropbox_url: Optional[str] = None,
+        artist: Optional[str] = None,
+        title: Optional[str] = None,
+        job_id: Optional[str] = None,
+        feedback_url: Optional[str] = None,
+    ) -> str:
+        """
+        Render the job completion email template.
+
+        Args:
+            name: User's display name (defaults to "there")
+            youtube_url: YouTube video URL
+            dropbox_url: Dropbox folder URL
+            artist: Artist name
+            title: Song title
+            job_id: Job ID
+            feedback_url: Feedback form URL (optional)
+
+        Returns:
+            Rendered email content
+        """
+        template = self.get_job_completion_template()
+        variables = {
+            "name": name or "there",
+            "youtube_url": youtube_url or "[YouTube URL not available]",
+            "dropbox_url": dropbox_url or "[Dropbox URL not available]",
+            "artist": artist or "Unknown Artist",
+            "title": title or "Unknown Title",
+            "job_id": job_id or "",
+            "feedback_url": feedback_url,
+        }
+        return self.render_template(template, variables)
+
+    def render_action_needed_lyrics(
+        self,
+        name: Optional[str] = None,
+        artist: Optional[str] = None,
+        title: Optional[str] = None,
+        review_url: str = "",
+    ) -> str:
+        """
+        Render the lyrics review reminder template.
+
+        Args:
+            name: User's display name
+            artist: Artist name
+            title: Song title
+            review_url: Lyrics review URL
+
+        Returns:
+            Rendered email content
+        """
+        template = self.get_action_needed_lyrics_template()
+        variables = {
+            "name": name or "there",
+            "artist": artist or "Unknown Artist",
+            "title": title or "Unknown Title",
+            "review_url": review_url,
+        }
+        return self.render_template(template, variables)
+
+    def render_action_needed_instrumental(
+        self,
+        name: Optional[str] = None,
+        artist: Optional[str] = None,
+        title: Optional[str] = None,
+        instrumental_url: str = "",
+    ) -> str:
+        """
+        Render the instrumental selection reminder template.
+
+        Args:
+            name: User's display name
+            artist: Artist name
+            title: Song title
+            instrumental_url: Instrumental selection URL
+
+        Returns:
+            Rendered email content
+        """
+        template = self.get_action_needed_instrumental_template()
+        variables = {
+            "name": name or "there",
+            "artist": artist or "Unknown Artist",
+            "title": title or "Unknown Title",
+            "instrumental_url": instrumental_url,
+        }
+        return self.render_template(template, variables)
+
+    def upload_template(self, template_name: str, content: str) -> bool:
+        """
+        Upload a template to GCS.
+
+        Args:
+            template_name: Name of template file (e.g., "job-completion.txt")
+            content: Template content
+
+        Returns:
+            True if upload successful
+        """
+        blob_path = f"{self.TEMPLATE_PREFIX}{template_name}"
+        try:
+            blob = self.bucket.blob(blob_path)
+            blob.upload_from_string(content, content_type="text/plain")
+            logger.info(f"Uploaded template to GCS: {blob_path}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to upload template {blob_path}: {e}")
+            return False
+
+
+# Global instance
+_template_service: Optional[TemplateService] = None
+
+
+def get_template_service() -> TemplateService:
+    """Get the global template service instance."""
+    global _template_service
+    if _template_service is None:
+        _template_service = TemplateService()
+    return _template_service