karaoke-gen 0.101.0__py3-none-any.whl → 0.103.1__py3-none-any.whl

backend/api/routes/rate_limits.py

@@ -0,0 +1,428 @@
+"""
+Admin API routes for rate limit management.
+
+Handles:
+- Rate limit statistics and monitoring
+- Blocklist management (disposable domains, blocked emails, blocked IPs)
+- User override management (whitelist/bypass permissions)
+"""
+
+import logging
+from datetime import datetime, timezone
+from typing import List, Optional
+
+from fastapi import APIRouter, Depends, HTTPException
+from pydantic import BaseModel, EmailStr
+
+from backend.api.dependencies import require_admin
+from backend.services.auth_service import AuthResult
+from backend.services.rate_limit_service import get_rate_limit_service, RateLimitService
+from backend.services.email_validation_service import get_email_validation_service, EmailValidationService
+from backend.config import settings
+
+logger = logging.getLogger(__name__)
+router = APIRouter(prefix="/admin/rate-limits", tags=["admin", "rate-limits"])
+
+
+# =============================================================================
+# Response Models
+# =============================================================================
+
+class RateLimitStatsResponse(BaseModel):
+    """Current rate limit statistics."""
+    # Configuration
+    jobs_per_day_limit: int
+    youtube_uploads_per_day_limit: int
+    beta_ip_per_day_limit: int
+    rate_limiting_enabled: bool
+
+    # Current usage
+    youtube_uploads_today: int
+    youtube_uploads_remaining: int
+
+    # Blocklist stats
+    disposable_domains_count: int
+    blocked_emails_count: int
+    blocked_ips_count: int
+
+    # User override stats
+    total_overrides: int
+
+
+class UserRateLimitStatusResponse(BaseModel):
+    """Rate limit status for a specific user."""
+    email: str
+    jobs_today: int
+    jobs_limit: int
+    jobs_remaining: int
+    has_bypass: bool
+    custom_limit: Optional[int]
+    bypass_reason: Optional[str]
+
+
+class BlocklistsResponse(BaseModel):
+    """All blocklist data."""
+    disposable_domains: List[str]
+    blocked_emails: List[str]
+    blocked_ips: List[str]
+    updated_at: Optional[datetime]
+    updated_by: Optional[str]
+
+
+class DomainRequest(BaseModel):
+    """Request to add/remove a domain."""
+    domain: str
+
+
+class EmailRequest(BaseModel):
+    """Request to add/remove an email."""
+    email: str
+
+
+class IPRequest(BaseModel):
+    """Request to add/remove an IP."""
+    ip_address: str
+
+
+class UserOverride(BaseModel):
+    """User override configuration."""
+    email: str
+    bypass_job_limit: bool
+    custom_daily_job_limit: Optional[int]
+    reason: str
+    created_by: str
+    created_at: datetime
+
+
+class UserOverrideRequest(BaseModel):
+    """Request to set/update a user override."""
+    bypass_job_limit: bool = False
+    custom_daily_job_limit: Optional[int] = None
+    reason: str
+
+
+class UserOverridesListResponse(BaseModel):
+    """List of all user overrides."""
+    overrides: List[UserOverride]
+    total: int
+
+
+class SuccessResponse(BaseModel):
+    """Generic success response."""
+    success: bool
+    message: str
+
+
+# =============================================================================
+# Statistics Endpoints
+# =============================================================================
+
+@router.get("/stats", response_model=RateLimitStatsResponse)
+async def get_rate_limit_stats(
+    auth_result: AuthResult = Depends(require_admin),
+):
+    """
+    Get current rate limit statistics.
+
+    Returns configuration values, current usage counts, and blocklist stats.
+    """
+    rate_limit_service = get_rate_limit_service()
+    email_validation = get_email_validation_service()
+
+    # Get YouTube uploads today
+    youtube_today = rate_limit_service.get_youtube_uploads_today()
+    youtube_limit = settings.rate_limit_youtube_uploads_per_day
+    youtube_remaining = max(0, youtube_limit - youtube_today) if youtube_limit > 0 else -1
+
+    # Get blocklist stats
+    blocklist_stats = email_validation.get_blocklist_stats()
+
+    # Get override count
+    overrides = rate_limit_service.get_all_overrides()
+
+    return RateLimitStatsResponse(
+        jobs_per_day_limit=settings.rate_limit_jobs_per_day,
+        youtube_uploads_per_day_limit=settings.rate_limit_youtube_uploads_per_day,
+        beta_ip_per_day_limit=settings.rate_limit_beta_ip_per_day,
+        rate_limiting_enabled=settings.enable_rate_limiting,
+        youtube_uploads_today=youtube_today,
+        youtube_uploads_remaining=youtube_remaining,
+        disposable_domains_count=blocklist_stats["disposable_domains_count"],
+        blocked_emails_count=blocklist_stats["blocked_emails_count"],
+        blocked_ips_count=blocklist_stats["blocked_ips_count"],
+        total_overrides=len(overrides),
+    )
+
+
+@router.get("/users/{email}", response_model=UserRateLimitStatusResponse)
+async def get_user_rate_limit_status(
+    email: str,
+    auth_result: AuthResult = Depends(require_admin),
+):
+    """
+    Get rate limit status for a specific user.
+
+    Returns their current usage and any override settings.
+    """
+    rate_limit_service = get_rate_limit_service()
+
+    jobs_today = rate_limit_service.get_user_job_count_today(email)
+    override = rate_limit_service.get_user_override(email)
+
+    if override:
+        has_bypass = override.get("bypass_job_limit", False)
+        custom_limit = override.get("custom_daily_job_limit")
+        bypass_reason = override.get("reason")
+        jobs_limit = custom_limit if custom_limit is not None else settings.rate_limit_jobs_per_day
+    else:
+        has_bypass = False
+        custom_limit = None
+        bypass_reason = None
+        jobs_limit = settings.rate_limit_jobs_per_day
+
+    jobs_remaining = max(0, jobs_limit - jobs_today) if not has_bypass else -1
+
+    return UserRateLimitStatusResponse(
+        email=email,
+        jobs_today=jobs_today,
+        jobs_limit=jobs_limit,
+        jobs_remaining=jobs_remaining,
+        has_bypass=has_bypass,
+        custom_limit=custom_limit,
+        bypass_reason=bypass_reason,
+    )
+
+
+# =============================================================================
+# Blocklist Management Endpoints
+# =============================================================================
+
+@router.get("/blocklists", response_model=BlocklistsResponse)
+async def get_blocklists(
+    auth_result: AuthResult = Depends(require_admin),
+):
+    """
+    Get all blocklist data.
+
+    Returns disposable domains, blocked emails, and blocked IPs.
+    """
+    email_validation = get_email_validation_service()
+
+    # Force refresh to get latest data
+    config = email_validation.get_blocklist_config(force_refresh=True)
+
+    # Get the raw document for metadata
+    from google.cloud import firestore
+    from backend.services.firestore_service import get_firestore_client
+    from backend.services.email_validation_service import BLOCKLISTS_COLLECTION, BLOCKLIST_CONFIG_DOC
+
+    db = get_firestore_client()
+    doc = db.collection(BLOCKLISTS_COLLECTION).document(BLOCKLIST_CONFIG_DOC).get()
+
+    updated_at = None
+    updated_by = None
+    if doc.exists:
+        data = doc.to_dict()
+        updated_at = data.get("updated_at")
+        updated_by = data.get("updated_by")
+
+    return BlocklistsResponse(
+        disposable_domains=sorted(list(config["disposable_domains"])),
+        blocked_emails=sorted(list(config["blocked_emails"])),
+        blocked_ips=sorted(list(config["blocked_ips"])),
+        updated_at=updated_at,
+        updated_by=updated_by,
+    )
+
+
+@router.post("/blocklists/disposable-domains", response_model=SuccessResponse)
+async def add_disposable_domain(
+    request: DomainRequest,
+    auth_result: AuthResult = Depends(require_admin),
+):
+    """Add a domain to the disposable domains blocklist."""
+    email_validation = get_email_validation_service()
+
+    domain = request.domain.lower().strip()
+    if not domain or "." not in domain:
+        raise HTTPException(status_code=400, detail="Invalid domain format")
+
+    email_validation.add_disposable_domain(domain, auth_result.user_email)
+
+    return SuccessResponse(
+        success=True,
+        message=f"Domain '{domain}' added to disposable domains blocklist"
+    )
+
+
+@router.delete("/blocklists/disposable-domains/{domain}", response_model=SuccessResponse)
+async def remove_disposable_domain(
+    domain: str,
+    auth_result: AuthResult = Depends(require_admin),
+):
+    """Remove a domain from the disposable domains blocklist."""
+    email_validation = get_email_validation_service()
+
+    domain = domain.lower().strip()
+    if not email_validation.remove_disposable_domain(domain, auth_result.user_email):
+        raise HTTPException(status_code=404, detail="Domain not found in blocklist")
+
+    return SuccessResponse(
+        success=True,
+        message=f"Domain '{domain}' removed from disposable domains blocklist"
+    )
+
+
+@router.post("/blocklists/blocked-emails", response_model=SuccessResponse)
+async def add_blocked_email(
+    request: EmailRequest,
+    auth_result: AuthResult = Depends(require_admin),
+):
+    """Add an email to the blocked emails list."""
+    email_validation = get_email_validation_service()
+
+    email = request.email.lower().strip()
+    if not email or "@" not in email:
+        raise HTTPException(status_code=400, detail="Invalid email format")
+
+    email_validation.add_blocked_email(email, auth_result.user_email)
+
+    return SuccessResponse(
+        success=True,
+        message=f"Email '{email}' added to blocked emails list"
+    )
+
+
+@router.delete("/blocklists/blocked-emails/{email}", response_model=SuccessResponse)
+async def remove_blocked_email(
+    email: str,
+    auth_result: AuthResult = Depends(require_admin),
+):
+    """Remove an email from the blocked emails list."""
+    email_validation = get_email_validation_service()
+
+    email = email.lower().strip()
+    if not email_validation.remove_blocked_email(email, auth_result.user_email):
+        raise HTTPException(status_code=404, detail="Email not found in blocklist")
+
+    return SuccessResponse(
+        success=True,
+        message=f"Email '{email}' removed from blocked emails list"
+    )
+
+
+@router.post("/blocklists/blocked-ips", response_model=SuccessResponse)
+async def add_blocked_ip(
+    request: IPRequest,
+    auth_result: AuthResult = Depends(require_admin),
+):
+    """Add an IP address to the blocked IPs list."""
+    email_validation = get_email_validation_service()
+
+    ip_address = request.ip_address.strip()
+    if not ip_address:
+        raise HTTPException(status_code=400, detail="Invalid IP address")
+
+    email_validation.add_blocked_ip(ip_address, auth_result.user_email)
+
+    return SuccessResponse(
+        success=True,
+        message=f"IP '{ip_address}' added to blocked IPs list"
+    )
+
+
+@router.delete("/blocklists/blocked-ips/{ip_address}", response_model=SuccessResponse)
+async def remove_blocked_ip(
+    ip_address: str,
+    auth_result: AuthResult = Depends(require_admin),
+):
+    """Remove an IP address from the blocked IPs list."""
+    email_validation = get_email_validation_service()
+
+    ip_address = ip_address.strip()
+    if not email_validation.remove_blocked_ip(ip_address, auth_result.user_email):
+        raise HTTPException(status_code=404, detail="IP not found in blocklist")
+
+    return SuccessResponse(
+        success=True,
+        message=f"IP '{ip_address}' removed from blocked IPs list"
+    )
+
+
+# =============================================================================
+# User Override Management Endpoints
+# =============================================================================
+
+@router.get("/overrides", response_model=UserOverridesListResponse)
+async def get_all_overrides(
+    auth_result: AuthResult = Depends(require_admin),
+):
+    """Get all user overrides."""
+    rate_limit_service = get_rate_limit_service()
+
+    overrides_data = rate_limit_service.get_all_overrides()
+
+    overrides = [
+        UserOverride(
+            email=email,
+            bypass_job_limit=data.get("bypass_job_limit", False),
+            custom_daily_job_limit=data.get("custom_daily_job_limit"),
+            reason=data.get("reason", ""),
+            created_by=data.get("created_by", "unknown"),
+            created_at=data.get("created_at", datetime.now(timezone.utc)),
+        )
+        for email, data in overrides_data.items()
+    ]
+
+    return UserOverridesListResponse(
+        overrides=overrides,
+        total=len(overrides),
+    )
+
+
+@router.put("/overrides/{email}", response_model=SuccessResponse)
+async def set_user_override(
+    email: str,
+    request: UserOverrideRequest,
+    auth_result: AuthResult = Depends(require_admin),
+):
+    """Set or update a user override."""
+    rate_limit_service = get_rate_limit_service()
+
+    email = email.lower().strip()
+    if not email or "@" not in email:
+        raise HTTPException(status_code=400, detail="Invalid email format")
+
+    if not request.reason or len(request.reason.strip()) < 3:
+        raise HTTPException(status_code=400, detail="Reason is required (min 3 characters)")
+
+    rate_limit_service.set_user_override(
+        user_email=email,
+        bypass_job_limit=request.bypass_job_limit,
+        custom_daily_job_limit=request.custom_daily_job_limit,
+        reason=request.reason,
+        admin_email=auth_result.user_email,
+    )
+
+    return SuccessResponse(
+        success=True,
+        message=f"Override set for user '{email}'"
+    )
+
+
+@router.delete("/overrides/{email}", response_model=SuccessResponse)
+async def remove_user_override(
+    email: str,
+    auth_result: AuthResult = Depends(require_admin),
+):
+    """Remove a user override."""
+    rate_limit_service = get_rate_limit_service()
+
+    email = email.lower().strip()
+    if not rate_limit_service.remove_user_override(email, auth_result.user_email):
+        raise HTTPException(status_code=404, detail="Override not found for user")
+
+    return SuccessResponse(
+        success=True,
+        message=f"Override removed for user '{email}'"
+    )

backend/api/routes/users.py

@@ -53,6 +53,10 @@ from backend.services.user_service import get_user_service, UserService, USERS_C
 from backend.services.email_service import get_email_service, EmailService
 from backend.services.stripe_service import get_stripe_service, StripeService, CREDIT_PACKAGES
 from backend.services.theme_service import get_theme_service
+from backend.services.job_defaults_service import (
+    get_effective_distribution_settings,
+    resolve_cdg_txt_defaults,
+)
 from backend.api.dependencies import require_admin
 from backend.api.routes.file_upload import _prepare_theme_for_job
 from backend.services.auth_service import UserType
@@ -475,7 +479,6 @@ async def _handle_made_for_you_order(
         AudioSearchError,
     )
     from backend.services.storage_service import StorageService
-    from backend.config import get_settings
     import asyncio
     import tempfile
     import os
@@ -496,7 +499,6 @@ async def _handle_made_for_you_order(
         job_manager = JobManager()
         worker_service = get_worker_service()
         storage_service = StorageService()
-        settings = get_settings()
 
         # Apply default theme (Nomad) - same as audio_search endpoint
         theme_service = get_theme_service()
@@ -504,13 +506,12 @@ async def _handle_made_for_you_order(
         if effective_theme_id:
             logger.info(f"Applying default theme '{effective_theme_id}' for made-for-you order")
 
-        # Get distribution defaults from settings (same as audio_search endpoint)
-        effective_dropbox_path = settings.default_dropbox_path
-        effective_gdrive_folder_id = settings.default_gdrive_folder_id
-        effective_enable_youtube_upload = settings.default_enable_youtube_upload
-        effective_brand_prefix = settings.default_brand_prefix
-        effective_discord_webhook_url = settings.default_discord_webhook_url
-        effective_youtube_description = settings.default_youtube_description
+        # Get distribution defaults using centralized service
+        dist = get_effective_distribution_settings()
+
+        # Resolve CDG/TXT defaults based on theme (uses centralized service)
+        # This ensures made-for-you jobs get the same CDG/TXT defaults as regular jobs
+        resolved_cdg, resolved_txt = resolve_cdg_txt_defaults(effective_theme_id)
 
         # Create job with admin ownership during processing
         # CRITICAL: made_for_you=True, user_email=ADMIN_EMAIL, customer_email for delivery
@@ -530,15 +531,19 @@ async def _handle_made_for_you_order(
             audio_search_artist=artist if not youtube_url else None,
             audio_search_title=title if not youtube_url else None,
             auto_download=False,  # Pause at audio selection for admin to choose
-            # Distribution settings from server defaults
-            enable_youtube_upload=effective_enable_youtube_upload,
-            dropbox_path=effective_dropbox_path,
-            gdrive_folder_id=effective_gdrive_folder_id,
-            brand_prefix=effective_brand_prefix,
-            discord_webhook_url=effective_discord_webhook_url,
-            youtube_description=effective_youtube_description,
+            # CDG/TXT settings (resolved via centralized service)
+            enable_cdg=resolved_cdg,
+            enable_txt=resolved_txt,
+            # Distribution settings from centralized defaults
+            enable_youtube_upload=dist.enable_youtube_upload,
+            dropbox_path=dist.dropbox_path,
+            gdrive_folder_id=dist.gdrive_folder_id,
+            brand_prefix=dist.brand_prefix,
+            discord_webhook_url=dist.discord_webhook_url,
+            youtube_description=dist.youtube_description,
         )
-        job = job_manager.create_job(job_create)
+        # Made-for-you jobs are created by admin (via Stripe webhook) - bypass rate limits
+        job = job_manager.create_job(job_create, is_admin=True)
         job_id = job.job_id
 
         logger.info(f"Created made-for-you job {job_id} for {_mask_email(customer_email)} (owned by {_mask_email(ADMIN_EMAIL)})")
@@ -818,6 +823,9 @@ async def enroll_beta_tester(
 
     Returns free credits and optionally a session token for new users.
     """
+    from backend.services.email_validation_service import get_email_validation_service
+    from backend.services.rate_limit_service import get_rate_limit_service
+
     # Check if email service is configured
     if not email_service.is_configured():
         logger.error("Email service not configured - cannot send beta welcome emails")
@@ -827,6 +835,52 @@ async def enroll_beta_tester(
         )
 
     email = request.email.lower()
+    email_validation = get_email_validation_service()
+    rate_limit_service = get_rate_limit_service()
+
+    # ----- ABUSE PREVENTION CHECKS -----
+
+    # 1. Validate email (disposable domain, blocked email checks)
+    is_valid, error_message = email_validation.validate_email_for_beta(email)
+    if not is_valid:
+        logger.warning(f"Beta enrollment rejected - email validation failed: {email} - {error_message}")
+        raise HTTPException(
+            status_code=400,
+            detail=error_message
+        )
+
+    # 2. Check IP blocking
+    ip_address = http_request.client.host if http_request.client else None
+    if ip_address and email_validation.is_ip_blocked(ip_address):
+        logger.warning(f"Beta enrollment rejected - IP blocked: {ip_address}")
+        raise HTTPException(
+            status_code=403,
+            detail="Access denied from this location"
+        )
+
+    # 3. Check IP-based enrollment rate limit (1 per 24h per IP)
+    if ip_address:
+        allowed, remaining, message = rate_limit_service.check_beta_ip_limit(ip_address)
+        if not allowed:
+            logger.warning(f"Beta enrollment rejected - IP rate limit: {ip_address} - {message}")
+            raise HTTPException(
+                status_code=429,
+                detail="Too many beta enrollments from your location. Please try again tomorrow."
+            )
+
+    # 4. Check for duplicate enrollment via normalized email
+    normalized_email = email_validation.normalize_email(email)
+    if normalized_email != email:
+        # Check if normalized version is already enrolled
+        normalized_user = user_service.get_user(normalized_email)
+        if normalized_user and normalized_user.is_beta_tester:
+            logger.warning(f"Beta enrollment rejected - normalized email already enrolled: {email} -> {normalized_email}")
+            raise HTTPException(
+                status_code=400,
+                detail="An account with this email address is already enrolled in the beta program"
+            )
+
+    # ----- END ABUSE PREVENTION CHECKS -----
 
     # Validate acceptance
     if not request.accept_corrections_work:
@@ -851,8 +905,7 @@ async def enroll_beta_tester(
             detail="You are already enrolled in the beta program"
         )
 
-    # Get client info
-    ip_address = http_request.client.host if http_request.client else None
+    # Get client info (ip_address already set above in abuse prevention)
     user_agent = http_request.headers.get("user-agent")
 
     # Enroll as beta tester
@@ -872,6 +925,13 @@ async def enroll_beta_tester(
         reason="beta_tester_enrollment",
     )
 
+    # Record IP enrollment for rate limiting
+    if ip_address:
+        try:
+            rate_limit_service.record_beta_enrollment(ip_address, email)
+        except Exception as e:
+            logger.warning(f"Failed to record beta IP enrollment: {e}")
+
     # Create session for the user (so they can start using the service immediately)
     session = user_service.create_session(email, ip_address=ip_address, user_agent=user_agent)
 

backend/config.py

@@ -101,12 +101,28 @@ class Settings(BaseSettings):
     # Default values for web service jobs (YouTube/Dropbox distribution)
     default_enable_youtube_upload: bool = os.getenv("DEFAULT_ENABLE_YOUTUBE_UPLOAD", "false").lower() in ("true", "1", "yes")
     default_brand_prefix: Optional[str] = os.getenv("DEFAULT_BRAND_PREFIX")
+
+    # Rate Limiting Configuration
+    # Enable/disable rate limiting system-wide (useful for development)
+    enable_rate_limiting: bool = os.getenv("ENABLE_RATE_LIMITING", "true").lower() in ("true", "1", "yes")
+    # Maximum jobs a user can create per day (0 = unlimited)
+    rate_limit_jobs_per_day: int = int(os.getenv("RATE_LIMIT_JOBS_PER_DAY", "5"))
+    # Maximum YouTube uploads system-wide per day (0 = unlimited)
+    rate_limit_youtube_uploads_per_day: int = int(os.getenv("RATE_LIMIT_YOUTUBE_UPLOADS_PER_DAY", "10"))
+    # Maximum beta enrollments from same IP per day (0 = unlimited)
+    rate_limit_beta_ip_per_day: int = int(os.getenv("RATE_LIMIT_BETA_IP_PER_DAY", "1"))
     default_youtube_description: str = os.getenv(
         "DEFAULT_YOUTUBE_DESCRIPTION",
         "Karaoke video created with Nomad Karaoke (https://nomadkaraoke.com)\n\n"
         "AI-powered vocal separation and synchronized lyrics.\n\n"
         "#karaoke #music #singing #instrumental #lyrics"
     )
+
+    # Default CDG/TXT generation settings
+    # When True, CDG and TXT packages are generated by default (when a theme is set)
+    # These can be overridden per-request via explicit enable_cdg/enable_txt parameters
+    default_enable_cdg: bool = os.getenv("DEFAULT_ENABLE_CDG", "true").lower() in ("true", "1", "yes")
+    default_enable_txt: bool = os.getenv("DEFAULT_ENABLE_TXT", "true").lower() in ("true", "1", "yes")
     
     # Secret Manager cache
     _secret_cache: Dict[str, str] = {}

backend/exceptions.py

@@ -0,0 +1,66 @@
+"""
+Custom exceptions for the karaoke generation backend.
+
+These exceptions are used for structured error handling across the application,
+particularly for rate limiting and validation errors.
+"""
+
+
+class RateLimitExceededError(Exception):
+    """
+    Raised when a rate limit is exceeded.
+
+    Includes information about when the limit will reset for client retry logic.
+
+    Attributes:
+        message: Human-readable error message
+        limit_type: Type of limit exceeded (e.g., "jobs_per_day", "youtube_uploads")
+        remaining_seconds: Seconds until the limit resets (for Retry-After header)
+        current_count: Current usage count
+        limit_value: The limit that was exceeded
+    """
+
+    def __init__(
+        self,
+        message: str,
+        limit_type: str = "unknown",
+        remaining_seconds: int = 0,
+        current_count: int = 0,
+        limit_value: int = 0
+    ):
+        self.message = message
+        self.limit_type = limit_type
+        self.remaining_seconds = remaining_seconds
+        self.current_count = current_count
+        self.limit_value = limit_value
+        super().__init__(message)
+
+
+class EmailValidationError(Exception):
+    """
+    Raised when email validation fails.
+
+    Attributes:
+        message: Human-readable error message
+        reason: Specific reason for failure (e.g., "disposable", "blocked", "invalid")
+    """
+
+    def __init__(self, message: str, reason: str = "invalid"):
+        self.message = message
+        self.reason = reason
+        super().__init__(message)
+
+
+class IPBlockedError(Exception):
+    """
+    Raised when a request comes from a blocked IP address.
+
+    Attributes:
+        message: Human-readable error message
+        ip_address: The blocked IP address (may be partially masked)
+    """
+
+    def __init__(self, message: str, ip_address: str = ""):
+        self.message = message
+        self.ip_address = ip_address
+        super().__init__(message)