superbrain-server 1.0.2-beta.0

package/payload/docker-compose.yml

@@ -0,0 +1,120 @@
+# ────────────────────────────────────────────────────────────────────────────────
+# SuperBrain Production Docker Compose Configuration
+# ────────────────────────────────────────────────────────────────────────────────
+
+services:
+
+  # ──── Main SuperBrain API Service ────
+  superbrain-api:
+    image: superbrain:latest
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: superbrain-api
+    
+    # ──── Network & Ports ────
+    ports:
+      - "${API_PORT:-5000}:5000"
+    networks:
+      - superbrain-network
+    
+    # ──── Environment Variables ────
+    environment:
+      PYTHONUNBUFFERED: 1
+      PYTHONDONTWRITEBYTECODE: 1
+      ENVIRONMENT: ${ENVIRONMENT:-production}
+      HOST: 0.0.0.0
+      PORT: 5000
+      WORKERS: ${WORKERS:-4}
+      LOG_LEVEL: ${LOG_LEVEL:-INFO}
+      
+      # Database
+      DATABASE_PATH: /app/data/superbrain.db
+      DATABASE_TIMEOUT: 30
+      
+      # API Keys (load from .env file)
+      GROQ_API_KEY: ${GROQ_API_KEY}
+      GEMINI_API_KEY: ${GEMINI_API_KEY}
+      GOOGLE_API_KEY: ${GOOGLE_API_KEY}
+      OPENROUTER_API_KEY: ${OPENROUTER_API_KEY}
+      
+      # Instagram credentials
+      INSTAGRAM_USERNAME: ${INSTAGRAM_USERNAME}
+      INSTAGRAM_PASSWORD: ${INSTAGRAM_PASSWORD}
+      
+      # AI Configuration
+      WHISPER_MODEL: ${WHISPER_MODEL:-base}
+      WHISPER_USE_CLOUD: ${WHISPER_USE_CLOUD:-true}
+      
+      # Performance
+      MAX_UPLOAD_SIZE: ${MAX_UPLOAD_SIZE:-52428800}
+      REQUEST_TIMEOUT: ${REQUEST_TIMEOUT:-60}
+      ANALYSIS_TIMEOUT: ${ANALYSIS_TIMEOUT:-120}
+    
+    # ──── Volumes ────
+    volumes:
+      # Database persistence
+      - superbrain-db:/app/data
+      # Configuration persistence
+      - superbrain-config:/app/config
+      # Temporary files
+      - superbrain-temp:/app/temp
+      # Uploads
+      - superbrain-uploads:/app/static/uploads
+      # Logs
+      - superbrain-logs:/app/logs
+    
+    # ──── Restart Policy ────
+    restart: unless-stopped
+    
+    # ──── Health Check ────
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:5000/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    
+    # ──── Resource Limits (optional, enable for production) ────
+    # uncomment to enable
+    # deploy:
+    #   resources:
+    #     limits:
+    #       cpus: '2'
+    #       memory: 2G
+    #     reservations:
+    #       cpus: '1'
+    #       memory: 1G
+    
+    # ──── Security Options ────
+    security_opt:
+      - no-new-privileges:true
+    read_only: false
+    cap_drop:
+      - ALL
+    cap_add:
+      - NET_BIND_SERVICE
+
+# ────────────────────────────────────────────────────────────────────────────────
+# Networks
+# ────────────────────────────────────────────────────────────────────────────────
+
+networks:
+  superbrain-network:
+    driver: bridge
+
+# ────────────────────────────────────────────────────────────────────────────────
+# Volumes for data persistence
+# ────────────────────────────────────────────────────────────────────────────────
+
+volumes:
+  superbrain-db:
+    driver: local
+  superbrain-config:
+    driver: local
+  superbrain-temp:
+    driver: local
+  superbrain-uploads:
+    driver: local
+  superbrain-logs:
+    driver: local

package/payload/instagram/instagram_downloader.py

@@ -0,0 +1,253 @@
+"""
+Instagram post downloader — instaloader engine.
+
+Uses an authenticated instaloader session if one exists
+(created by instagram_login.py), falls back to anonymous otherwise.
+
+Session file : backend/.instaloader_session  (gitignored)
+"""
+
+import os
+import re
+import sys
+import pathlib
+import contextlib
+
+# Ensure backend root is in sys.path (needed when run as a subprocess)
+import sys as _sys_
+_sys_.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
+del _sys_
+from urllib.request import urlretrieve as _urlretrieve_il
+
+
+class RetryableDownloadError(Exception):
+    """Raised when Instagram download fails due to a transient issue
+    (rate limit / login required) that should be retried later."""
+    pass
+
+# ── instaloader ───────────────────────────────────────────────────────────────
+try:
+    import instaloader
+    INSTALOADER_AVAILABLE = True
+except ImportError:
+    INSTALOADER_AVAILABLE = False
+
+# ── Audio extraction ──────────────────────────────────────────────────────────
+try:
+    from moviepy.editor import VideoFileClip
+    MOVIEPY_AVAILABLE = True
+except ImportError:
+    MOVIEPY_AVAILABLE = False
+
+# ── Paths ─────────────────────────────────────────────────────────────────────
+BACKEND_DIR     = pathlib.Path(__file__).parent.parent
+TEMP_DIR        = BACKEND_DIR / "temp"
+IL_SESSION_FILE = BACKEND_DIR / ".instaloader_session"
+API_KEYS_FILE   = BACKEND_DIR / "config" / ".api_keys"
+LEGACY_API_KEYS_FILE = BACKEND_DIR / ".api_keys"
+
+
+# ── Credential loader ─────────────────────────────────────────────────────────
+def _load_credentials() -> tuple[str, str]:
+    """Read INSTAGRAM_USERNAME / INSTAGRAM_PASSWORD from .api_keys or env."""
+    creds: dict[str, str] = {}
+    key_file = API_KEYS_FILE if API_KEYS_FILE.exists() else LEGACY_API_KEYS_FILE
+    if key_file.exists():
+        for line in key_file.read_text(encoding="utf-8", errors="ignore").splitlines():
+            line = line.strip()
+            if "=" in line and not line.startswith("#"):
+                k, _, v = line.partition("=")
+                creds[k.strip()] = v.strip()
+
+    username = creds.get("INSTAGRAM_USERNAME") or os.getenv("INSTAGRAM_USERNAME", "")
+    password = creds.get("INSTAGRAM_PASSWORD") or os.getenv("INSTAGRAM_PASSWORD", "")
+    return username, password
+
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+def sanitize_folder_name(text: str, max_length: int = 50) -> str:
+    """Strip non-ASCII and filesystem-unsafe characters; trim to max_length."""
+    text = text.encode("ascii", "ignore").decode("ascii")
+    text = re.sub(r'[<>:"/\\|?*\n\r]', "", text)
+    text = re.sub(r"\s+", " ", text).strip()[:max_length]
+    return text or "instagram_post"
+
+
+def _unique_folder(base: pathlib.Path, name: str) -> pathlib.Path:
+    """Return a non-existing path under base, appending _N suffix if needed."""
+    folder = base / name
+    counter = 1
+    while folder.exists():
+        folder = base / f"{name}_{counter}"
+        counter += 1
+    folder.mkdir(parents=True, exist_ok=True)
+    return folder
+
+
+def extract_audio_from_video(video_path: str, audio_path: str) -> bool:
+    """Extract audio track from a video file and save as MP3."""
+    if not MOVIEPY_AVAILABLE:
+        print("  ⚠ moviepy not installed — skipping audio extraction")
+        return False
+    try:
+        print("  Extracting audio...")
+        video = VideoFileClip(video_path)
+        if video.audio is not None:
+            video.audio.write_audiofile(audio_path, verbose=False, logger=None)
+            video.close()
+            print(f"  ✓ Audio saved: {os.path.basename(audio_path)}")
+            return True
+        video.close()
+        print("  ⚠ No audio track in video")
+        return False
+    except Exception as e:
+        print(f"  ⚠ Audio extraction failed: {e}")
+        return False
+
+
+# ── Main download function ────────────────────────────────────────────────────
+def download_instagram_content(url: str) -> str | None:
+    """
+    Download an Instagram post (photo / video / carousel) to temp/<folder>/.
+
+    Uses authenticated instaloader session if available, anonymous otherwise.
+    Returns the folder path string on success, or None on failure.
+    """
+    TEMP_DIR.mkdir(exist_ok=True)
+
+    if INSTALOADER_AVAILABLE:
+        return _download_via_instaloader(url)
+
+    print("✗ instaloader is not installed. Run: pip install instaloader")
+    return None
+
+
+
+# ── instaloader engine ───────────────────────────────────────────────────────
+def _download_via_instaloader(url: str) -> str | None:
+    """Download using instaloader — authenticated session if available, else anonymous."""
+    match = re.search(r"/(?:reels?|p|tv)/([^/?#&]+)", url)
+    if not match:
+        print("  ✗ Invalid Instagram URL.")
+        return None
+    shortcode = match.group(1)
+
+    L = instaloader.Instaloader(
+        download_video_thumbnails=False,
+        download_geotags=False,
+        download_comments=False,
+        save_metadata=False,
+        compress_json=False,
+        iphone_support=False,          # disable mobile API — use web only, no 403s
+        user_agent=(
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/120.0.0.0 Safari/537.36"
+        ),
+        max_connection_attempts=3,
+    )
+
+    # Load saved instaloader session if one exists (set up by instagram_login.py)
+    username, _ = _load_credentials()
+    if not username and not IL_SESSION_FILE.exists():
+        print("  ℹ No Instagram credentials/session found — anonymous mode may fail for some posts.")
+    if IL_SESSION_FILE.exists() and username:
+        try:
+            L.load_session_from_file(username, str(IL_SESSION_FILE))
+            print(f"  ✓ Using authenticated instaloader session (@{username})")
+        except Exception as e:
+            print(f"  ⚠ Could not load instaloader session ({e}) — using anonymous")
+
+    try:
+        print(f"  Fetching post for shortcode: {shortcode}...")
+        with contextlib.redirect_stderr(open(os.devnull, "w")):
+            post = instaloader.Post.from_shortcode(L.context, shortcode)
+
+        caption           = post.caption if post.caption else f"post_{shortcode}"
+        caption_first_line = caption.split("\n")[0]
+        folder_name       = sanitize_folder_name(caption_first_line)
+        folder            = _unique_folder(TEMP_DIR, folder_name)
+
+        print(f"  User   : @{post.owner_username}")
+        print(f"  Caption: {caption_first_line[:80]}...")
+
+        file_counter = 1
+        if post.is_video:
+            video_path = str(folder / f"{folder_name}.mp4")
+            print("  Downloading video...")
+            _urlretrieve_il(post.video_url, video_path)
+            extract_audio_from_video(video_path,
+                                     str(folder / f"{folder_name}_audio.mp3"))
+            _urlretrieve_il(post.url, str(folder / f"{folder_name}_thumbnail.jpg"))
+
+        elif post.typename == "GraphSidecar":
+            print(f"  Downloading carousel ({post.mediacount} items)...")
+            for node in post.get_sidecar_nodes():
+                if node.is_video:
+                    fp = str(folder / f"{folder_name}_{file_counter}.mp4")
+                    _urlretrieve_il(node.video_url, fp)
+                    extract_audio_from_video(
+                        fp,
+                        str(folder / f"{folder_name}_{file_counter}_audio.mp3")
+                    )
+                else:
+                    _urlretrieve_il(
+                        node.display_url,
+                        str(folder / f"{folder_name}_{file_counter}.jpg")
+                    )
+                print(f"  → item {file_counter}/{post.mediacount}")
+                file_counter += 1
+
+        else:
+            print("  Downloading image...")
+            _urlretrieve_il(post.url, str(folder / f"{folder_name}.jpg"))
+
+        # info.txt (instaloader variant — uses post object fields)
+        with open(folder / "info.txt", "w", encoding="utf-8") as f:
+            f.write("Instagram Post Information\n")
+            f.write("=" * 50 + "\n\n")
+            f.write(f"URL: {url}\n")
+            f.write(f"Username: @{post.owner_username}\n")
+            f.write(f"Date: {post.date_utc}\n")
+            f.write(f"Likes: {post.likes}\n")
+            f.write(f"Type: {'Video' if post.is_video else 'Image'}\n")
+            if post.typename == "GraphSidecar":
+                f.write(f"Media Count: {post.mediacount} items\n")
+            f.write("\n")
+            f.write(f"Caption:\n{'-' * 50}\n")
+            f.write(post.caption if post.caption else "No caption")
+            f.write(f"\n{'-' * 50}\n\n")
+            hashtags = re.findall(r"#\w+", post.caption or "")
+            if hashtags:
+                f.write("Hashtags:\n")
+                f.write(", ".join(hashtags) + "\n")
+
+        print(f"\n✓ Download complete (instaloader): {folder}")
+        return str(folder)
+
+    except instaloader.exceptions.LoginRequiredException:
+        raise RetryableDownloadError(
+            "Login required — Instagram blocked anonymous access. Will retry later."
+        )
+    except instaloader.exceptions.ConnectionException as e:
+        _msg = str(e).lower()
+        if any(k in _msg for k in ("too many", "rate", "wait", "429", "blocked", "checkpoint")):
+            raise RetryableDownloadError(f"Instagram rate-limited: {e}")
+        print(f"  ✗ instaloader: Connection error — {e}")
+    except Exception as e:
+        print(f"  ✗ instaloader error: {e}")
+        import traceback; traceback.print_exc()
+    return None
+
+
+# ── CLI entry-point ───────────────────────────────────────────────────────────
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        _url = sys.argv[1]
+    else:
+        _url = input("Enter Instagram Reel / Post / IGTV link: ").strip()
+
+    if _url:
+        download_instagram_content(_url)
+    else:
+        print("No URL provided.")

package/payload/instagram/instagram_login.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+"""
+Instagram Session Setup Script
+===============================
+Run this ONCE to authenticate with Instagram and save an instaloader session.
+After that, instagram_downloader.py will reuse the saved session automatically.
+
+Usage:
+    python instagram_login.py
+"""
+
+import os
+import sys
+import pathlib
+
+# Ensure backend root is in sys.path (needed when run directly)
+import sys as _sys_
+_sys_.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
+del _sys_
+
+# ─── Paths ────────────────────────────────────────────────────────────────────
+BACKEND_DIR     = pathlib.Path(__file__).parent.parent
+API_KEYS_FILE   = BACKEND_DIR / ".api_keys"
+IL_SESSION_FILE = BACKEND_DIR / ".instaloader_session"
+
+
+def _banner(msg: str) -> None:
+    print("\n" + "=" * 60)
+    print(f"  {msg}")
+    print("=" * 60)
+
+
+def _load_credentials() -> tuple[str, str]:
+    creds: dict[str, str] = {}
+    if API_KEYS_FILE.exists():
+        for line in API_KEYS_FILE.read_text(encoding="utf-8", errors="ignore").splitlines():
+            line = line.strip()
+            if "=" in line and not line.startswith("#"):
+                k, _, v = line.partition("=")
+                creds[k.strip()] = v.strip()
+    username = creds.get("INSTAGRAM_USERNAME") or os.getenv("INSTAGRAM_USERNAME", "")
+    password = creds.get("INSTAGRAM_PASSWORD") or os.getenv("INSTAGRAM_PASSWORD", "")
+    return username, password
+
+
+def _save_credentials(username: str, password: str) -> None:
+    lines: list[str] = []
+    if API_KEYS_FILE.exists():
+        for line in API_KEYS_FILE.read_text(encoding="utf-8", errors="ignore").splitlines():
+            stripped = line.strip()
+            if "=" in stripped and not stripped.startswith("#"):
+                k = stripped.split("=", 1)[0].strip()
+                if k not in ("INSTAGRAM_USERNAME", "INSTAGRAM_PASSWORD"):
+                    lines.append(line)
+            else:
+                lines.append(line)
+    lines.append(f"INSTAGRAM_USERNAME={username}")
+    lines.append(f"INSTAGRAM_PASSWORD={password}")
+    API_KEYS_FILE.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    print(f"✓ Credentials saved to {API_KEYS_FILE.name}")
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+#  PART 1 — instaloader session (web API, works on any IP)
+# ══════════════════════════════════════════════════════════════════════════════
+
+def setup_instaloader_session(username: str, password: str) -> bool:
+    """
+    Log in with instaloader (Instagram Web API), save session to a file.
+    Handles 2FA interactively.
+    Returns True on success.
+    """
+    _banner("📥 Setting up instaloader session (web API)")
+
+    try:
+        import instaloader
+    except ImportError:
+        print("✗ instaloader not installed.  Run:  pip install instaloader")
+        return False
+
+    L = instaloader.Instaloader(
+        download_video_thumbnails=False,
+        download_geotags=False,
+        download_comments=False,
+        save_metadata=False,
+        compress_json=False,
+    )
+
+    # Try reusing existing session
+    if IL_SESSION_FILE.exists():
+        try:
+            L.load_session_from_file(username, str(IL_SESSION_FILE))
+            if L.context.is_logged_in:
+                print(f"✓ Reused saved instaloader session for @{username}")
+                return True
+        except Exception:
+            print("  Existing instaloader session invalid — re-logging in.")
+            IL_SESSION_FILE.unlink(missing_ok=True)
+
+    # Fresh login
+    print(f"  Logging in as @{username} ...")
+    try:
+        L.login(username, password)
+    except instaloader.exceptions.TwoFactorAuthRequiredException:
+        print("\n  📱 Two-factor authentication required.")
+        print("     Check your authenticator app or SMS for the 6-digit code.\n")
+        while True:
+            code = input("  Enter 2FA code: ").strip().replace(" ", "")
+            if len(code) == 6 and code.isdigit():
+                break
+            print("  Code should be 6 digits.")
+        try:
+            L.two_factor_login(code)
+        except Exception as e:
+            print(f"  ✗ 2FA failed: {e}")
+            return False
+    except instaloader.exceptions.BadCredentialsException:
+        print("  ✗ Incorrect username or password.")
+        return False
+    except Exception as e:
+        print(f"  ✗ Login error: {e}")
+        return False
+
+    if not L.context.is_logged_in:
+        print("  ✗ Login did not succeed.")
+        return False
+
+    # Save session
+    L.save_session_to_file(str(IL_SESSION_FILE))
+    print(f"✓ instaloader session saved → {IL_SESSION_FILE.name}")
+    return True
+
+
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+#  Main
+# ══════════════════════════════════════════════════════════════════════════════
+
+def main() -> None:
+    _banner("SuperBrain — Instagram Session Setup")
+    print()
+
+    try:
+        import instaloader  # noqa: F401
+    except ImportError:
+        print("✗ instaloader is not installed.  Run:  pip install instaloader")
+        sys.exit(1)
+
+    username, password = _load_credentials()
+
+    if username and password:
+        print(f"  Found credentials for: @{username}")
+        answer = input("  Use these? [Y/n]: ").strip().lower()
+        if answer == "n":
+            username = ""
+            password = ""
+
+    if not username:
+        username = input("\n  Instagram username: ").strip().lstrip("@")
+    if not password:
+        import getpass
+        password = getpass.getpass("  Instagram password: ")
+
+    if not username or not password:
+        print("✗ Username and password are required.")
+        sys.exit(1)
+
+    il_ok = setup_instaloader_session(username, password)
+
+    # Save credentials if entered interactively
+    existing_user, _ = _load_credentials()
+    if not existing_user:
+        _save_credentials(username, password)
+
+    _banner("Summary")
+    il_status = "\u2713 saved" if il_ok else "\u2717 not saved"
+    print(f"  instaloader session : {il_status}")
+    print()
+    if il_ok:
+        print("  SuperBrain will now use the authenticated instaloader session")
+        print("  for all Instagram downloads.")
+        print()
+        print("  Re-run this script if Instagram ever asks you to log in again.")
+    else:
+        print("  ⚠  No session saved. Anonymous instaloader will be used.")
+
+
+if __name__ == "__main__":
+    main()