superbrain-server 1.0.2-beta.1 → 1.0.2-beta.3

package/README.md

@@ -25,6 +25,24 @@ superbrain-server
 npx -y superbrain-server@beta
 ```
 
+### GitHub Packages (auth + install)
+
+Note: This is only needed for GitHub Packages. It is not required for npmjs (`superbrain-server`) installs.
+
+Use a GitHub token with `read:packages`, then run:
+
+```bash
+npm config set @sidinsearch:registry https://npm.pkg.github.com
+npm config set //npm.pkg.github.com/:_authToken YOUR_GITHUB_TOKEN
+npx -y @sidinsearch/superbrain-server@latest
+# beta channel
+npx -y @sidinsearch/superbrain-server@beta
+```
+
+GitHub Packages docs: https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-npm-registry
+
+GitHub Packages is separate from the npmjs package page and may not appear in the repository Packages tab until it has been published and linked there.
+
 ## What It Does on First Run
 
 1. Unpacks backend files into `~/.superbrain-server`

package/package.json

@@ -1,23 +1,25 @@
-{
-  "name": "superbrain-server",
-  "version": "1.0.2-beta.1",
-  "description": "1-Line Auto-Installer and Server Execution wrapper for SuperBrain",
-  "main": "index.js",
-  "bin": {
-    "superbrain-server": "bin/superbrain.js"
-  },
-  "scripts": {
-    "build": "node scripts/build.js"
-  },
-  "keywords": [
-    "superbrain",
-    "server"
-  ],
-  "author": "sidinsearch",
-  "license": "MIT",
-  "dependencies": {},
-  "files": [
-    "bin/",
-    "payload/"
-  ]
-}
+{
+  "name": "superbrain-server",
+  "version": "1.0.2-beta.3",
+  "description": "1-Line Auto-Installer and Server Execution wrapper for SuperBrain",
+  "main": "index.js",
+  "bin": {
+    "superbrain-server": "bin/superbrain.js"
+  },
+  "scripts": {
+    "build": "node scripts/build.js"
+  },
+  "keywords": [
+    "superbrain",
+    "server"
+  ],
+  "repository": "https://github.com/sidinsearch/superbrain.git",
+  "homepage": "https://github.com/sidinsearch/superbrain#readme",
+  "author": "sidinsearch",
+  "license": "MIT",
+  "dependencies": {},
+  "files": [
+    "bin/",
+    "payload/"
+  ]
+}

package/payload/api.py

@@ -72,17 +72,19 @@ def load_or_create_api_token():
 
 API_TOKEN = load_or_create_api_token()
 
-async def verify_token(request: Request, x_api_key: str = Header(..., description="Access Token for authentication")):
+async def verify_token(request: Request, x_api_key: str = Header(None, description="Access Token for authentication")):
     """
     Verify authentication using Access Token.
+    Can be passed in X-API-Key header or token query parameter.
     """
-    if x_api_key != API_TOKEN:
-        logger.warning("Invalid Access Token attempt from IP: %s", request.client.host if hasattr(request, 'client') else 'unknown')
+    actual_token = x_api_key or request.query_params.get("token")
+    if actual_token != API_TOKEN:
+        logger.warning("Invalid Access Token attempt from IP: %s", request.client.host if hasattr(request, 'client') and request.client else 'unknown')
         raise HTTPException(
             status_code=401,
             detail="Invalid Access Token. Use the token from backend/token.txt."
         )
-    return x_api_key
+    return actual_token
 
 # Initialize FastAPI app
 app = FastAPI(

package/payload/config/model_rankings.json

@@ -1,11 +1,11 @@
 {
   "groq_gpt_oss_20b": {
     "key": "groq_gpt_oss_20b",
-    "avg_response_s": 1.3587389144652606,
-    "success_count": 48,
+    "avg_response_s": 1.325513668372554,
+    "success_count": 50,
     "fail_count": 1,
     "down_until": null,
-    "last_used": "2026-02-28T15:46:20.843218",
+    "last_used": "2026-04-06T17:49:12.415601",
     "last_error": null,
     "base_priority": 0.5
   },
@@ -323,8 +323,8 @@
     "key": "gemini_25_flash_lite_vision",
     "avg_response_s": 6.709401964075168,
     "success_count": 14,
-    "fail_count": 9,
-    "down_until": "2026-04-01T11:15:39.217625",
+    "fail_count": 10,
+    "down_until": "2026-04-06T17:54:03.900203",
     "last_used": "2026-02-24T09:45:09.831171",
     "last_error": "No module named 'google.generativeai'",
     "base_priority": 1.5
@@ -333,8 +333,8 @@
     "key": "gemini_25_pro_vision",
     "avg_response_s": null,
     "success_count": 0,
-    "fail_count": 12,
-    "down_until": "2026-04-01T11:15:39.214310",
+    "fail_count": 13,
+    "down_until": "2026-04-06T17:54:03.898186",
     "last_used": null,
     "last_error": "No module named 'google.generativeai'",
     "base_priority": 2
@@ -353,8 +353,8 @@
     "key": "gemini_3_pro_vision",
     "avg_response_s": null,
     "success_count": 0,
-    "fail_count": 11,
-    "down_until": "2026-04-01T11:15:39.217625",
+    "fail_count": 12,
+    "down_until": "2026-04-06T17:54:03.899185",
     "last_used": null,
     "last_error": "No module named 'google.generativeai'",
     "base_priority": 3
@@ -363,8 +363,8 @@
     "key": "gemini_31_pro_vision",
     "avg_response_s": null,
     "success_count": 0,
-    "fail_count": 10,
-    "down_until": "2026-04-01T11:15:39.217625",
+    "fail_count": 11,
+    "down_until": "2026-04-06T17:54:03.901209",
     "last_used": null,
     "last_error": "No module named 'google.generativeai'",
     "base_priority": 3.5
@@ -373,8 +373,8 @@
     "key": "gemini_20_flash_vision",
     "avg_response_s": null,
     "success_count": 0,
-    "fail_count": 9,
-    "down_until": "2026-04-01T11:15:39.220783",
+    "fail_count": 10,
+    "down_until": "2026-04-06T17:54:03.901209",
     "last_used": null,
     "last_error": "No module named 'google.generativeai'",
     "base_priority": 4
@@ -383,8 +383,8 @@
     "key": "gemini_20_flash_lite_vision",
     "avg_response_s": null,
     "success_count": 0,
-    "fail_count": 9,
-    "down_until": "2026-04-01T11:15:39.220783",
+    "fail_count": 10,
+    "down_until": "2026-04-06T17:54:03.902186",
     "last_used": null,
     "last_error": "No module named 'google.generativeai'",
     "base_priority": 4.5
@@ -393,19 +393,19 @@
     "key": "gemini_15_flash_vision",
     "avg_response_s": null,
     "success_count": 0,
-    "fail_count": 9,
-    "down_until": "2026-04-01T11:15:39.220783",
+    "fail_count": 10,
+    "down_until": "2026-04-06T17:54:03.903185",
     "last_used": null,
     "last_error": "No module named 'google.generativeai'",
     "base_priority": 4.8
   },
   "groq_llama4_scout_vision": {
     "key": "groq_llama4_scout_vision",
-    "avg_response_s": 2.1663871744251377,
-    "success_count": 28,
+    "avg_response_s": 1.9110158948201241,
+    "success_count": 30,
     "fail_count": 0,
     "down_until": null,
-    "last_used": "2026-02-28T15:46:16.547477",
+    "last_used": "2026-04-06T17:49:07.508119",
     "last_error": null,
     "base_priority": 5
   },

package/payload/core/database.py

@@ -315,7 +315,7 @@ class Database:
         try:
             cur = self._conn.cursor()
 
-            cur.execute("SELECT COUNT(*) FROM analyses")
+            cur.execute("SELECT COUNT(*) FROM analyses WHERE (is_hidden IS NULL OR is_hidden = 0)")
             total = cur.fetchone()[0]
 
             cur.execute("SELECT COUNT(*) FROM collections")
@@ -323,7 +323,7 @@ class Database:
 
             cur.execute(
                 "SELECT COALESCE(category,'Uncategorized') as cat, COUNT(*) as cnt "
-                "FROM analyses GROUP BY cat"
+                "FROM analyses WHERE (is_hidden IS NULL OR is_hidden = 0) GROUP BY cat"
             )
             category_counts = {r["cat"]: r["cnt"] for r in cur.fetchall()}
 

package/payload/start.py

@@ -118,17 +118,57 @@ def run_q(cmd, **kwargs):
     return subprocess.run(cmd, check=True, capture_output=True, text=True, **kwargs)
 
 
+def _load_saved_api_keys() -> dict[str, str]:
+    """Load saved API keys and credentials from config/.api_keys."""
+    keys: dict[str, str] = {}
+    if API_KEYS.exists():
+        for line in API_KEYS.read_text(encoding="utf-8", errors="ignore").splitlines():
+            line = line.strip()
+            if "=" in line and not line.startswith("#"):
+                k, _, v = line.partition("=")
+                keys[k.strip()] = v.strip()
+    return keys
+
+
+CORE_PACKAGES = [
+    "fastapi>=0.111.0",
+    "uvicorn[standard]>=0.29.0",
+    "pydantic>=2.0.0",
+    "python-multipart>=0.0.9",
+    "requests>=2.31.0",
+    "httpx>=0.27.0",
+    "groq>=0.9.0",
+    "google-genai>=0.8.0",
+    "beautifulsoup4>=4.12.0",
+    "trafilatura>=1.12.0",
+    "newspaper4k>=0.9.0",
+    "lxml>=5.0.0",
+    "lxml_html_clean>=0.1.0",
+    "htmldate>=1.9.0",
+    "instaloader>=4.11.0",
+    "rich>=13.0.0",
+    "segno>=1.6.0",
+]
+
+
 def ensure_runtime_dependencies():
     """Install must-have runtime packages if missing in the active venv."""
     required = [
+        ("fastapi", "fastapi"),
+        ("uvicorn", "uvicorn"),
         ("multipart", "python-multipart"),
+        ("instaloader", "instaloader"),
+        ("segno", "segno"),
     ]
     missing: list[str] = []
 
     for module_name, package_name in required:
-        try:
-            importlib.import_module(module_name)
-        except Exception:
+        # Check if the module is actually importable in the virtual environment
+        rc = subprocess.run(
+            [str(VENV_PYTHON), "-c", f"import {module_name}"],
+            capture_output=True
+        ).returncode
+        if rc != 0:
             missing.append(package_name)
 
     if not missing:
@@ -179,79 +219,16 @@ def setup_venv():
 # Step 2 — Install Dependencies
 # ══════════════════════════════════════════════════════════════════════════════
 def install_deps():
-    h1("Step 2 of 7 — Installing Python Dependencies")
-    req = BASE_DIR / "requirements.txt"
-    if not req.exists():
-        err("requirements.txt not found — cannot install dependencies.")
-        sys.exit(1)
+    h1("Step 2 of 7 — Installing Core Python Dependencies")
 
     h2("Upgrading pip …")
     run([str(VENV_PYTHON), "-m", "pip", "install", "--quiet", "--upgrade", "pip"])
     ok("pip up to date")
 
-    h2("Installing packages from requirements.txt …")
-    nl()
-
-    # ── stream pip output and display each package live ────────────────────────
-    cmd = [str(VENV_PIP), "install", "--progress-bar", "off", "-r", str(req)]
-    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-                            text=True, bufsize=1)
-
-    collecting: list[str] = []
-    n_cached   = 0
-    n_download = 0
-    n_install  = 0
-    current_pkg = ""
-
-    for raw in proc.stdout:  # type: ignore[union-attr]
-        line = raw.rstrip()
-        if not line:
-            continue
-
-        if line.startswith("Collecting "):
-            pkg = line.split()[1]
-            current_pkg = pkg
-            collecting.append(pkg)
-            idx = len(collecting)
-            print(f"  {CYAN}↓{RESET}  [{idx:>3}] {BOLD}{pkg}{RESET}")
-
-        elif "Downloading" in line and ".whl" in line or ".tar.gz" in line:
-            # e.g. "  Downloading fastapi-0.111.0-py3..whl (92 kB)"
-            parts = line.strip().split()
-            if len(parts) >= 2:
-                filename = parts[1]
-                size_str = " ".join(parts[2:]).strip("()")
-                print(f"       {DIM}↓ {filename}  {size_str}{RESET}")
-            n_download += 1
-
-        elif line.strip().startswith("Requirement already satisfied"):
-            n_cached += 1
-
-        elif line.startswith("Installing collected packages:"):
-            pkgs = line.split(":", 1)[1].strip()
-            n_install = len(pkgs.split(","))
-            nl()
-            print(f"  {BLUE}{BOLD}  ▶  Linking {n_install} package(s) into virtual environment …{RESET}")
-
-        elif line.startswith("Successfully installed"):
-            tail = line.replace("Successfully installed", "").strip()
-            count = len(tail.split())
-            nl()
-            ok(f"{count} package(s) installed successfully")
-            if n_cached:
-                info(f"{n_cached} package(s) already satisfied (cached)")
-
-        elif line.upper().startswith("WARNING") or line.upper().startswith("DEPRECATION"):
-            pass   # suppress pip noise
-
-        else:
-            # Any other line (build output, etc.) show dimmed
-            if line.strip():
-                print(f"       {DIM}{line.strip()}{RESET}")
-
-    proc.wait()
-    if proc.returncode != 0:
-        raise subprocess.CalledProcessError(proc.returncode, cmd)
+    h2("Installing core runtime packages …")
+    info("Optional packages such as local Whisper, OpenCV, and music ID are installed only when enabled.")
+    run([str(VENV_PIP), "install", "--progress-bar", "off", *CORE_PACKAGES])
+    ok(f"Installed {len(CORE_PACKAGES)} core package(s)")
 
 # ══════════════════════════════════════════════════════════════════════════════
 # ── API key validators ────────────────────────────────────────────────────────
@@ -431,6 +408,9 @@ OLLAMA_MODEL = "qwen3-vl:4b"   # vision-language model, fits ~6 GB VRAM / ~8 GB
 def setup_ollama():
     h1("Step 4 of 7 — Offline AI Model (Ollama)")
 
+    keys = _load_saved_api_keys()
+    has_cloud_key = any(keys.get(k) for k in ("GEMINI_API_KEY", "GROQ_API_KEY", "OPENROUTER_API_KEY"))
+
     print(f"""
   Ollama runs AI models {BOLD}locally on your machine{RESET} — no internet or API
   key required. SuperBrain uses it as a last-resort fallback if all
@@ -441,7 +421,10 @@ def setup_ollama():
   Other options: llama3.2:3b (2 GB / 4 GB RAM), gemma2:2b (1.5 GB / 4 GB RAM)
 """)
 
-    if not ask_yn("Set up Ollama offline model?", default=True):
+    if has_cloud_key:
+        info("Cloud API key(s) detected — Ollama is optional and skipped by default.")
+
+    if not ask_yn("Set up Ollama offline model?", default=not has_cloud_key):
         warn("Skipping Ollama. Cloud providers only — make sure you have API keys.")
         return
 
@@ -562,6 +545,9 @@ WHISPER_MODELS = {
 def setup_whisper():
     h1("Step 5 of 7 — Offline Audio Transcription (Whisper)")
 
+    keys = _load_saved_api_keys()
+    has_groq_key = bool(keys.get("GROQ_API_KEY") or os.getenv("GROQ_API_KEY"))
+
     print(f"""
   OpenAI Whisper transcribes audio and video {BOLD}entirely on your machine{RESET}.
   SuperBrain uses it to extract speech from Instagram Reels, YouTube
@@ -571,6 +557,16 @@ def setup_whisper():
   It also pre-downloads a speech model the first time it runs.
 """)
 
+    if has_groq_key:
+        info("Groq API key detected — cloud Whisper is available, so local Whisper is optional.")
+        if not ask_yn("Also install local Whisper fallback?", default=False):
+            warn("Skipping local Whisper. Groq Whisper will be used when Groq is available.")
+            return
+    else:
+        if not ask_yn("Set up local Whisper offline transcription?", default=True):
+            warn("Skipping Whisper setup. Audio transcription will rely on cloud providers if available.")
+            return
+
     # ── ffmpeg check ──────────────────────────────────────────────────────────
     if shutil.which("ffmpeg"):
         ok("ffmpeg is installed")
@@ -597,7 +593,7 @@ def setup_whisper():
         warn("openai-whisper not found — installing now …")
         nl()
         try:
-            cmd = [str(VENV_PIP), "install", "--progress-bar", "off", "openai-whisper"]
+            cmd = [str(VENV_PIP), "install", "--progress-bar", "off", "openai-whisper>=20231117"]
             proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                                     text=True, bufsize=1)
             for raw in proc.stdout:  # type: ignore[union-attr]
@@ -998,80 +994,108 @@ def _detect_local_ip() -> str:
     return "127.0.0.1"
 
 def _display_connect_qr(url: str, token: str):
-    """Display a proper QR code in the terminal using segno + Unicode half-block chars.
+    """Display a QR code in the terminal without depending on the active interpreter."""
+
+    payload = json.dumps({"url": url, "token": token}, separators=(',', ':'))
+    qr_script = textwrap.dedent(r'''
+        import json
+        import sys
 
-    The QR encodes a JSON string:  {"url": "...", "token": "..."}
-    which the mobile app's QR scanner can read to auto-configure connection.
-    """
-    try:
         import segno
-    except ImportError:
-        # segno not installed — try installing it on the fly
-        try:
-            info("Installing segno for QR code display …")
-            run_q([str(VENV_PIP), "install", "--quiet", "segno"])
-            import segno
-        except Exception:
-            warn("Could not generate QR code (segno not available).")
-            info("Install it:  pip install segno")
-            return
 
-    payload = json.dumps({"url": url, "token": token}, separators=(',', ':'))
-    qr = segno.make(payload, error='L')
-
-    # Convert to a matrix of booleans (True = dark module)
-    matrix = [list(row) for row in qr.matrix]
-    rows = len(matrix)
-    cols = len(matrix[0]) if rows else 0
-
-    # Add quiet zone (2 modules on each side)
-    quiet = 2
-    padded_cols = cols + quiet * 2
-    padded_rows = rows + quiet * 2
-    padded = []
-    empty_row = [0] * padded_cols
-    for _ in range(quiet):
-        padded.append(list(empty_row))
-    for row in matrix:
-        padded.append([0] * quiet + row + [0] * quiet)
-    for _ in range(quiet):
-        padded.append(list(empty_row))
-
-    # Render using Unicode half-block characters for double vertical resolution
-    # Each output line encodes TWO rows of QR modules:
-    #   top=dark, bottom=dark → "█"  (full block)
-    #   top=dark, bottom=light → "▀" (upper half)
-    #   top=light, bottom=dark → "▄" (lower half)
-    #   top=light, bottom=light → " " (space)
-    BG_WHITE  = "\033[47m"   # white background
-    FG_BLACK  = "\033[30m"   # black foreground
-    ANSI_RST  = "\033[0m"
+        payload = sys.argv[1]
+        qr = segno.make(payload, error='L')
+
+        matrix = [list(row) for row in qr.matrix]
+        rows = len(matrix)
+        cols = len(matrix[0]) if rows else 0
+
+        quiet = 2
+        padded_cols = cols + quiet * 2
+        padded_rows = rows + quiet * 2
+        padded = []
+        empty_row = [0] * padded_cols
+        for _ in range(quiet):
+            padded.append(list(empty_row))
+        for row in matrix:
+            padded.append([0] * quiet + row + [0] * quiet)
+        for _ in range(quiet):
+            padded.append(list(empty_row))
+
+        print(f"  ┌{'─' * (padded_cols + 4)}┐")
+        
+        # Center the title within the inner frame
+        inner_width = padded_cols + 4
+        title = "Scan with SuperBrain App"
+        # If the width is too small, truncate it just in case, though QR is usually wider
+        if len(title) > inner_width - 2:
+            title = title[:inner_width - 2]
+        
+        print(f"  │{title.center(inner_width)}│")
+        print(f"  ├{'─' * inner_width}┤")
+
+        for y in range(0, padded_rows, 2):
+            line_chars = []
+            for x in range(padded_cols):
+                top = padded[y][x] if y < padded_rows else 0
+                bottom = padded[y + 1][x] if y + 1 < padded_rows else 0
+
+                if top and bottom:
+                    line_chars.append("█")
+                elif top and not bottom:
+                    line_chars.append("▀")
+                elif not top and bottom:
+                    line_chars.append("▄")
+                else:
+                    line_chars.append(" ")
 
-    nl()
-    print(f"  {BOLD}{CYAN}┌{'─' * (padded_cols + 4)}┐{RESET}")
-    print(f"  {BOLD}{CYAN}│{RESET}  {BOLD}Scan with SuperBrain app{RESET}  {BOLD}{CYAN}│{RESET}")
-    print(f"  {BOLD}{CYAN}├{'─' * (padded_cols + 4)}┤{RESET}")
-
-    for y in range(0, padded_rows, 2):
-        line_chars = []
-        for x in range(padded_cols):
-            top = padded[y][x] if y < padded_rows else 0
-            bottom = padded[y + 1][x] if y + 1 < padded_rows else 0
-
-            if top and bottom:
-                line_chars.append("█")
-            elif top and not bottom:
-                line_chars.append("▀")
-            elif not top and bottom:
-                line_chars.append("▄")
-            else:
-                line_chars.append(" ")
+            print(f"  │  {''.join(line_chars)}  │")
 
-        line = "".join(line_chars)
-        print(f"  {BOLD}{CYAN}│{RESET}  {line}  {BOLD}{CYAN}│{RESET}")
+        print(f"  └{'─' * (padded_cols + 4)}┘")
+    ''')
 
-    print(f"  {BOLD}{CYAN}└{'─' * (padded_cols + 4)}┘{RESET}")
-    nl()
+    interpreters = []
+    if VENV_PYTHON.exists():
+        interpreters.append(str(VENV_PYTHON))
+    if sys.executable not in interpreters:
+        interpreters.append(sys.executable)
+
+    import os
+    env = os.environ.copy()
+    env["PYTHONIOENCODING"] = "utf-8"
+    
+    for python_executable in interpreters:
+        try:
+            result = subprocess.run(
+                [python_executable, "-c", qr_script, payload],
+                check=True,
+                capture_output=True,
+                text=True,
+                encoding="utf-8",
+                errors="replace",
+                env=env,
+            )
+            nl()
+            print(result.stdout, end="")
+            nl()
+            return
+        except subprocess.CalledProcessError as e:
+            try:
+                run([python_executable, "-m", "pip", "install", "--quiet", "segno"])
+                result = run_q([python_executable, "-c", qr_script, payload], encoding="utf-8", errors="replace", env=env)
+                nl()
+                print(result.stdout, end="")
+                nl()
+                return
+            except subprocess.CalledProcessError as e2:
+                warn(f"Failed to generate QR code using {python_executable}. STDOUT: {e2.stdout} STDERR: {e2.stderr}")
+                continue
+            except Exception as ex:
+                warn(f"Failed to generate QR code using {python_executable}: {ex}")
+                continue
+
+    warn("Could not generate QR code.")
+    info("Use the server URL and Access Token shown below to connect manually.")
 
 
 def launch_backend():
@@ -1197,7 +1221,7 @@ def launch_backend():
         tunnel_hint = f"         · public     →  install Node.js first, then run: {DIM}npx localtunnel --port {PORT}{RESET}"
 
     # ── Generate and display QR code ──────────────────────────────────────────
-    qr_url = f"http://{local_ip}:{PORT}"
+    qr_url = localtunnel_url if localtunnel_url else f"http://{local_ip}:{PORT}"
     _display_connect_qr(qr_url, token)
 
     print(f"""