openvidia 2.2.0__py3-none-any.whl

openvidia/safe_file.py

@@ -0,0 +1,100 @@
+"""Safe file operations with automatic backups."""
+
+from __future__ import annotations
+
+import os
+import shutil
+from datetime import datetime
+from pathlib import Path
+
+
+def create_backup(
+    file_path: Path,
+    backup_dir: Path | None = None,
+    max_backups: int = 5,
+) -> Path | None:
+    """Create a timestamped backup of a file.
+
+    Args:
+        file_path: Path to the file to backup
+        backup_dir: Directory for backups (default: same dir as file)
+        max_backups: Maximum number of backups to keep
+
+    Returns:
+        Path to the backup file, or None if backup failed
+    """
+    if not file_path.exists():
+        return None
+
+    backup_dir = backup_dir or file_path.parent
+    backup_dir.mkdir(parents=True, exist_ok=True)
+
+    # Microseconds, not seconds. At second granularity two backups taken in the
+    # same second resolved to the same filename, so the second silently
+    # overwrote the first — the rotation kept one file where it claimed five,
+    # and the test that was supposed to catch it passed for the same reason.
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
+    backup_name = f"{file_path.stem}_backup_{timestamp}{file_path.suffix}"
+    backup_path = backup_dir / backup_name
+
+    try:
+        shutil.copy2(file_path, backup_path)
+        # copy2 preserves the source mode, which is right for a key file — but
+        # a backup taken from a file an older version wrote 0644 would inherit
+        # 0644. Pin it: a backup of a secret is still a secret.
+        if os.name != "nt":
+            os.chmod(backup_path, 0o600)
+
+        # Cleanup old backups
+        cleanup_old_backups(file_path, backup_dir, max_backups)
+
+        return backup_path
+    except OSError as e:
+        print(f"Warning: Could not create backup: {e}")
+        return None
+
+
+def cleanup_old_backups(
+    original_file: Path,
+    backup_dir: Path | None = None,
+    max_backups: int = 5,
+) -> int:
+    """Remove old backups, keeping only the most recent ones.
+
+    Returns:
+        Number of backups removed
+    """
+    backup_dir = backup_dir or original_file.parent
+
+    # Find all backups for this file
+    prefix = f"{original_file.stem}_backup_"
+    suffix = original_file.suffix
+
+    backups = []
+    for f in backup_dir.iterdir():
+        if f.is_file() and f.name.startswith(prefix) and f.name.endswith(suffix):
+            backups.append(f)
+
+    # Sort by modification time (newest first)
+    backups.sort(key=lambda p: p.stat().st_mtime, reverse=True)
+
+    # Remove old backups
+    removed = 0
+    for old_backup in backups[max_backups:]:
+        try:
+            old_backup.unlink()
+            removed += 1
+        except OSError:
+            pass
+
+    return removed
+
+
+# Removed with this commit: safe_write_with_backup, restore_from_backup and
+# list_backups. None had a production caller — the test suite was their only
+# consumer, which made a dead API look maintained. safe_write_with_backup also
+# duplicated config.atomic_write with a *different* temp-file convention
+# (keys.json.tmp vs keys.tmp) and without the 0600 mode or the fsync, so the
+# two would have raced each other had anything used both. restore_from_backup
+# built its path from an unvalidated timestamp, so a caller that ever exposed
+# it over HTTP would have handed out arbitrary file overwrite.

openvidia/server_manager.py

@@ -0,0 +1,106 @@
+"""
+Owns the actual uvicorn server instance(s) for one proxy run.
+
+Binds loopback on BOTH IP stacks so a client reaching us as "localhost"
+works whether its resolver picks IPv4 (127.0.0.1) or IPv6 (::1) — the
+latter is what Node/Bun try first. Loopback-only, never all-interfaces:
+the proxy serves unauthenticated API keys. IPv6 loopback is best-effort —
+don't fail proxy startup on hosts with IPv6 off.
+"""
+
+import asyncio
+import socket as socket_mod
+from collections.abc import Callable
+from pathlib import Path
+
+import uvicorn
+
+from .proxy_app import create_app
+from .proxy_state import ProxyState, ProxyStats
+
+
+class ProxyServer:
+    def __init__(
+        self,
+        servers: list[uvicorn.Server],
+        tasks: list[asyncio.Task],
+        state: ProxyState,
+    ):
+        self._servers = servers
+        self._tasks = tasks
+        self.state = state
+
+    async def shutdown(self):
+        for s in self._servers:
+            s.should_exit = True
+        for t in self._tasks:
+            try:
+                await asyncio.wait_for(t, timeout=2.0)
+            except (TimeoutError, asyncio.CancelledError):
+                t.cancel()
+
+
+def _bind(family: int, host: str, port: int) -> socket_mod.socket | None:
+    try:
+        sock = socket_mod.socket(family, socket_mod.SOCK_STREAM)
+    except OSError:
+        return None
+    sock.setsockopt(socket_mod.SOL_SOCKET, socket_mod.SO_REUSEADDR, 1)
+    try:
+        sock.bind((host, port))
+        sock.listen(128)
+        sock.setblocking(False)
+        return sock
+    except OSError:
+        sock.close()
+        return None
+
+
+async def start(
+    port: int,
+    keys: list[str],
+    log_cb: Callable[[str], None],
+    stats: ProxyStats,
+    web_dir: Path | None = None,
+    initial_model: str = "",
+) -> ProxyServer:
+    state = ProxyState(keys=keys, stats=stats, log_cb=log_cb, port=port)
+    if initial_model:
+        state.active_model = initial_model
+    app = create_app(state, web_dir=web_dir)
+
+    # Bind explicitly first (like Rust's TcpListener::bind) so a host with
+    # IPv6 off just skips that stack instead of taking the whole proxy down.
+    v4_sock = _bind(socket_mod.AF_INET, "127.0.0.1", port)
+    if v4_sock is None:
+        raise RuntimeError(f"proxy failed to bind :{port} (IPv4 loopback)")
+
+    v6_sock = _bind(socket_mod.AF_INET6, "::1", port)
+
+    servers: list[uvicorn.Server] = []
+    tasks: list[asyncio.Task] = []
+    socks = [v4_sock]
+    if v6_sock is not None:
+        socks.append(v6_sock)
+
+    v4_config = uvicorn.Config(app, log_level="warning", lifespan="on")
+    v4_server = uvicorn.Server(v4_config)
+    servers.append(v4_server)
+    tasks.append(asyncio.create_task(v4_server.serve(sockets=[v4_sock])))
+
+    if v6_sock is not None:
+        v6_config = uvicorn.Config(app, log_level="warning", lifespan="off")
+        v6_server = uvicorn.Server(v6_config)
+        servers.append(v6_server)
+        tasks.append(asyncio.create_task(v6_server.serve(sockets=[v6_sock])))
+
+    for _ in range(100):
+        if v4_server.started:
+            break
+        await asyncio.sleep(0.02)
+    else:
+        for s in socks:
+            s.close()
+        raise RuntimeError(f"proxy failed to start on :{port}")
+
+    return ProxyServer(servers, tasks, state)

openvidia/upstream_router.py

@@ -0,0 +1,50 @@
+"""Round-robin router over multiple NVIDIA-compatible upstream endpoints.
+
+A single integrate.api.nvidia.com can 429 the whole account: keeping a
+secondary endpoint (same key family, different host) and blacklisting a host
+for a short window after a 5xx/timeout spreads load and survives partial
+outages.
+"""
+
+from __future__ import annotations
+
+import time
+
+ENDPOINT_RETRY_AFTER = 60.0
+
+
+class EndpointRouter:
+    """Tracks per-endpoint health and hands out the healthy ones in order."""
+
+    def __init__(self, endpoints: list[str]) -> None:
+        self.endpoints = [e.rstrip("/") + "/" for e in endpoints]
+        self._blacklisted_until: dict[str, float] = {}
+
+    def healthy_endpoints(self) -> list[str]:
+        now = time.monotonic()
+        return [e for e in self.endpoints if self._blacklisted_until.get(e, 0.0) <= now]
+
+    def mark_failure(self, url: str) -> None:
+        for endpoint in self.endpoints:
+            if url.startswith(endpoint):
+                self._blacklisted_until[endpoint] = time.monotonic() + ENDPOINT_RETRY_AFTER
+                return
+
+    def mark_success(self, url: str) -> None:
+        for endpoint in self.endpoints:
+            if url.startswith(endpoint):
+                self._blacklisted_until.pop(endpoint, None)
+                return
+
+    def as_dict(self) -> dict[str, object]:
+        now = time.monotonic()
+        return {
+            "endpoints": [
+                {
+                    "url": e,
+                    "healthy": self._blacklisted_until.get(e, 0.0) <= now,
+                    "retry_in_s": max(0.0, self._blacklisted_until.get(e, 0.0) - now),
+                }
+                for e in self.endpoints
+            ]
+        }

openvidia/web/index.html

@@ -0,0 +1,166 @@
+<!doctype html>
+<html lang="en" data-theme="dark">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
+    <meta name="description" content="OpenVidia — local multi-key proxy control panel for NVIDIA-hosted models." />
+    <title>OpenVidia</title>
+    <link rel="icon" type="image/x-icon" href="favicon.ico">
+    <link rel="stylesheet" href="styles.css" />
+  </head>
+  <body>
+    <div id="bg-canvas"></div>
+    <div id="toast-container"></div>
+
+    <div id="app">
+      <header>
+        <div class="logo">
+          <img src="logo.png" alt="OpenVidia" class="logo-img">
+          <span class="logo-text">OpenVidia</span>
+        </div>
+        <div class="header-right">
+          <span class="header-stat upstream-badge" id="hUpstream" title="NVIDIA upstream health — probed every 30s">
+            <span class="up-dot" id="upDot"></span> NVIDIA <span id="upText">…</span>
+          </span>
+          <span class="header-stat" id="hKeyCount">0 keys</span>
+          <span class="header-stat" id="hRpm">0 RPM</span>
+          <button class="theme-btn" id="themeToggle" title="Toggle theme">🌙</button>
+        </div>
+      </header>
+
+      <div class="bento">
+        <!-- ── STATUS (cell-status) ── -->
+        <div class="bento-cell cell-status">
+          <span class="indicator-dot" id="statusDot"><span class="pulse-dot running"></span></span>
+          <div class="status-info">
+            <span class="status-label" id="statusText">Running</span>
+            <span class="status-meta">Port <strong id="portDisplay">1919</strong> · <strong id="activeModelDisplay">none</strong></span>
+            <!-- Reasoning toggle: a hybrid model emits nothing until it stops
+                 thinking, which is the difference between a 2s and a 160s
+                 first token. Applies to the active model. -->
+            <span class="status-meta think-row">
+              Thinking
+              <button class="btn btn-xs think-btn" data-think="auto" title="Leave it to the model">auto</button>
+              <button class="btn btn-xs think-btn" data-think="on" title="Force reasoning on">on</button>
+              <button class="btn btn-xs think-btn" data-think="off" title="Force reasoning off — much faster first token">off</button>
+            </span>
+            <!-- Reasoning effort: finer granularity than on/off.
+                 low = fast/zero thinking, medium = balanced, high = deep/focused -->
+            <span class="status-meta effort-row">
+              Effort
+              <button class="btn btn-xs effort-btn" data-effort="auto" title="Defer to thinking toggle">auto</button>
+              <button class="btn btn-xs effort-btn" data-effort="low" title="Fast — no reasoning, temp 0.7">low</button>
+              <button class="btn btn-xs effort-btn" data-effort="medium" title="Balanced — thinking on, temp 0.5">med</button>
+              <button class="btn btn-xs effort-btn" data-effort="high" title="Deep — thinking on, temp 0.2">high</button>
+            </span>
+            <span class="status-meta fallback-row">
+              Fallback
+              <button class="btn btn-xs fallback-btn" data-fallback="off" title="Never failover — return 503 if model is down">off</button>
+              <button class="btn btn-xs fallback-btn" data-fallback="on" title="Always failover to next healthy preset">on</button>
+              <button class="btn btn-xs fallback-btn" data-fallback="auto" title="Failover only when circuit is open">auto</button>
+            </span>
+          </div>
+          <div class="status-acts">
+            <button class="btn btn-sm" id="stopBtn" title="Stop">⏹</button>
+            <button class="btn btn-sm" id="startBtn" title="Start" style="display:none">▶</button>
+            <button class="btn btn-sm" id="restartBtn" title="Restart">⟳</button>
+          </div>
+        </div>
+
+        <!-- ── STATS (cell-stats, col-span-3) ── -->
+        <div class="bento-cell cell-stats col-span-3">
+          <div class="stat-mini"><span class="stat-num" id="statReqs">0</span><span class="stat-label">req</span></div>
+          <div class="stat-mini"><span class="stat-num" id="statOk">0</span><span class="stat-label">ok</span></div>
+          <div class="stat-mini"><span class="stat-num" id="statRots">0</span><span class="stat-label">rot</span></div>
+          <div class="stat-mini" id="cdMini"><span class="stat-num" id="statCooldowns">0</span><span class="stat-label">⏳</span></div>
+        </div>
+
+        <!-- ── MODELS (single list; ★ Starred = quick-pick shortlist) ── -->
+        <div class="bento-cell cell-models col-span-2" data-dropdown>
+          <div class="bento-hd" data-toggle data-idx="01">
+            <span class="bento-ttl">Models</span>
+            <span class="bento-badge" id="modelCount">0</span>
+            <span class="chevron">▸</span>
+          </div>
+          <div class="bento-body" data-body>
+            <div class="bento-body-inner">
+              <div class="browser-filters" id="modelFilters"></div>
+              <input type="text" class="browser-input" id="modelSearch" placeholder="Search models…" spellcheck="false" />
+              <div class="browser-list" id="modelList"><div class="browser-empty">Loading…</div></div>
+            </div>
+          </div>
+        </div>
+
+        <!-- ── KEY MANAGEMENT (right after status/stats, full width, open by default) ── -->
+        <div class="bento-cell cell-keys col-span-4 open" data-dropdown>
+          <div class="bento-hd" data-toggle data-idx="02">
+            <span class="bento-ttl">Keys</span>
+            <span class="bento-badge" id="keyCount">0</span>
+            <span class="chevron">▸</span>
+          </div>
+          <div class="bento-body" data-body>
+            <div class="bento-body-inner">
+              <div class="keys-toolbar">
+                <div class="key-filters" id="keyFilters">
+                  <button class="key-filter" data-filter="all">All <span class="filter-count" id="fAll">0</span></button>
+                  <button class="key-filter active" data-filter="active">Active <span class="filter-count" id="fActive">0</span></button>
+                  <button class="key-filter" data-filter="cooldown">⏳ <span class="filter-count" id="fCooldown">0</span></button>
+                  <button class="key-filter" data-filter="fresh">● <span class="filter-count" id="fFresh">0</span></button>
+                  <button class="key-filter" data-filter="idle">Idle <span class="filter-count" id="fIdle">0</span></button>
+                </div>
+                <button class="btn btn-sm btn-primary" id="addKeyBtn">+ Add Key</button>
+              </div>
+              <div class="slide-form hidden" id="keyAddForm">
+                <input type="text" id="newKeyInput" spellcheck="false" placeholder="nvapi-…" autocomplete="off" />
+                <div class="slide-form-actions">
+                  <button class="btn btn-sm btn-primary" id="confirmAddKeyBtn">Add</button>
+                  <button class="btn btn-sm" id="cancelAddKeyBtn">Cancel</button>
+                </div>
+              </div>
+              <div class="empty-state hidden" id="keysEmpty">
+                <div class="empty-ttl">No keys loaded</div>
+                <div class="empty-desc">Add keys with the button above</div>
+              </div>
+              <div id="keysList" class="keys-list"></div>
+            </div>
+          </div>
+        </div>
+
+        <!-- ── ACTIVITY LOG (cell-log, col-span-2) ── -->
+        <div class="bento-cell cell-log col-span-2" data-dropdown>
+          <div class="bento-hd" data-toggle data-idx="04">
+            <span class="bento-ttl"><span class="log-dot" id="logDot"></span>Activity</span>
+            <span class="bento-badge" id="logCount">0</span>
+            <span class="chevron">▸</span>
+          </div>
+          <div class="bento-body" data-body>
+            <div class="bento-body-inner">
+              <div class="log-toolbar">
+                <button class="btn btn-sm" id="clearLogBtn">Clear</button>
+              </div>
+              <div class="log-box" id="logArea">
+                <div class="log-empty">Waiting for events…</div>
+              </div>
+            </div>
+          </div>
+        </div>
+
+        <!-- ── CLI SETUP (cell-cli, full width) ── -->
+        <div class="bento-cell cell-cli col-span-4" data-dropdown>
+          <div class="bento-hd" data-toggle data-idx="05">
+            <span class="bento-ttl">CLI Setup</span>
+            <span class="chevron">▸</span>
+          </div>
+          <div class="bento-body" data-body>
+            <div class="bento-body-inner">
+              <div class="cli-tabs" id="cliTabs"></div>
+              <div class="cli-setup" id="cliSetup"></div>
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
+
+    <script type="module" src="main.js"></script>
+  </body>
+</html>