scroot 0.2.0__py3-none-any.whl

scroot/dashboard/routers/settings.py

@@ -0,0 +1,291 @@
+"""Settings router - /api/settings endpoints."""
+from __future__ import annotations
+
+import json
+import os
+import time
+
+from fastapi import APIRouter, HTTPException
+
+from scroot.config.corrector import (
+    APIConfig,
+    CorrectorConfig,
+    LocalConfig,
+    default_config_path,
+)
+from scroot.corrector.models import MODEL_REGISTRY, get_model_path
+from scroot.dashboard.security import mask_api_key, validate_base_url
+
+_SETTINGS_FILE = os.path.join(os.getcwd(), ".scroot_settings.json")
+
+DEFAULT_WEIGHTS = {
+    "groundedness": 0.35,
+    "completeness": 0.25,
+    "relevance": 0.20,
+    "consistency": 0.15,
+    "confidence": 0.05,
+}
+
+DEFAULT_CONFIG: dict = {
+    "iqs_threshold": 0.70,
+    "metric_weights": DEFAULT_WEIGHTS,
+    "provider": "none",
+    "model": "",
+    "api_key": "",
+    "base_url": "",
+    "trigger_mode": "manual",
+}
+
+
+def _load() -> dict:
+    if os.path.exists(_SETTINGS_FILE):
+        with open(_SETTINGS_FILE, encoding="utf-8") as f:
+            return {**DEFAULT_CONFIG, **json.load(f)}
+    return DEFAULT_CONFIG.copy()
+
+
+def _save(config: dict) -> None:
+    # M-1: write key-bearing settings as owner-only (0600) so other local
+    # accounts can't read the stored API key.
+    flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
+    fd = os.open(_SETTINGS_FILE, flags, 0o600)
+    with os.fdopen(fd, "w", encoding="utf-8") as f:
+        json.dump(config, f, indent=2)
+    try:
+        os.chmod(_SETTINGS_FILE, 0o600)  # tighten if the file pre-existed
+    except OSError:
+        pass
+
+
+def _store_info(store) -> dict:
+    """Return record count and human-readable store size."""
+    if store is None:
+        return {"record_count": 0, "store_size": "—", "store_path": "~/.scroot/feedback.jsonl"}
+    try:
+        records = store.get_all()
+        count = len(records)
+        path = getattr(store, "_path", str(getattr(store, "path", "~/.scroot/feedback.jsonl")))
+        size = "—"
+        if os.path.exists(path):
+            b = os.path.getsize(path)
+            if b < 1024:
+                size = f"{b} B"
+            elif b < 1024 ** 2:
+                size = f"{b / 1024:.1f} KB"
+            else:
+                size = f"{b / 1024 ** 2:.1f} MB"
+        return {"record_count": count, "store_size": size, "store_path": path}
+    except (OSError, AttributeError):
+        return {"record_count": 0, "store_size": "—", "store_path": "~/.scroot/feedback.jsonl"}
+
+
+def settings_router(store=None):
+    router = APIRouter()
+
+    # ─── Unified settings (used by Settings page) ─────────────────────
+
+    def _corrector_state() -> dict:
+        """Build the corrector sub-object returned in GET /settings."""
+        cc = CorrectorConfig.load(default_config_path())
+        spec = MODEL_REGISTRY.get(cc.local.model_id)
+        model_path = get_model_path(cc.local.model_id) if spec else None
+        downloaded = model_path.exists() if model_path else False
+        api_key = cc.api.api_key
+        return {
+            "mode": cc.mode,
+            "local": {
+                "model_id":          cc.local.model_id,
+                "model_name":        spec.name if spec else cc.local.model_id,
+                "model_downloaded":  downloaded,
+                "model_size_gb":     spec.size_gb if spec else 0,
+                "model_path":        str(model_path) if downloaded else None,
+            },
+            "api": {
+                "api_key_set":    bool(api_key),
+                "api_key_prefix": api_key[:6] if len(api_key) >= 6 else api_key,
+                "base_url":       cc.api.base_url,
+                "model":          cc.api.model,
+            },
+        }
+
+    @router.get("")
+    def get_settings():
+        cfg = _load()
+        info = _store_info(store)
+        return {
+            "iqs_threshold": cfg.get("iqs_threshold", 0.70),
+            "metric_weights": cfg.get("metric_weights", DEFAULT_WEIGHTS),
+            "corrector": _corrector_state(),
+            # Legacy field kept for any old clients. H-1: never echo the raw
+            # key back - expose only a masked hint and a boolean.
+            "llm_corrector": {
+                "provider":     cfg.get("provider", "none"),
+                "api_key_set":  bool(cfg.get("api_key", "")),
+                "api_key_hint": mask_api_key(cfg.get("api_key", "")),
+                "base_url":     cfg.get("base_url", ""),
+                "model":        cfg.get("model", ""),
+            },
+            **info,
+        }
+
+    @router.put("")
+    def update_settings(body: dict):
+        """Patch settings. Handles iqs_threshold, metric_weights, corrector, clear_all_records."""
+        cfg = _load()
+
+        if "iqs_threshold" in body:
+            cfg["iqs_threshold"] = float(body["iqs_threshold"])
+
+        if "metric_weights" in body:
+            cfg["metric_weights"] = body["metric_weights"]
+
+        if "corrector" in body:
+            cc = CorrectorConfig.load(default_config_path())
+            patch = body["corrector"]
+            if "mode" in patch:
+                cc.mode = patch["mode"]
+            if "local" in patch:
+                lp = patch["local"]
+                cc.local = LocalConfig(
+                    model_id=lp.get("model_id", cc.local.model_id),
+                    n_threads=lp.get("n_threads", cc.local.n_threads),
+                    n_gpu_layers=lp.get("n_gpu_layers", cc.local.n_gpu_layers),
+                    context_window=lp.get("context_window", cc.local.context_window),
+                )
+            if "api" in patch:
+                ap = patch["api"]
+                # Write-only key semantics (H-1): a blank/absent api_key means
+                # "leave the stored key unchanged" rather than wiping it. This
+                # also stops an unrelated edit (e.g. model only) from clearing
+                # the key, since the UI never reads the real key back.
+                new_key = ap.get("api_key")
+                api_key = new_key if new_key else cc.api.api_key
+                new_base_url = ap.get("base_url", cc.api.base_url)
+                # M-2: reject untrusted/internal endpoints before persisting.
+                try:
+                    validate_base_url(new_base_url)
+                except ValueError as exc:
+                    raise HTTPException(status_code=400, detail=str(exc)) from exc
+                cc.api = APIConfig(
+                    api_key=api_key,
+                    base_url=new_base_url,
+                    model=ap.get("model", cc.api.model),
+                    system_prompt=ap.get("system_prompt", cc.api.system_prompt),
+                )
+            cc.save(default_config_path())
+
+        # Legacy llm_corrector key - still accepted
+        if "llm_corrector" in body:
+            lc = body["llm_corrector"]
+            cfg["provider"]  = lc.get("provider", cfg.get("provider", "none"))
+            # Blank/absent key = leave unchanged (H-1 write-only semantics).
+            new_key = lc.get("api_key")
+            cfg["api_key"]   = new_key if new_key else cfg.get("api_key", "")
+            new_base_url     = lc.get("base_url", cfg.get("base_url", ""))
+            try:
+                validate_base_url(new_base_url)  # M-2
+            except ValueError as exc:
+                raise HTTPException(status_code=400, detail=str(exc)) from exc
+            cfg["base_url"]  = new_base_url
+            cfg["model"]     = lc.get("model",    cfg.get("model", ""))
+
+        if body.get("clear_all_records") and store is not None:
+            try:
+                store.purge()
+            except Exception:
+                pass
+
+        _save(cfg)
+        return {"status": "ok"}
+
+    @router.post("/test-connection")
+    def test_connection():
+        cfg = _load()
+        provider = cfg.get("provider", "none")
+        if provider == "llm":
+            from .records import _detect_provider
+            provider = _detect_provider(cfg)
+
+        if provider == "none":
+            return {"status": "error", "latency_ms": 0, "message": "No provider configured"}
+
+        api_key = cfg.get("api_key", "")
+        base_url = cfg.get("base_url") or None
+        model = cfg.get("model", "")
+
+        # M-2: never send the key to an unvetted endpoint, even on a test ping.
+        try:
+            validate_base_url(base_url)
+        except ValueError as exc:
+            return {"status": "error", "latency_ms": 0, "message": str(exc)}
+
+        start = time.time()
+        sample = ""
+        status = "ok"
+        message = ""
+
+        try:
+            if provider == "anthropic":
+                import anthropic
+                client = anthropic.Anthropic(api_key=api_key, base_url=base_url)
+                msg = client.messages.create(
+                    model=model or "claude-haiku-4-5-20251001",
+                    max_tokens=16,
+                    messages=[{"role": "user", "content": "Reply: ok"}],
+                )
+                sample = msg.content[0].text
+
+            elif provider in ("openai", "groq", "openrouter"):
+                import openai
+                client = openai.OpenAI(api_key=api_key, base_url=base_url)
+                resp = client.chat.completions.create(
+                    model=model or "gpt-4o-mini",
+                    messages=[{"role": "user", "content": "Reply: ok"}],
+                    max_tokens=16,
+                )
+                sample = resp.choices[0].message.content
+
+            elif provider == "ollama":
+                import requests
+                url = (base_url or "http://localhost:11434") + "/api/generate"
+                resp = requests.post(
+                    url,
+                    json={"model": model or "llama3.2", "prompt": "Reply: ok", "stream": False},
+                    timeout=10,
+                )
+                sample = resp.json().get("response", "")
+
+        except Exception as e:
+            status = "error"
+            message = str(e)
+
+        latency_ms = int((time.time() - start) * 1000)
+        return {"status": status, "latency_ms": latency_ms, "sample_output": sample[:200], "message": message}
+
+    # ─── Legacy llm-judge sub-routes (kept for backwards compat) ──────
+
+    @router.get("/llm-judge")
+    def get_llm_judge():
+        cfg = _load()
+        return {
+            "provider":       cfg.get("provider", "none"),
+            "model":          cfg.get("model", ""),
+            "trigger_mode":   cfg.get("trigger_mode", "manual"),
+            "budget_cap_usd": cfg.get("budget_cap_usd"),
+            "api_key_env_var": cfg.get("api_key_env_var", ""),
+        }
+
+    @router.put("/llm-judge")
+    def save_llm_judge(body: dict):
+        cfg = _load()
+        for k in ("provider", "model", "trigger_mode", "budget_cap_usd", "api_key_env_var"):
+            if k in body:
+                cfg[k] = body[k]
+        _save(cfg)
+        return body
+
+    @router.post("/llm-judge/test")
+    def test_llm_judge():
+        return test_connection()
+
+    return router

scroot/dashboard/security.py

@@ -0,0 +1,135 @@
+"""Shared dashboard security helpers.
+
+Covers three hardening controls for the local Review Console:
+
+* **H-1** - ``mask_api_key()`` so stored provider keys are never echoed back
+  in plaintext over the API.
+* **M-2** - ``validate_base_url()`` so the server cannot be pointed at an
+  attacker-controlled host (key exfiltration) or used as an SSRF pivot to
+  internal / cloud-metadata endpoints.
+* **H-2** - ``require_token`` middleware factory + ``is_loopback_host()`` so a
+  network-exposed dashboard can require a shared token and a non-loopback bind
+  warns the operator.
+"""
+
+from __future__ import annotations
+
+import hmac
+import ipaddress
+import os
+from urllib.parse import urlparse
+
+# ---------------------------------------------------------------------------
+# H-1: API key masking
+# ---------------------------------------------------------------------------
+
+def mask_api_key(key: str | None) -> str:
+    """Return a non-reversible hint for an API key (never the full value).
+
+    ``"sk-abcdefgh...wxyz"`` -> ``"sk-a…wxyz"``. Empty/short keys collapse to a
+    placeholder so the real value never leaves the process.
+    """
+    if not key:
+        return ""
+    if len(key) <= 8:
+        return "…"
+    return f"{key[:4]}…{key[-4:]}"
+
+
+# ---------------------------------------------------------------------------
+# M-2: outbound base_url allowlist
+# ---------------------------------------------------------------------------
+
+#: Known hosted LLM provider hosts that may receive an API key.
+ALLOWED_LLM_HOSTS: frozenset[str] = frozenset({
+    "api.openai.com",
+    "api.anthropic.com",
+    "api.groq.com",
+    "openrouter.ai",
+    "api.openrouter.ai",
+    "generativelanguage.googleapis.com",  # Google Gemini
+})
+
+#: Loopback hosts permitted for self-hosted endpoints (e.g. local Ollama).
+_LOCAL_HOSTS: frozenset[str] = frozenset({"localhost", "127.0.0.1", "::1"})
+
+#: Escape hatch for operators who deliberately use a custom gateway.
+_OVERRIDE_ENV = "SCROOT_ALLOW_ANY_BASE_URL"
+
+
+def validate_base_url(base_url: str | None, *, allow_local: bool = True) -> None:
+    """Reject base URLs that aren't a known provider or an allowed local host.
+
+    Args:
+        base_url: The configured endpoint. Empty/None means "use the provider
+            SDK's default endpoint" and is always allowed.
+        allow_local: Permit loopback hosts (needed for local Ollama). Set False
+            for hosted providers that should never be local.
+
+    Raises:
+        ValueError: If the host is neither an allowlisted provider nor an
+            allowed loopback host, and the override env var is not set.
+    """
+    if not base_url:
+        return
+    if os.environ.get(_OVERRIDE_ENV) == "1":
+        return
+
+    parsed = urlparse(base_url)
+    if parsed.scheme not in ("http", "https"):
+        raise ValueError(
+            f"base_url must use http(s), got {base_url!r}"
+        )
+
+    host = (parsed.hostname or "").lower()
+    if host in ALLOWED_LLM_HOSTS:
+        return
+    if allow_local and host in _LOCAL_HOSTS:
+        return
+
+    raise ValueError(
+        f"base_url host {host!r} is not an allowed LLM provider endpoint. "
+        f"This blocks pointing the server at an untrusted host (key theft) or "
+        f"an internal/metadata address (SSRF). Allowed providers: "
+        f"{', '.join(sorted(ALLOWED_LLM_HOSTS))}. To override for a trusted "
+        f"custom gateway, set {_OVERRIDE_ENV}=1."
+    )
+
+
+# ---------------------------------------------------------------------------
+# H-2: bind-host inspection + token auth
+# ---------------------------------------------------------------------------
+
+def is_loopback_host(host: str) -> bool:
+    """True if ``host`` is a loopback / localhost address."""
+    if host in ("localhost", ""):
+        return True
+    try:
+        return ipaddress.ip_address(host).is_loopback
+    except ValueError:
+        return False
+
+
+def resolve_dashboard_token(explicit: str | None = None) -> str | None:
+    """Return the configured dashboard token, if any.
+
+    Precedence: explicit argument, then ``SCROOT_DASHBOARD_TOKEN`` env var.
+    Returns None when no token is configured (auth disabled).
+    """
+    token = explicit or os.environ.get("SCROOT_DASHBOARD_TOKEN") or ""
+    return token or None
+
+
+def token_matches(provided: str | None, expected: str) -> bool:
+    """Constant-time comparison of a presented token against the expected one."""
+    if not provided:
+        return False
+    return hmac.compare_digest(provided, expected)
+
+
+def extract_request_token(headers) -> str | None:
+    """Pull a token from ``Authorization: Bearer`` or ``X-Scroot-Token``."""
+    auth = headers.get("authorization") or headers.get("Authorization") or ""
+    if auth.lower().startswith("bearer "):
+        return auth[7:].strip()
+    return headers.get("x-scroot-token") or headers.get("X-Scroot-Token")

scroot/dashboard/server.py

@@ -0,0 +1,181 @@
+"""Scroot Dashboard - FastAPI application factory."""
+from __future__ import annotations
+
+import warnings
+from pathlib import Path
+
+from fastapi import FastAPI
+from fastapi.responses import JSONResponse
+from fastapi.staticfiles import StaticFiles
+
+from scroot.feedback.store import FeedbackStore
+from .security import (
+    extract_request_token,
+    is_loopback_host,
+    resolve_dashboard_token,
+    token_matches,
+)
+from .routers.queue import queue_router
+from .routers.records import records_router
+from .routers.analytics import analytics_router
+from .routers.export import export_router
+from .routers.settings import settings_router
+from .routers.pipeline import pipeline_router
+from .routers.corrector import corrector_router
+from .routers.guardrails import guardrails_router
+
+# Resolved at import time - ui/dist is built by `npm run build`
+UI_DIST_PATH = str(Path(__file__).parent.parent / "ui" / "dist")
+
+# Endpoints reachable without a token even when auth is enabled.
+_UNAUTHENTICATED_PATHS = frozenset({"/api/health"})
+
+
+def create_app(
+    store_path: str,
+    hosted: bool = False,
+    host: str = "127.0.0.1",
+    auth_token: str | None = None,
+) -> FastAPI:
+    """Create the Scroot dashboard FastAPI application.
+
+    Args:
+        store_path: Path to the JSONL FeedbackStore file.
+        hosted: Reserved for Scroot Enterprise hosted mode.
+        host: The interface the app will be served on. Used only to decide
+            whether to warn about an unauthenticated non-loopback bind (H-2).
+        auth_token: Optional shared token. When set (or
+            ``SCROOT_DASHBOARD_TOKEN`` is in the environment), every
+            ``/api/*`` route except the health check requires the token via an
+            ``Authorization: Bearer <token>`` or ``X-Scroot-Token`` header.
+
+    Raises:
+        NotImplementedError: If hosted=True (enterprise-only feature).
+    """
+    if hosted:
+        raise NotImplementedError(
+            "Hosted mode is available in Scroot Cloud. "
+            "Visit https://scroot.dev/cloud for enterprise pricing."
+        )
+
+    store = FeedbackStore(store_path)
+    token = resolve_dashboard_token(auth_token)
+
+    # H-2: the dashboard has no per-user auth. A loopback bind is single-user
+    # safe; binding to a routable interface exposes the full correction store
+    # and the corrector API key to the network. Warn unless a token is set.
+    if not is_loopback_host(host) and token is None:
+        warnings.warn(
+            f"Scroot dashboard is binding to a non-loopback host ({host!r}) "
+            f"with no authentication. The correction store and stored LLM API "
+            f"key would be reachable by anyone on the network. Set "
+            f"SCROOT_DASHBOARD_TOKEN (or pass --token) and/or run behind an "
+            f"authenticating reverse proxy.",
+            stacklevel=2,
+        )
+
+    app = FastAPI(
+        title="Scroot Review Console",
+        description="Local feedback loop review dashboard",
+        version="0.2.0",
+    )
+
+    # H-2: optional shared-token gate for network-exposed deployments.
+    if token is not None:
+        @app.middleware("http")
+        async def _require_token(request, call_next):
+            path = request.url.path
+            if path.startswith("/api/") and path not in _UNAUTHENTICATED_PATHS:
+                provided = extract_request_token(request.headers)
+                if not token_matches(provided, token):
+                    return JSONResponse(
+                        status_code=401,
+                        content={"detail": "Missing or invalid dashboard token."},
+                    )
+            return await call_next(request)
+
+    # CORS for Vite dev server (dev only - not needed in production)
+    try:
+        from fastapi.middleware.cors import CORSMiddleware
+        app.add_middleware(
+            CORSMiddleware,
+            allow_origins=["http://localhost:5173", "http://127.0.0.1:5173"],
+            allow_methods=["*"],
+            allow_headers=["*"],
+        )
+    except ImportError:
+        pass
+
+    # API routers
+    app.include_router(queue_router(store),        prefix="/api/queue",    tags=["queue"])
+    app.include_router(records_router(store),      prefix="/api/records",  tags=["records"])
+    app.include_router(analytics_router(store),    prefix="/api/analytics", tags=["analytics"])
+    app.include_router(export_router(store),       prefix="/api/export",   tags=["export"])
+    app.include_router(settings_router(store),     prefix="/api/settings", tags=["settings"])
+    app.include_router(pipeline_router(store),     prefix="/api/pipeline", tags=["pipeline"])
+    app.include_router(corrector_router(),         prefix="/api/corrector", tags=["corrector"])
+    app.include_router(guardrails_router(store),   prefix="/api/guardrails", tags=["guardrails"])
+
+    # Health check
+    @app.get("/api/health")
+    def health():
+        pending = sum(
+            1 for r in store.get_all()
+            if getattr(r, "status", "pending") == "pending"
+        )
+        iqs_vals = [
+            r.scores.get("iqs", 0)
+            for r in store.get_all()
+            if isinstance(r.scores, dict)
+        ]
+        avg_iqs = round(sum(iqs_vals) / len(iqs_vals), 3) if iqs_vals else None
+        return {
+            "status": "ok",
+            "version": "0.2.0",
+            "pending_count": pending,
+            "avg_iqs_today": avg_iqs,
+        }
+
+    # Serve built React SPA - must be registered last.
+    dist = Path(UI_DIST_PATH)
+    if dist.exists():
+        from fastapi.responses import FileResponse
+
+        index_file = dist / "index.html"
+        # Hashed build assets (JS/CSS) live under /assets.
+        assets_dir = dist / "assets"
+        if assets_dir.exists():
+            app.mount(
+                "/assets",
+                StaticFiles(directory=str(assets_dir)),
+                name="assets",
+            )
+
+        # SPA history-API fallback: the dashboard uses BrowserRouter, so deep
+        # links and refreshes hit the server with a client-side path (e.g.
+        # /queue, /analytics). Serve a real file when one exists (favicon
+        # etc.), otherwise return index.html so the SPA can route. /api/* is
+        # never caught here (those routes are registered above).
+        @app.get("/{full_path:path}")
+        def serve_spa(full_path: str):
+            # Don't shadow the API: unknown /api paths get a real 404, not the SPA.
+            if full_path.startswith("api/"):
+                from fastapi import HTTPException
+                raise HTTPException(status_code=404, detail="Not Found")
+            candidate = (dist / full_path).resolve()
+            if (
+                full_path
+                and dist.resolve() in candidate.parents
+                and candidate.is_file()
+            ):
+                return FileResponse(str(candidate))
+            return FileResponse(str(index_file))
+    else:
+        @app.get("/")
+        def ui_not_built():
+            return {
+                "error": "UI not built",
+                "hint": "cd src/scroot/ui && npm install && npm run build",
+            }
+
+    return app