PyPI - arbiter-cli - Versions diffs - 0.1.0__py3-none-any.whl - Mend

arbiter-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (35) hide show

arbiter/__init__.py +3 -0
arbiter/cli/__init__.py +0 -0
arbiter/cli/app.py +699 -0
arbiter/cli/display.py +381 -0
arbiter/core/__init__.py +0 -0
arbiter/core/benchmarks.py +804 -0
arbiter/core/config.py +137 -0
arbiter/core/discover.py +184 -0
arbiter/core/judge.py +193 -0
arbiter/core/leaderboard.py +197 -0
arbiter/core/metrics.py +367 -0
arbiter/core/providers/__init__.py +19 -0
arbiter/core/providers/anthropic_provider.py +133 -0
arbiter/core/providers/base.py +62 -0
arbiter/core/providers/factory.py +79 -0
arbiter/core/providers/google_provider.py +126 -0
arbiter/core/providers/ollama.py +103 -0
arbiter/core/providers/openai_provider.py +120 -0
arbiter/core/runner.py +257 -0
arbiter/core/swe/__init__.py +1 -0
arbiter/core/swe/container.py +158 -0
arbiter/core/swe/runner.py +220 -0
arbiter/core/swe/sandbox.py +111 -0
arbiter/core/swe/test_packs.py +548 -0
arbiter/dashboard/__init__.py +0 -0
arbiter/dashboard/frontend/dist/assets/index-1tkxJouQ.css +1 -0
arbiter/dashboard/frontend/dist/assets/index-dHa4zmvw.js +298 -0
arbiter/dashboard/frontend/dist/index.html +16 -0
arbiter/dashboard/server.py +426 -0
arbiter_cli-0.1.0.dist-info/METADATA +299 -0
arbiter_cli-0.1.0.dist-info/RECORD +35 -0
arbiter_cli-0.1.0.dist-info/WHEEL +5 -0
arbiter_cli-0.1.0.dist-info/entry_points.txt +2 -0
arbiter_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
arbiter_cli-0.1.0.dist-info/top_level.txt +1 -0

arbiter/dashboard/frontend/dist/index.html ADDED Viewed

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html lang="en" class="dark">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Arbiter - The Final Word on Your Local Models</title>
+    <link rel="preconnect" href="https://fonts.googleapis.com" />
+    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
+    <link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono:wght@400;500;600;700&family=Inter:wght@400;500;600;700;800&display=swap" rel="stylesheet" />
+    <script type="module" crossorigin src="/assets/index-dHa4zmvw.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-1tkxJouQ.css">
+  </head>
+  <body class="bg-arbiter-bg text-white font-sans antialiased">
+    <div id="root"></div>
+  </body>
+</html>

arbiter/dashboard/server.py ADDED Viewed

@@ -0,0 +1,426 @@
+"""FastAPI dashboard backend with WebSocket streaming."""
+from __future__ import annotations
+import asyncio
+import json
+import os
+import signal
+import webbrowser
+from pathlib import Path
+from typing import Optional
+import uvicorn
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse, HTMLResponse
+from fastapi.staticfiles import StaticFiles
+from arbiter.core.judge import judge_comparison
+from arbiter.core.leaderboard import Leaderboard
+from arbiter.core.metrics import ComparisonResult, ModelMetrics
+from arbiter.core.runner import stream_comparison, run_single_model
+from arbiter.core.config import resolve_model
+FRONTEND_DIR = Path(__file__).parent / "frontend" / "dist"
+app = FastAPI(title="Arbiter Dashboard")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+# State for the current comparison
+_comparison_state: dict = {
+    "model_specs": [],
+    "prompt": "",
+    "image_path": None,
+    "judge_model": "auto",
+    "no_judge": False,
+    "system": None,
+    "result": None,
+    "leaderboard": None,
+    "sequential": False,
+}
+@app.get("/api/health")
+async def health():
+    return {"status": "ok"}
+@app.get("/api/leaderboard")
+async def get_leaderboard():
+    lb = Leaderboard()
+    return lb.to_dict()
+@app.get("/api/config")
+async def get_config():
+    return {
+        "model_specs": _comparison_state["model_specs"],
+        "prompt": _comparison_state["prompt"],
+    }
+@app.get("/api/result")
+async def get_result():
+    if _comparison_state["result"]:
+        return _comparison_state["result"].to_dict()
+    return {"status": "pending"}
+@app.get("/api/models")
+async def get_models():
+    """List all available models with memory fitness info."""
+    from arbiter.core.discover import (
+        discover_ollama, discover_openai, discover_anthropic,
+        discover_google, get_system_memory,
+    )
+    all_models = []
+    for discover_fn in [discover_ollama, discover_openai, discover_anthropic, discover_google]:
+        try:
+            found = await discover_fn()
+            all_models.extend(found)
+        except Exception:
+            pass
+    mem = get_system_memory()
+    return {
+        "system": {
+            "total_ram_gb": mem["total_gb"],
+            "available_ram_gb": mem["available_gb"],
+            "ram_percent_used": mem["percent"],
+        },
+        "models": [
+            {
+                "name": m.name,
+                "provider": m.provider,
+                "size_gb": m.size_gb,
+                "parameter_size": m.parameter_size,
+                "family": m.family,
+                "multimodal": m.multimodal,
+                "spec": m.spec,
+                "fits_in_memory": m.fits_in_memory,
+                "memory_warning": m.memory_warning,
+            }
+            for m in all_models
+        ],
+    }
+@app.post("/api/run")
+async def start_run(body: dict):
+    """Start a comparison from the dashboard.
+    Body: {"models": ["gemma4:e2b", "qwen3.5:4b"], "prompt": "...", "sequential": true}
+    """
+    models = body.get("models", [])
+    prompt = body.get("prompt", "")
+    sequential = body.get("sequential", True)
+    if not models or not prompt:
+        return {"error": "Need at least one model and a prompt"}
+    # Update state so the WebSocket picks it up
+    _comparison_state.update({
+        "model_specs": models,
+        "prompt": prompt,
+        "sequential": sequential,
+        "no_judge": body.get("no_judge", False),
+        "judge_model": body.get("judge_model", "auto"),
+        "system": body.get("system"),
+        "image_path": None,
+        "result": None,
+    })
+    return {"status": "started", "models": models, "prompt": prompt}
+@app.post("/api/benchmark")
+async def start_benchmark(body: dict):
+    """Start a benchmark run from the dashboard.
+    Body: {"models": ["gemma4:e2b"], "quick": false}
+    """
+    from arbiter.core.benchmarks import run_benchmark_comparison, run_benchmark_suite
+    models = body.get("models", [])
+    quick = body.get("quick", False)
+    if not models:
+        return {"error": "Need at least one model"}
+    results = await run_benchmark_comparison(models, quick=quick)
+    return {"results": [r.to_dict() for r in results]}
+@app.get("/api/benchmark/categories")
+async def get_benchmark_categories():
+    """Get available benchmark test categories for the UI."""
+    from arbiter.core.benchmarks import CATEGORIES, ALL_TESTS, QUICK_TESTS
+    return {
+        "categories": CATEGORIES,
+        "total_tests": len(ALL_TESTS),
+        "quick_tests": len(QUICK_TESTS),
+    }
+@app.websocket("/ws")
+async def websocket_endpoint(ws: WebSocket):
+    """WebSocket endpoint for real-time comparison streaming.
+    Sends events:
+        {"type": "config", "models": [...], "prompt": "..."}
+        {"type": "start", "model": "gemma4"}
+        {"type": "token", "model": "gemma4", "text": "...", "metrics": {...}}
+        {"type": "done", "model": "gemma4", "metrics": {...}}
+        {"type": "judging"}
+        {"type": "result", "data": {...}}
+        {"type": "leaderboard", "data": {...}}
+    """
+    await ws.accept()
+    try:
+        # Send config
+        await ws.send_json(
+            {
+                "type": "config",
+                "models": _comparison_state["model_specs"],
+                "prompt": _comparison_state["prompt"],
+            }
+        )
+        # Always send leaderboard data
+        lb = Leaderboard()
+        await ws.send_json({"type": "leaderboard", "data": lb.to_dict()})
+        # If no models configured, just keep connection alive (dashboard-only mode)
+        if not _comparison_state["model_specs"]:
+            await ws.send_json({"type": "idle", "message": "No comparison running."})
+            while True:
+                try:
+                    await asyncio.wait_for(ws.receive_text(), timeout=30)
+                except asyncio.TimeoutError:
+                    try:
+                        await ws.send_json({"type": "ping"})
+                    except Exception:
+                        break
+                except WebSocketDisconnect:
+                    break
+            return
+        # If we already have a result, send it immediately
+        if _comparison_state["result"]:
+            await ws.send_json(
+                {
+                    "type": "result",
+                    "data": _comparison_state["result"].to_dict(),
+                }
+            )
+            lb = Leaderboard()
+            await ws.send_json({"type": "leaderboard", "data": lb.to_dict()})
+            return
+        # Stream the comparison
+        all_metrics = []
+        is_seq = _comparison_state.get("sequential", False)
+        if is_seq:
+            # Sequential: run one model at a time, queue tokens for ordered sends
+            for spec in _comparison_state["model_specs"]:
+                cfg = resolve_model(spec)
+                model_name = cfg.extra["model"]
+                await ws.send_json({"type": "start", "model": model_name})
+                token_queue = asyncio.Queue()
+                def _on_token_sync(name, text, m, _q=token_queue):
+                    _q.put_nowait({
+                        "type": "token",
+                        "model": name,
+                        "text": m.output[-50:],
+                        "total_text_length": len(m.output),
+                        "metrics": {
+                            "tokens": m._token_count,
+                            "tokens_per_sec": round(m.tokens_per_sec, 1) if m.tokens_per_sec else None,
+                            "ttft_ms": round(m.ttft_ms, 1) if m.ttft_ms else None,
+                        },
+                    })
+                async def _drain_queue(q, done_event):
+                    while not done_event.is_set() or not q.empty():
+                        try:
+                            msg = q.get_nowait()
+                            await ws.send_json(msg)
+                        except asyncio.QueueEmpty:
+                            await asyncio.sleep(0.05)
+                try:
+                    done_event = asyncio.Event()
+                    drain_task = asyncio.create_task(_drain_queue(token_queue, done_event))
+                    metrics = await run_single_model(
+                        model_spec=spec,
+                        prompt=_comparison_state["prompt"],
+                        system=_comparison_state.get("system"),
+                        image_path=_comparison_state.get("image_path"),
+                        on_token=_on_token_sync,
+                    )
+                    done_event.set()
+                    await drain_task
+                    all_metrics.append(metrics)
+                    await ws.send_json({"type": "done", "model": model_name, "metrics": metrics.to_dict()})
+                except Exception as e:
+                    done_event.set()
+                    m = ModelMetrics(model=model_name, provider=cfg.provider, output=f"[ERROR] {e}")
+                    all_metrics.append(m)
+                    await ws.send_json({"type": "error", "model": model_name, "metrics": m.to_dict()})
+        else:
+            # Parallel: stream all at once
+            async for event_type, model_name, metrics in stream_comparison(
+                model_specs=_comparison_state["model_specs"],
+                prompt=_comparison_state["prompt"],
+                system=_comparison_state.get("system"),
+                image_path=_comparison_state.get("image_path"),
+            ):
+                if event_type == "start":
+                    await ws.send_json({"type": "start", "model": model_name})
+                elif event_type == "token":
+                    await ws.send_json({
+                        "type": "token",
+                        "model": model_name,
+                        "text": metrics.output[-50:],
+                        "total_text_length": len(metrics.output),
+                        "metrics": {
+                            "tokens": metrics._token_count,
+                            "tokens_per_sec": round(metrics.tokens_per_sec, 1) if metrics.tokens_per_sec else None,
+                            "ttft_ms": round(metrics.ttft_ms, 1) if metrics.ttft_ms else None,
+                        },
+                    })
+                elif event_type in ("done", "error"):
+                    all_metrics.append(metrics)
+                    await ws.send_json({
+                        "type": "done" if event_type == "done" else "error",
+                        "model": model_name,
+                        "metrics": metrics.to_dict(),
+                    })
+        # Build comparison result
+        result = ComparisonResult(
+            prompt=_comparison_state["prompt"],
+            models=all_metrics,
+        )
+        # Judge
+        has_quality = False
+        if not _comparison_state["no_judge"] and len(all_metrics) > 1:
+            await ws.send_json({"type": "judging"})
+            result = await judge_comparison(
+                result, judge_model=_comparison_state["judge_model"]
+            )
+            has_quality = True
+        # Compute composite scores and winner
+        from arbiter.core.metrics import compute_composite_scores
+        result.scoring = compute_composite_scores(result, has_quality=has_quality)
+        result.winner = result.scoring.winner if result.scoring else None
+        # Update leaderboard
+        if len(all_metrics) > 1:
+            lb = Leaderboard()
+            lb.update_from_comparison(result)
+            await ws.send_json({"type": "leaderboard", "data": lb.to_dict()})
+        _comparison_state["result"] = result
+        await ws.send_json({"type": "result", "data": result.to_dict()})
+    except WebSocketDisconnect:
+        pass
+    except Exception as e:
+        try:
+            await ws.send_json({"type": "error", "message": str(e)})
+        except Exception:
+            pass
+def _find_frontend_dist() -> Optional[Path]:
+    """Find the frontend dist directory, checking multiple locations."""
+    candidates = [
+        Path(__file__).parent / "frontend" / "dist",
+        Path(__file__).resolve().parent / "frontend" / "dist",
+    ]
+    for c in candidates:
+        if c.exists() and (c / "index.html").exists():
+            return c
+    return None
+_DIST = _find_frontend_dist()
+if _DIST and (_DIST / "assets").exists():
+    app.mount("/assets", StaticFiles(directory=str(_DIST / "assets")), name="assets")
+@app.get("/")
+async def serve_index():
+    """Serve the dashboard index page."""
+    dist = _find_frontend_dist()
+    if dist:
+        return FileResponse(str(dist / "index.html"))
+    return HTMLResponse(
+        "<h1>Arbiter Dashboard</h1>"
+        "<p>Frontend not built yet. Run:</p>"
+        "<pre>cd arbiter/dashboard/frontend && npm install && npm run build</pre>"
+        "<p>WebSocket API is available at /ws</p>"
+    )
+async def start_server(
+    model_specs: list[str],
+    prompt: str,
+    image_path: Optional[str] = None,
+    judge_model: str = "auto",
+    no_judge: bool = False,
+    system: Optional[str] = None,
+    sequential: bool = False,
+    port: int = 7878,
+) -> None:
+    """Start the dashboard server and open the browser."""
+    _comparison_state.update(
+        {
+            "model_specs": model_specs,
+            "prompt": prompt,
+            "image_path": image_path,
+            "judge_model": judge_model,
+            "no_judge": no_judge,
+            "system": system,
+            "sequential": sequential,
+            "result": None,
+        }
+    )
+    config = uvicorn.Config(
+        app,
+        host="127.0.0.1",
+        port=port,
+        log_level="warning",
+    )
+    server = uvicorn.Server(config)
+    # Open browser after a short delay
+    async def _open_browser():
+        await asyncio.sleep(1)
+        webbrowser.open(f"http://127.0.0.1:{port}")
+    await asyncio.gather(
+        server.serve(),
+        _open_browser(),
+    )