getbased-dashboard 0.5.0__py3-none-any.whl

getbased_dashboard/__init__.py

@@ -0,0 +1,9 @@
+"""getbased-dashboard — web UI for getbased-agents.
+
+Orchestration layer that sits between the browser and the rag + mcp
+packages. Holds no data of its own: proxies knowledge-base operations
+to rag, spawns the mcp stdio process on demand for tool discovery and
+config generation, reads the mcp's activity log for the dashboard feed.
+"""
+
+__version__ = "0.1.0"

getbased_dashboard/api/__init__.py

@@ -0,0 +1,6 @@
+"""Dashboard HTTP API.
+
+Each submodule registers its routes onto a FastAPI app via a `register()`
+function. Keeps the server.py entry point readable and lets each concern
+(knowledge, mcp, activity) own its own dependencies.
+"""

getbased_dashboard/api/activity.py

@@ -0,0 +1,153 @@
+"""Activity API — tail the MCP's JSONL activity log and surface the
+recent records + simple aggregations.
+
+The log is written by getbased-mcp at `$XDG_STATE_HOME/getbased/mcp/
+activity.jsonl` (configurable via LENS_MCP_ACTIVITY_LOG). One record
+per tool call: tool name, timestamp, duration, ok flag, error class on
+failure. Args are never logged upstream so we don't have to strip them.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from collections import defaultdict
+from pathlib import Path
+
+from fastapi import APIRouter, FastAPI, Request
+
+from ..config import DashboardConfig
+from ..server import _require_auth
+
+
+def _cfg(request: Request) -> DashboardConfig:
+    return request.app.state.config
+
+
+# Cap on how much of the log we read per request. Users with heavy agent
+# usage can accumulate megabytes quickly — loading the entire file on
+# every poll is wasteful. Tailing from the end keeps the endpoint O(cap)
+# regardless of how long the log has been running.
+_TAIL_BYTES = 512 * 1024
+
+
+def _read_records(path: Path, limit: int) -> list[dict]:
+    """Return up to `limit` most-recent records. If the file is under
+    _TAIL_BYTES read the whole thing; otherwise seek from the end. Malformed
+    lines (partial last write, corrupt records) are silently skipped so
+    one bad line can't hide the rest."""
+    if not path.exists():
+        return []
+    size = path.stat().st_size
+    try:
+        if size <= _TAIL_BYTES:
+            text = path.read_text(encoding="utf-8", errors="replace")
+        else:
+            with path.open("rb") as f:
+                f.seek(size - _TAIL_BYTES)
+                # Drop the first (likely partial) line so we don't parse
+                # garbage. There will always be a complete line after the
+                # first newline we find, assuming writers use line-atomic
+                # append — which Python's text-mode write does.
+                f.readline()
+                text = f.read().decode("utf-8", errors="replace")
+    except OSError:
+        return []
+
+    records: list[dict] = []
+    for line in text.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            rec = json.loads(line)
+            if isinstance(rec, dict):
+                records.append(rec)
+        except json.JSONDecodeError:
+            continue
+
+    return records[-limit:]
+
+
+def _aggregate(records: list[dict]) -> dict:
+    """Per-tool counts, success rate, and P50/P95 latency. O(N log N) —
+    fine up to the ~thousand records our tail window holds."""
+    by_tool: dict[str, list[dict]] = defaultdict(list)
+    for r in records:
+        t = r.get("tool")
+        if isinstance(t, str):
+            by_tool[t].append(r)
+
+    def _percentile(sorted_vals: list[int], p: float) -> int | None:
+        if not sorted_vals:
+            return None
+        idx = int(p * (len(sorted_vals) - 1))
+        return sorted_vals[idx]
+
+    tools: list[dict] = []
+    for name, group in sorted(by_tool.items()):
+        durations = sorted(
+            int(r.get("duration_ms", 0)) for r in group if isinstance(r.get("duration_ms"), (int, float))
+        )
+        errors = sum(1 for r in group if not r.get("ok", True))
+        tools.append(
+            {
+                "tool": name,
+                "calls": len(group),
+                "errors": errors,
+                "error_rate": (errors / len(group)) if group else 0.0,
+                "p50_ms": _percentile(durations, 0.5),
+                "p95_ms": _percentile(durations, 0.95),
+            }
+        )
+
+    total_errors = sum(1 for r in records if not r.get("ok", True))
+    return {
+        "total_calls": len(records),
+        "total_errors": total_errors,
+        "overall_error_rate": (total_errors / len(records)) if records else 0.0,
+        "tools": tools,
+    }
+
+
+def register(app: FastAPI) -> None:
+    router = APIRouter(prefix="/api/activity", tags=["activity"])
+
+    @router.get("")
+    async def activity_feed(request: Request, limit: int = 200):
+        cfg = _cfg(request)
+        _require_auth(request, cfg)
+        # Bound limit so a client can't ask us to return 10 million records
+        # in one payload. 1000 is plenty for a dashboard tick.
+        limit = max(1, min(1000, int(limit)))
+        records = _read_records(cfg.activity_log, limit)
+        stats = _aggregate(records)
+        return {
+            "log_path": str(cfg.activity_log),
+            "log_exists": cfg.activity_log.exists(),
+            "records": records,
+            "stats": stats,
+        }
+
+    @router.delete("")
+    async def clear_activity(request: Request):
+        """Wipe the log. Useful for resetting the dashboard's view after
+        a period of testing. Returns the new (empty) state so the UI can
+        refresh in one round-trip."""
+        cfg = _cfg(request)
+        _require_auth(request, cfg)
+        try:
+            if cfg.activity_log.exists():
+                os.unlink(cfg.activity_log)
+        except OSError:
+            # File may have been created by another process or removed in
+            # a race; either way we want to return "nothing here" state.
+            pass
+        return {
+            "log_path": str(cfg.activity_log),
+            "log_exists": False,
+            "records": [],
+            "stats": _aggregate([]),
+        }
+
+    app.include_router(router)

getbased_dashboard/api/knowledge.py

@@ -0,0 +1,367 @@
+"""Knowledge tab API — thin proxy to getbased-rag.
+
+We don't replicate rag's data model here; we just forward the browser's
+bearer-authed requests to rag's endpoints, with one layer of timeout +
+error normalisation so the frontend sees consistent JSON shapes.
+
+All endpoints require the dashboard bearer token (same value as rag's).
+Dashboard validates the browser's token, then uses the same key to
+authenticate its upstream call to rag.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from pathlib import Path
+
+import httpx
+from fastapi import APIRouter, FastAPI, File, HTTPException, Request, UploadFile
+from fastapi.responses import StreamingResponse
+
+from ..config import DashboardConfig
+from ..server import _require_auth
+
+_KNOWLEDGE_TIMEOUT = 30.0
+_INGEST_TIMEOUT = 300.0  # real-model ingest can run minutes on large files
+
+# Size cap enforced at the dashboard hop. Mirrors rag's default so users
+# see one consistent ceiling regardless of which layer catches an oversize
+# upload. Configurable via env so power users with large corpora can
+# raise it on both services deliberately. Note: rag has its own cap that
+# ultimately bounds on-disk writes — this dashboard-side cap prevents
+# full buffering into RAM before we reach rag.
+_MAX_INGEST_BYTES = int(
+    os.environ.get("DASHBOARD_MAX_INGEST_BYTES", str(256 * 1024 * 1024))
+)
+# Stream chunk size — 64 KB is plenty for network forwarding and keeps
+# per-request memory predictable.
+_STREAM_CHUNK = 64 * 1024
+
+
+def _cfg(request: Request) -> DashboardConfig:
+    return request.app.state.config
+
+
+async def _proxy_json(
+    request: Request,
+    method: str,
+    path: str,
+    json_body: dict | None = None,
+    timeout: float = _KNOWLEDGE_TIMEOUT,
+):
+    """Forward a JSON call to rag with the dashboard's bearer key.
+    Normalises common failure modes into FastAPI HTTPException so the
+    frontend sees uniform `{error: ...}` bodies."""
+    cfg = _cfg(request)
+    _require_auth(request, cfg)
+    key = cfg.read_api_key()
+    try:
+        async with httpx.AsyncClient(timeout=timeout) as client:
+            r = await client.request(
+                method,
+                f"{cfg.lens_url}{path}",
+                headers={"Authorization": f"Bearer {key}"},
+                json=json_body,
+            )
+    except httpx.ConnectError:
+        raise HTTPException(
+            status_code=502,
+            detail=f"rag server not reachable at {cfg.lens_url}",
+        )
+    except httpx.RequestError as e:
+        raise HTTPException(status_code=502, detail=f"rag request failed: {e}")
+    if r.status_code >= 400:
+        # Bubble rag's error body (already JSON with an `error` key per
+        # rag's exception handler) with its status code.
+        try:
+            body = r.json()
+        except ValueError:
+            body = {"error": r.text or f"rag returned {r.status_code}"}
+        raise HTTPException(status_code=r.status_code, detail=body.get("error") or body)
+    return r.json() if r.content else {}
+
+
+def register(app: FastAPI) -> None:
+    router = APIRouter(prefix="/api/knowledge", tags=["knowledge"])
+
+    @router.get("/libraries")
+    async def list_libraries(request: Request):
+        return await _proxy_json(request, "GET", "/libraries")
+
+    @router.post("/libraries")
+    async def create_library(request: Request, body: dict):
+        # Forward the body verbatim — rag's LibraryCreateRequest handles
+        # validation. Wrapping with our own pydantic model would add no
+        # value and would drift when rag changes its schema.
+        return await _proxy_json(request, "POST", "/libraries", json_body=body)
+
+    @router.post("/libraries/{library_id}/activate")
+    async def activate_library(request: Request, library_id: str):
+        return await _proxy_json(
+            request, "POST", f"/libraries/{library_id}/activate"
+        )
+
+    @router.patch("/libraries/{library_id}")
+    async def rename_library(request: Request, library_id: str, body: dict):
+        return await _proxy_json(
+            request, "PATCH", f"/libraries/{library_id}", json_body=body
+        )
+
+    @router.delete("/libraries/{library_id}")
+    async def delete_library(request: Request, library_id: str):
+        return await _proxy_json(request, "DELETE", f"/libraries/{library_id}")
+
+    @router.post("/search")
+    async def search(request: Request, body: dict):
+        """Proxy to rag's /query. Frontend sends {query, top_k}; we stitch
+        on the protocol version so the UI stays simpler."""
+        payload = {
+            "version": 1,
+            "query": body.get("query", ""),
+            "top_k": int(body.get("top_k", 5)),
+        }
+        return await _proxy_json(request, "POST", "/query", json_body=payload)
+
+    @router.get("/stats")
+    async def stats(request: Request):
+        return await _proxy_json(request, "GET", "/stats")
+
+    @router.get("/info")
+    async def info(request: Request):
+        """Proxy rag's /info for the Knowledge tab's engine badge."""
+        return await _proxy_json(request, "GET", "/info")
+
+    @router.get("/models")
+    async def models(request: Request):
+        """Proxy rag's curated embedding-model list for the create-library
+        picker. Lets the dashboard render the dropdown without hard-coding
+        model ids + dims — if rag adds new ones, they appear automatically."""
+        return await _proxy_json(request, "GET", "/models")
+
+    @router.delete("/sources")
+    async def clear_sources(request: Request):
+        return await _proxy_json(request, "DELETE", "/sources")
+
+    @router.delete("/sources/{source:path}")
+    async def delete_source(request: Request, source: str):
+        return await _proxy_json(request, "DELETE", f"/sources/{source}")
+
+    @router.post("/ingest")
+    async def ingest(
+        request: Request,
+        files: list[UploadFile] = File(...),
+    ):
+        """Forward a multipart upload to rag's /ingest.
+
+        Streams the upload to a temp file in chunks, enforcing a byte
+        cap as we read — keeps the dashboard from buffering multi-GB
+        uploads into RAM while letting rag see the size quickly.
+        Filenames are basename-sanitised at the dashboard layer too.
+
+        When the browser sends `Accept: application/x-ndjson`, we open
+        a streaming request to rag and pipe each progress line through
+        as it arrives. Otherwise: single-shot JSON.
+        """
+        cfg = _cfg(request)
+        _require_auth(request, cfg)
+        key = cfg.read_api_key()
+
+        if not files:
+            raise HTTPException(status_code=400, detail="No files uploaded")
+
+        # Create tempdir without a `with` block — streaming needs the
+        # directory to outlive the handler return. Cleanup is explicit,
+        # either at end-of-handler (single-shot path) or in the generator's
+        # finally (streaming path).
+        tmpdir = tempfile.mkdtemp(prefix="gbd-ingest-")
+        tmp_path = Path(tmpdir)
+
+        def _cleanup_tmpdir() -> None:
+            import shutil as _shutil
+
+            _shutil.rmtree(tmpdir, ignore_errors=True)
+
+        try:
+            total_bytes = 0
+            saved: list[tuple[str, Path, str]] = []
+
+            for upload in files:
+                raw_name = (upload.filename or "").replace("\\", "/")
+                safe_name = os.path.basename(raw_name)
+                if not safe_name or safe_name in (".", ".."):
+                    continue
+                dest = tmp_path / safe_name
+                with dest.open("wb") as out:
+                    while True:
+                        chunk = await upload.read(_STREAM_CHUNK)
+                        if not chunk:
+                            break
+                        total_bytes += len(chunk)
+                        if total_bytes > _MAX_INGEST_BYTES:
+                            raise HTTPException(
+                                status_code=413,
+                                detail=f"Upload exceeds {_MAX_INGEST_BYTES} bytes",
+                            )
+                        out.write(chunk)
+                saved.append(
+                    (
+                        safe_name,
+                        dest,
+                        upload.content_type or "application/octet-stream",
+                    )
+                )
+
+            if not saved:
+                raise HTTPException(status_code=400, detail="No valid files in upload")
+        except HTTPException:
+            _cleanup_tmpdir()
+            raise
+        except Exception:
+            _cleanup_tmpdir()
+            raise
+
+        accept = (request.headers.get("accept") or "").lower()
+        want_stream = "application/x-ndjson" in accept
+
+        if want_stream:
+            async def _pipe_stream():
+                """Pipe rag's NDJSON stream through to the browser. Each
+                line from rag's `r.aiter_raw()` yields as it arrives —
+                the browser sees progress events live instead of waiting
+                for ingest to complete. Temp dir + file handles are kept
+                alive here and cleaned in the finally."""
+                fhs: list = []
+                multipart: list[tuple[str, tuple[str, object, str]]] = []
+                client = httpx.AsyncClient(timeout=_INGEST_TIMEOUT)
+                try:
+                    for name, disk_path, mime in saved:
+                        fh = disk_path.open("rb")
+                        fhs.append(fh)
+                        multipart.append(("files", (name, fh, mime)))
+
+                    try:
+                        async with client.stream(
+                            "POST",
+                            f"{cfg.lens_url}/ingest",
+                            headers={
+                                "Authorization": f"Bearer {key}",
+                                "Accept": "application/x-ndjson",
+                            },
+                            files=multipart,
+                        ) as r:
+                            if r.status_code >= 400:
+                                body_bytes = b""
+                                async for c in r.aiter_bytes():
+                                    body_bytes += c
+                                    if len(body_bytes) > 16 * 1024:
+                                        break
+                                try:
+                                    body_obj = json.loads(body_bytes.decode())
+                                except Exception:
+                                    body_obj = {
+                                        "error": body_bytes.decode(errors="replace")
+                                        or f"rag returned {r.status_code}"
+                                    }
+                                msg = (
+                                    body_obj.get("error")
+                                    if isinstance(body_obj, dict)
+                                    else str(body_obj)
+                                ) or f"rag returned {r.status_code}"
+                                yield (
+                                    json.dumps(
+                                        {
+                                            "event": "error",
+                                            "message": msg,
+                                            "status": r.status_code,
+                                        }
+                                    )
+                                    + "\n"
+                                ).encode()
+                                return
+                            # Pass through each chunk verbatim — rag
+                            # emits one JSON object per line.
+                            async for c in r.aiter_raw():
+                                if c:
+                                    yield c
+                    except httpx.ConnectError:
+                        yield (
+                            json.dumps(
+                                {
+                                    "event": "error",
+                                    "message": f"rag server not reachable at {cfg.lens_url}",
+                                }
+                            )
+                            + "\n"
+                        ).encode()
+                    except httpx.RequestError as e:
+                        yield (
+                            json.dumps(
+                                {
+                                    "event": "error",
+                                    "message": f"ingest request failed: {e}",
+                                }
+                            )
+                            + "\n"
+                        ).encode()
+                finally:
+                    await client.aclose()
+                    for fh in fhs:
+                        try:
+                            fh.close()
+                        except Exception:
+                            pass
+                    _cleanup_tmpdir()
+
+            return StreamingResponse(
+                _pipe_stream(), media_type="application/x-ndjson"
+            )
+
+        # Default: single-shot JSON path — open handles, POST, close,
+        # return the summary dict.
+        try:
+            fhs: list = []
+            multipart: list[tuple[str, tuple[str, object, str]]] = []
+            try:
+                for name, disk_path, mime in saved:
+                    fh = disk_path.open("rb")
+                    fhs.append(fh)
+                    multipart.append(("files", (name, fh, mime)))
+
+                try:
+                    async with httpx.AsyncClient(timeout=_INGEST_TIMEOUT) as client:
+                        r = await client.post(
+                            f"{cfg.lens_url}/ingest",
+                            headers={"Authorization": f"Bearer {key}"},
+                            files=multipart,
+                        )
+                except httpx.ConnectError:
+                    raise HTTPException(
+                        status_code=502,
+                        detail=f"rag server not reachable at {cfg.lens_url}",
+                    )
+                except httpx.RequestError as e:
+                    raise HTTPException(
+                        status_code=502, detail=f"ingest request failed: {e}"
+                    )
+            finally:
+                for fh in fhs:
+                    try:
+                        fh.close()
+                    except Exception:
+                        pass
+
+            if r.status_code >= 400:
+                try:
+                    body = r.json()
+                except ValueError:
+                    body = {"error": r.text or f"rag returned {r.status_code}"}
+                raise HTTPException(
+                    status_code=r.status_code, detail=body.get("error") or body
+                )
+            return r.json() if r.content else {}
+        finally:
+            _cleanup_tmpdir()
+
+    app.include_router(router)