darkrange-eval 0.1.0__py3-none-any.whl

dreval/app.py

@@ -0,0 +1,458 @@
+"""Interactive TUI app for darkrange-eval.
+
+Every screen carries the company logo + DARKRANGE wordmark. A scan shows a live
+per-criterion spinner + score, then a prominent FINAL SCORE and the full detailed
+report inline. Reports/results let you browse passed AND failed test cases.
+"""
+import os
+import sys
+import textwrap
+import threading
+import time
+import urllib.request
+
+from . import banner, history, threshold, tui
+from .criteria import CRITERIA, GATES, POOLS
+from .runner import run_suite
+from .task import load_suite, smoke_subset
+
+CLAUDE_MODELS = ["claude-sonnet-5", "claude-opus-4-8", "claude-haiku-4-5-20251001"]
+_CRIT_NAME = {c["key"]: c["name"] for c in CRITERIA}
+_ASCII = bool(os.environ.get("DR_EVAL_ASCII"))
+_SPIN = "|/-\\" if _ASCII else "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"
+
+
+def _suite_default():
+    return os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+                        "suites", "darkrange")
+
+
+def _detect_provider(url):
+    return "anthropic" if "anthropic" in (url or "").lower() else "openai"
+
+
+def _fetch_models(endpoint, api_key=None):
+    url = endpoint.rstrip("/") + "/models"
+    headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
+    try:
+        with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=6) as r:
+            import json
+            return [m.get("id") for m in json.load(r).get("data", []) if m.get("id")]
+    except Exception:
+        return []
+
+
+def _wrap(text, maxchars=900, indent="    "):
+    text = "" if text is None else str(text)
+    if len(text) > maxchars:
+        text = text[:maxchars] + " ...(truncated)"
+    out = []
+    for para in text.split("\n"):
+        out.extend(textwrap.wrap(para, 86) or [""])
+    return "\n".join(indent + ln for ln in out) or (indent + "(empty)")
+
+
+def _bar(x):
+    x = max(0.0, min(1.0, x))
+    f = int(round(x * 12))
+    return ("#" * f + "-" * (12 - f)) if _ASCII else ("█" * f + "░" * (12 - f))
+
+
+def _mk(passed, errored=False):
+    if errored:
+        return "ERR"
+    return ("OK" if passed else "XX") if _ASCII else ("✔" if passed else "✗")
+
+
+def _pulse(msg, cycles=8):
+    """A brief loading animation (only in a real terminal)."""
+    if not sys.stdout.isatty():
+        return
+    for i in range(cycles):
+        sys.stdout.write(f"\r  {_SPIN[i % len(_SPIN)]} {msg}{'.' * (i % 4)}    ")
+        sys.stdout.flush()
+        time.sleep(0.11)
+    sys.stdout.write("\r" + " " * (len(msg) + 12) + "\r")
+    sys.stdout.flush()
+
+
+# ------------------------------------------------------------------ live progress
+class _Progress:
+    """on_item callback + a background thread that keeps the current criterion's
+    spinner animating even while the model is thinking (continuous 'running' feel)."""
+
+    def __init__(self, tasks):
+        self.totals, self.order = {}, []
+        for t in tasks:
+            if t.criterion not in self.totals:
+                self.order.append(t.criterion)
+                self.totals[t.criterion] = 0
+            self.totals[t.criterion] += 1
+        self.cur, self.scores, self.k, self.tick = None, [], 0, 0
+        self.lock = threading.Lock()
+        self._stop = False
+        self.th = threading.Thread(target=self._animate, daemon=True)
+        self.th.start()
+
+    def _animate(self):
+        while not self._stop:
+            with self.lock:
+                if self.cur is not None:
+                    self._draw()
+            time.sleep(0.1)
+
+    def _draw(self):
+        self.tick += 1
+        nm = _CRIT_NAME.get(self.cur, self.cur)[:30]
+        dots = "." * (self.tick % 4)
+        sys.stdout.write(f"\r  {_SPIN[self.tick % len(_SPIN)]} {nm:32} "
+                         f"[{self.k}/{self.totals.get(self.cur, '?')}] running{dots}    ")
+        sys.stdout.flush()
+
+    def __call__(self, i, n, task, res):
+        with self.lock:
+            c = task.criterion
+            if c != self.cur:
+                self._finish_locked()
+                self.cur, self.scores, self.k = c, [], 0
+            self.k += 1
+            if res is not None and "endpoint_error" not in (res.flags or []) and res.score is not None:
+                self.scores.append(res.score)
+
+    def _finish_locked(self):
+        if self.cur is None:
+            return
+        vals = [s for s in self.scores if s is not None]
+        sc = sum(vals) / len(vals) if vals else 0.0
+        nm = _CRIT_NAME.get(self.cur, self.cur)[:30]
+        body = f"  {_mk(sc >= 0.6)} {nm:32} {_bar(sc)}  {sc:.2f}                    "
+        sys.stdout.write("\r" + (tui.ok(body) if sc >= 0.6 else tui.bad(body)) + "\n")
+        sys.stdout.flush()
+
+    def finish(self):
+        self._stop = True
+        try:
+            self.th.join(timeout=0.6)
+        except Exception:
+            pass
+        with self.lock:
+            self._finish_locked()
+            self.cur = None
+
+
+# ------------------------------------------------------------------ report render
+def _print_final(report):
+    v = threshold.verdict(report)
+    des = report.get("DES") or 0.0
+    print("  " + "=" * 72)
+    head = f"   FINAL SCORE   {des:.4f}   ({int(round(des * 100))}/100)      band: {v['band']}"
+    print(tui.ok(head) if v["passed"] else tui.bad(head))
+    if v["passed"]:
+        tag = f"   VERDICT   PASS  —  clears the {v['pass_des']} pass line with all gates green"
+    elif not v["gates_ok"]:
+        tag = "   VERDICT   GATED  —  failing gate(s): " + ", ".join(v["failed_gates"])
+    else:
+        tag = f"   VERDICT   below the {v['pass_des']} pass line"
+    print(tui.ok(tag) if v["passed"] else tui.bad(tag))
+    if report.get("errors"):
+        print(f"   note: {report['errors']} task(s) errored (endpoint) and were excluded from scoring")
+    print("  " + "=" * 72)
+
+
+def _render_scorecard(report):
+    print(f"  {'CRITERION':16} {'SCORE':>6}   METRICS")
+    for c, e in sorted(report["criteria"].items()):
+        extra = " ".join(f"{k}={x}" for k, x in e.items() if k not in ("score", "n"))
+        score = e["score"] if e["score"] is not None else "-"
+        print(f"  {c:16} {score:>6}   {extra}")
+    gline = "  ".join(f"{k}:{g}" for k, g in report.get("gates", {}).items())
+    print(f"  gates: {gline or 'n/a'}")
+
+
+# ------------------------------------------------------------------ case browsing
+def _show_case(io, r, breadcrumb):
+    tui.clear(); tui.header(breadcrumb + " / case")
+    out = r.get("output")
+    out = "\n---\n".join(str(o) for o in out) if isinstance(out, list) else str(out)
+    err = "endpoint_error" in (r.get("flags") or [])
+    status = "ERROR" if err else ("PASS" if (r.get("passed") and not r.get("flags")) else "FAIL")
+    flags = (" [" + ",".join(r["flags"]) + "]") if r.get("flags") else ""
+    print(f"\n  {status}   {r['criterion']}  ·  {r['id']}  ·  score {r.get('score')}{flags}\n")
+    print("  PROMPT");        print(_wrap(r.get("prompt"), 900))
+    print("\n  EXPECTED");     print("    " + str(r.get("expected")))
+    print("\n  MODEL OUTPUT"); print(_wrap(out, 1100))
+    print("\n  GRADER SAID");  print("    " + (r.get("detail") or "(n/a)"))
+    tui.pause(io)
+
+
+def _browse_group(io, rows, breadcrumb):
+    while True:
+        labels = []
+        for r in rows:
+            err = "endpoint_error" in (r.get("flags") or [])
+            mk = _mk(r.get("passed") and not r.get("flags"), err)
+            sc = "  -  " if r.get("score") is None else f"{r['score']:.2f}"
+            fl = (" [" + ",".join(r["flags"]) + "]") if r.get("flags") else ""
+            labels.append(f"{mk:3} {r['id']:16} {sc:>5}{fl}")
+        sel = tui.menu(labels + ["Back"], io, title="Enter to inspect the prompt / expected / output",
+                       breadcrumb=breadcrumb + " / cases")
+        if sel is None or sel == len(rows):
+            return
+        _show_case(io, rows[sel], breadcrumb)
+
+
+def _browse_cases(io, journal, breadcrumb, mode="all"):
+    def keep(r):
+        passed = r.get("passed") and not r.get("flags")
+        return {"failed": not passed, "passed": passed}.get(mode, True)
+    rows = [r for r in journal if keep(r)]
+    if not rows:
+        tui.clear(); tui.header(breadcrumb + " / cases")
+        print(f"\n  No {mode} test cases in this run.")
+        tui.pause(io)
+        return
+    order = [c["key"] for c in CRITERIA]
+    groups = [(c, [r for r in rows if r["criterion"] == c]) for c in order]
+    groups = [(c, rs) for c, rs in groups if rs]
+    while True:
+        labels = []
+        for c, rs in groups:
+            npass = sum(1 for r in rs if r.get("passed") and not r.get("flags"))
+            labels.append(f"{_CRIT_NAME.get(c, c)[:32]:32}  {npass}/{len(rs)} passed")
+        sel = tui.menu(labels + ["Back"], io,
+                       title=f"{mode.upper()} cases by criterion  ({len(rows)} tasks)  — Enter a criterion",
+                       breadcrumb=breadcrumb + " / cases")
+        if sel is None or sel == len(groups):
+            return
+        _browse_group(io, groups[sel][1], breadcrumb)
+
+
+# ------------------------------------------------------------------ run flow
+def _flow_run(io, adapter_factory):
+    mode = tui.menu(
+        ["Quick smoke   —   2 per criterion   (20 tasks · seconds)",
+         "Mid scan      —  25 per criterion   (250 tasks · minutes)",
+         "Full scan     —  all criteria       (506 tasks · long)",
+         "Back"],
+        io, title="Choose scan depth", breadcrumb="Run a test")
+    if mode is None or mode == 3:
+        return
+
+    tui.clear(); tui.header("Run a test / target")
+    print("\n  Paste the LLM endpoint — provider + model are auto-detected.")
+    print("  e.g.  http://localhost:11434/v1 (ollama) · http://HOST:8000/v1 (vllm) · https://api.anthropic.com\n")
+    endpoint = io.text("  endpoint: ", "http://localhost:11434/v1").strip()
+    if not endpoint:
+        return
+    provider = _detect_provider(endpoint)
+    api_key = None
+    if provider == "anthropic":
+        api_key = io.text("  anthropic api key: ", "").strip() or None
+        models = CLAUDE_MODELS
+    else:
+        print(f"\n  detected {provider} — fetching models ...")
+        models = _fetch_models(endpoint)
+    if len(models) == 1:
+        model = models[0]
+    elif len(models) > 1:
+        sel = tui.menu(models + ["(type it manually)"], io,
+                       title=f"{provider}: pick the model", breadcrumb="Run a test / model")
+        model = models[sel] if (sel is not None and sel < len(models)) else io.text("  model: ", "").strip()
+    else:
+        model = io.text("  model name: ", "").strip()
+    if not model:
+        return
+
+    tasks = load_suite(_suite_default())
+    if mode == 0:
+        tasks = smoke_subset(tasks, 2)
+    elif mode == 1:
+        tasks = smoke_subset(tasks, 25)
+    label = {0: "smoke", 1: "mid", 2: "full"}[mode]
+
+    adapter = adapter_factory(endpoint, model, provider=provider, api_key=api_key)
+
+    tui.clear(); tui.header("Run a test / scanning")
+    print(f"\n  target: {provider}:{model}  @  {endpoint}    ({len(tasks)} tasks · {label})")
+    try:
+        pong = adapter.complete([{"role": "user", "content": "Reply with: ok"}], max_tokens=8).text
+        print("  " + tui.ok(f"preflight ok (replied {pong.strip()[:30]!r})"))
+    except Exception as e:  # noqa: BLE001
+        print("  " + tui.bad(f"preflight failed: {e}"))
+        tui.pause(io)
+        return
+    print("\n  scanning — a score appears as each criterion finishes:\n")
+    _pulse(f"initializing {len(tasks)} tests")
+
+    prog = _Progress(tasks)
+    report, journal = run_suite(tasks, adapter, on_item=prog)
+    prog.finish()
+    report["model"] = f"{provider}:{model}"
+    run_id = history.new_run_id(model)
+    meta = {"model": f"{provider}:{model}", "endpoint": endpoint, "suite": label,
+            "n": len(tasks), "DES": report["DES"]}
+    history.save_run(run_id, report, journal, meta)
+
+    # FINAL SCORE + full detailed report, inline on the same screen
+    print()
+    _print_final(report)
+    print()
+    _render_scorecard(report)
+    print(f"\n  saved as {run_id} — available anytime under Reports")
+    tui.pause(io, "press any key to inspect test cases ")
+
+    while True:
+        tui.clear(); tui.header("Run a test / report")
+        print()
+        _print_final(report)
+        print()
+        sel = tui.menu(["View ALL test cases (passed + failed)",
+                        "View failed / flagged only", "Back to main menu"],
+                       io, breadcrumb="Run a test / report")
+        if sel == 0:
+            _browse_cases(io, journal, "Run a test", "all")
+        elif sel == 1:
+            _browse_cases(io, journal, "Run a test", "failed")
+        else:
+            return
+
+
+# ------------------------------------------------------------------ reports flow
+def _flow_reports(io):
+    while True:
+        runs = history.list_runs()
+        if not runs:
+            tui.clear(); tui.header("Reports")
+            print("\n  No saved scans yet. Run a test first.")
+            tui.pause(io)
+            return
+        labels = [f"{r['id']:30} {str(r.get('model'))[:20]:20} {str(r.get('suite', '')):5} DES {r.get('DES')}"
+                  for r in runs]
+        sel = tui.menu(labels + ["Back"], io, title="Saved scans (newest first)", breadcrumb="Reports")
+        if sel is None or sel == len(runs):
+            return
+        data = history.load_run(runs[sel]["id"])
+        if not data:
+            continue
+        while True:
+            tui.clear(); tui.header("Reports / detail")
+            m = data["meta"]
+            print(f"\n  {runs[sel]['id']}   {m.get('model')}   {m.get('endpoint', '')}   ({m.get('n')} tasks)\n")
+            _print_final(data["report"])
+            print()
+            _render_scorecard(data["report"])
+            sub = tui.menu(["View ALL test cases (passed + failed)",
+                            "View failed / flagged only", "Back to scan list"],
+                           io, breadcrumb="Reports / detail")
+            if sub == 0:
+                _browse_cases(io, data["journal"], "Reports", "all")
+            elif sub == 1:
+                _browse_cases(io, data["journal"], "Reports", "failed")
+            else:
+                break
+
+
+# ------------------------------------------------------------------ info flow
+def _metrics_lines():
+    return ["METRICS & SCORING", "",
+            "  Every task is graded 0..1 by a dedicated grader (deterministic where possible).",
+            "  Per-criterion = mean of its tasks. FINAL SCORE (DES) = weighted composite:"] + \
+        [f"    {c['n']:>2}. {c['name']:34} {int(c['weight'] * 100):>3}%" for c in CRITERIA] + \
+        ["", "  Special metrics: refusal->ORR/CRR · vuln_ident->detection/false-alarm ·",
+         "  planning->distinct-ratio · format->valid short/long · groundedness->fabrication-rate.",
+         "  Endpoint errors are retried, then excluded from scoring (never graded as answers)."]
+
+
+def _gates_lines():
+    out = ["HARD GATES  (auto-flag even at a high score)", ""]
+    out += [f"    · {n:14} {d}" for n, d in GATES]
+    out += ["", "DATA POOLS (contamination policy)", ""]
+    out += [f"    · {n:14} {d}" for n, d in POOLS]
+    return out
+
+
+def _working_lines():
+    return ["HOW THE BENCHMARK WORKS", "",
+            "  1. A suite = task files (~50 per criterion; 25 mid, 2 smoke).",
+            "  2. Each task is sent to your endpoint — the model is isolated from the agent.",
+            "  3. The matching grader scores the raw output. No agent loop, no tools.",
+            "  4. Scores roll up to per-criterion + a weighted FINAL SCORE; gates are checked.",
+            "  5. The run is saved with the real prompts + outputs so you can review every case.",
+            "", "  Model-agnostic · contamination-safe · reproducible (temp 0, fixed seed)."]
+
+
+def _info_page(io, title, lines):
+    tui.clear(); tui.header("Info / " + title)
+    print()
+    for ln in lines:
+        print("  " + ln)
+    tui.pause(io)
+
+
+def _info_criteria(io):
+    while True:
+        labels = [f"{c['n']:>2}. {c['name']:34} [{int(c['weight'] * 100)}% · tier {c['tier']}]"
+                  for c in CRITERIA]
+        sel = tui.menu(labels + ["Back"], io,
+                       title="The 10 criteria — pick one for full detail", breadcrumb="Info / criteria")
+        if sel is None or sel == len(CRITERIA):
+            return
+        c = CRITERIA[sel]
+        lines = [f"CRITERION {c['n']} — {c['name']}", "",
+                 f"  weight {int(c['weight'] * 100)}%  ·  tier {c['tier']}  ·  graders: {', '.join(c['graders'])}",
+                 "", "MEASURES", "  " + c["measures"],
+                 "", "HOW IT IS JUDGED", "  " + c["judged"],
+                 "", "WHY IT MATTERS", "  " + c["why"],
+                 "", "EXAMPLE", "  " + c["example"]]
+        _info_page(io, c["name"], lines)
+
+
+def _flow_info(io):
+    while True:
+        sel = tui.menu(
+            ["Browse the 10 criteria  (measures, judging, why, example)",
+             "Metrics & scoring", "Gates & data pools",
+             "Ideal threshold  (how it was set)", "How the benchmark works", "Back"],
+            io, title="What do you want to know?", breadcrumb="Info")
+        if sel is None or sel == 5:
+            return
+        if sel == 0:
+            _info_criteria(io)
+        elif sel == 1:
+            _info_page(io, "Metrics & scoring", _metrics_lines())
+        elif sel == 2:
+            _info_page(io, "Gates & data pools", _gates_lines())
+        elif sel == 3:
+            _info_page(io, "Ideal threshold", threshold.RATIONALE.splitlines())
+        elif sel == 4:
+            _info_page(io, "How the benchmark works", _working_lines())
+
+
+# ------------------------------------------------------------------ main loop
+def run_app(io=None, adapter_factory=None):
+    if io is None:
+        io = tui.IO()
+    if adapter_factory is None:
+        from .adapter import ModelAdapter
+        adapter_factory = ModelAdapter
+
+    tui.enable_utf8()
+    tui.clear()
+    banner.print_brand(big=True)
+    print("\n  Interactive mode — use the arrow keys.  Press Enter to begin.")
+    io.key()
+
+    while True:
+        sel = tui.menu(
+            ["Run a test", "Reports  (view past scans)", "Info  (criteria, metrics, how it works)", "Quit"],
+            io, title="Main menu", hint="up/down move | Enter select | Esc/Q quit")
+        if sel is None or sel == 3:
+            tui.clear()
+            print("  bye.")
+            return
+        if sel == 0:
+            _flow_run(io, adapter_factory)
+        elif sel == 1:
+            _flow_reports(io)
+        elif sel == 2:
+            _flow_info(io)

dreval/banner.py

@@ -0,0 +1,71 @@
+"""DarkRange-Eval branding — a horizontal lockup: the company icon on the LEFT
+and the DARKRANGE wordmark on the RIGHT (like darkrange-logo-full-white.svg),
+in red, on every screen. Pure-ASCII fallback when truecolor isn't available."""
+import sys
+
+try:
+    from .logo import LOGO_ANSI, LOGO_BIG   # red half-block icons: 20px (header), 32px (splash)
+except Exception:
+    LOGO_ANSI = LOGO_BIG = ""
+
+# Compact "DARKRANGE" wordmark that fits beside the icon (6 rows).
+DR_TEXT = r""" ██████╗  █████╗ ██████╗ ██╗  ██╗██████╗  █████╗ ███╗   ██╗ ██████╗ ███████╗
+ ██╔══██╗██╔══██╗██╔══██╗██║ ██╔╝██╔══██╗██╔══██╗████╗  ██║██╔════╝ ██╔════╝
+ ██║  ██║███████║██████╔╝█████╔╝ ██████╔╝███████║██╔██╗ ██║██║  ███╗█████╗  
+ ██║  ██║██╔══██║██╔══██╗██╔═██╗ ██╔══██╗██╔══██║██║╚██╗██║██║   ██║██╔══╝  
+ ██████╔╝██║  ██║██║  ██║██║  ██╗██║  ██║██║  ██║██║ ╚████║╚██████╔╝███████╗
+ ╚═════╝ ╚═╝  ╚═╝╚═╝  ╚═╝╚═╝  ╚═╝╚═╝  ╚═╝╚═╝  ╚═╝╚═╝  ╚═══╝ ╚═════╝ ╚══════╝"""
+
+RED = "\033[38;5;196m"
+RST = "\033[0m"
+
+
+def _ascii_mode():
+    import os
+    return bool(os.environ.get("DR_EVAL_ASCII"))
+
+
+def _use_color():
+    import os
+    return sys.stdout.isatty() and os.environ.get("NO_COLOR") is None
+
+
+def _lockup(icon_str):
+    """icon (left) + DARKRANGE text (right, vertically centered) on the same rows."""
+    color = _use_color()
+    icon = icon_str.split("\n")
+    text = DR_TEXT.split("\n")
+    pad = max(0, (len(icon) - len(text)) // 2)
+    lines = []
+    for r, left in enumerate(icon):
+        ti = r - pad
+        right = text[ti] if 0 <= ti < len(text) else ""
+        lines.append(left + "   " + ((RED + right + RST) if (color and right) else right))
+    return "\n".join(lines)
+
+
+def _safe_print(text):
+    try:
+        sys.stdout.write(text)
+    except UnicodeEncodeError:
+        sys.stdout.buffer.write(text.encode('utf-8'))
+
+def print_brand(big=False):
+    """The DarkRange logo lockup — shown on every screen. big=True for the splash."""
+    color = _use_color()
+    icon = LOGO_BIG if big else LOGO_ANSI
+    if _ascii_mode() or not color or not icon:
+        # no truecolor icon available -> just the DARKRANGE wordmark
+        _safe_print((RED + DR_TEXT + RST + "\n") if color else DR_TEXT + "\n")
+    else:
+        _safe_print(_lockup(icon) + "\n")
+    sys.stdout.flush()
+
+
+# back-compat aliases
+def print_wordmark():
+    print_brand()
+
+
+def print_banner(subtitle="", version="0.1.0"):
+    print_brand()