wl-benchmark 0.4.0__py3-none-any.whl

wl_benchmark/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

wl_benchmark/cli.py

@@ -0,0 +1,250 @@
+"""CLI entry: wl-bench {run|list-tasks|report|doctor}
+
+`run` / `doctor` interactively ask for the target under test
+(endpoint / key / model) every time — the tool stores no provider presets.
+Use --endpoint/--key/--model flags to skip the interactive prompts.
+
+Run as:  ./wl-bench <cmd>   |   python3 -m wl_benchmark <cmd>
+         |   wl-bench <cmd> (after pip install)
+"""
+from __future__ import annotations
+
+import argparse
+import getpass
+import json
+import os
+import sys
+import urllib.request
+
+from .publisher import (cleanup_run, load_site_config, prompt_site_config,
+                        publish_run)
+from .reporter import write_report
+from .review_pdf import build_review_pdf
+from .runner import run_all
+from .tasks import TASK_TYPES, build_tasks
+
+PKG_DIR = os.path.dirname(os.path.abspath(__file__))
+DEFAULT_TASKS_ROOT = os.path.join(PKG_DIR, "tasks_data")
+DEFAULT_CONFIG = os.path.join("config", "bench.json")   # optional run params
+VERSION = "0.4.0"
+BRAND = "WL-Benchmark"
+
+
+# ------------------------------------------------------- interactive input
+def _list_models(base_url: str, api_key: str) -> list:
+    try:
+        req = urllib.request.Request(
+            base_url.rstrip("/") + "/models",
+            headers={"Authorization": f"Bearer {api_key}"})
+        with urllib.request.urlopen(req, timeout=15) as r:
+            return [m.get("id") for m in json.load(r).get("data", []) if m.get("id")]
+    except Exception:
+        return []
+
+
+def _prompt_provider(args) -> dict:
+    """Interactively collect endpoint / key / model.
+    --endpoint/--key/--model flags skip the matching prompt."""
+    print(f"{BRAND} — target under test (nothing is stored)")
+    base_url = (args.endpoint or "").strip()
+    while not base_url.startswith(("http://", "https://")):
+        base_url = input("Endpoint (OpenAI-compatible, e.g. "
+                         "https://api.example.com/v1): ").strip()
+
+    api_key = (args.key or "").strip()
+    while not api_key:
+        api_key = getpass.getpass("API key (input hidden): ").strip()
+
+    model = (args.model or "").strip()
+    if not model:
+        models = _list_models(base_url, api_key)
+        if models:
+            print(f"The endpoint offers {len(models)} models:")
+            for i, m in enumerate(models, 1):
+                print(f"  {i:2d}. {m}")
+            sel = input("Pick a number, or type a model name: ").strip()
+            if sel.isdigit() and 1 <= int(sel) <= len(models):
+                model = models[int(sel) - 1]
+            elif sel:
+                model = sel
+    while not model:
+        model = input("Model: ").strip()
+
+    return {"name": model, "base_url": base_url, "api_key": api_key,
+            "model": model}
+
+
+def _load_run_cfg(path: str) -> dict:
+    """Optional run-parameter overrides; sensible defaults otherwise."""
+    cfg = {"tasks_data_root": DEFAULT_TASKS_ROOT,
+           "rubric_modality": "auto",
+           "max_tokens": 2048, "essay_max_tokens": 4096,
+           "svg_max_tokens": 16384, "scheduling_max_tokens": 4096,
+           "scheduling_max_turns": 16, "quant_max_turns": 16,
+           "quant_max_tokens": 8192,
+           "temperature": 0.2, "timeout": 300}
+    if path and os.path.exists(path):
+        with open(path, encoding="utf-8") as f:
+            raw = json.load(f)
+        cfg["tasks_data_root"] = raw.get("tasks_data_root",
+                                         cfg["tasks_data_root"])
+        cfg.update(raw.get("run", {}))
+    return cfg
+
+
+def cmd_run(args) -> None:
+    provider = _prompt_provider(args)
+    run_cfg = _load_run_cfg(args.config)
+    out_dir = run_all(provider, run_cfg,
+                      only_types=args.tasks.split(",") if args.tasks else None,
+                      out_root=args.out)
+    summary = write_report(out_dir)
+    print(f"[cli] summary -> {summary}")
+    _maybe_publish(out_dir, keep=args.keep, skip=args.no_upload)
+
+
+def _maybe_publish(run_dir: str, keep: bool = False, skip: bool = False) -> None:
+    """Default flow: upload the run to the benchmark site, then wipe the
+    local data. Falls back to keeping everything if upload is skipped or
+    fails (a later `wlb publish <run-dir>` can retry)."""
+    run_id = os.path.basename(run_dir.rstrip("/"))
+    if skip:
+        print(f"[cli] upload skipped (--no-upload); local data kept at {run_dir}")
+        return
+    cfg = load_site_config()
+    if cfg is None and sys.stdin.isatty():
+        cfg = prompt_site_config()
+    if cfg is None:
+        print(f"[cli] site upload not configured — local data kept at {run_dir}")
+        print("      (set WL_BENCH_URL / WL_BENCH_TOKEN, or delete the")
+        print("       run dir manually)")
+        return
+    try:
+        url = publish_run(run_dir, cfg)
+    except Exception as e:  # noqa: BLE001
+        print(f"[cli] UPLOAD FAILED — local data kept at {run_dir}")
+        print(f"      {e}")
+        print("      retry later with: wlb publish " + run_dir)
+        return
+    print(f"[cli] published -> {url}")
+    print(f"[cli] share link: {url}")
+    if keep:
+        print(f"[cli] local data kept (--keep): {run_dir}")
+    elif cleanup_run(run_dir):
+        print(f"[cli] local run data deleted: {run_dir}")
+
+
+def cmd_list_tasks(args) -> None:
+    run_cfg = _load_run_cfg(args.config)
+    tasks = build_tasks(run_cfg.get("tasks_data_root", DEFAULT_TASKS_ROOT),
+                        run_cfg, artifacts_root="/tmp/wl-bench-list")
+    print(f"{len(tasks)} task(s):")
+    for t in tasks:
+        extra = ""
+        if t.task_type == "essay":
+            extra = f"  rubric={os.path.basename(t.spec.get('rubric') or '-')}"
+        elif t.task_type == "svg":
+            extra = f"  stage={t.spec.get('stage')}"
+        print(f"  [{t.task_type:10s}] {t.task_id}{extra}")
+
+
+def cmd_publish(args) -> None:
+    cfg = load_site_config()
+    if cfg is None:
+        cfg = prompt_site_config()
+    if cfg is None:
+        print("[publish] no Cloudflare configuration — nothing uploaded")
+        return
+    url = publish_run(args.run_dir, cfg)
+    print(f"[publish] {url}")
+    if args.keep:
+        print(f"[publish] local data kept: {args.run_dir}")
+    elif cleanup_run(args.run_dir):
+        print(f"[publish] local run data deleted: {args.run_dir}")
+
+
+def cmd_report(args) -> None:
+    write_report(args.run_dir)
+    pdf = build_review_pdf(args.run_dir)
+    print(f"summary  -> {os.path.relpath(os.path.join(args.run_dir, 'summary.md'))}")
+    print(f"review   -> {pdf}")
+
+
+def cmd_doctor(args) -> None:
+    """Connectivity check: interactive too; runs no tasks, stores nothing."""
+    provider = _prompt_provider(args)
+    models = _list_models(provider["base_url"], provider["api_key"])
+    print(f"\nendpoint : {provider['base_url']}")
+    print(f"model    : {provider['model']}")
+    if models:
+        found = provider["model"] in models
+        print(f"reachable: YES, {len(models)} models; "
+              f"model {'found' if found else 'NOT in list (may still work)'}")
+    else:
+        print("reachable: /models unavailable (may still work for chat)")
+
+    run_cfg = _load_run_cfg(args.config)
+    tasks = build_tasks(run_cfg.get("tasks_data_root", DEFAULT_TASKS_ROOT),
+                        run_cfg, artifacts_root="/tmp/wl-bench-doctor")
+    print(f"tasks    : {len(tasks)} loaded "
+          f"({', '.join(sorted({t.task_type for t in tasks}))})")
+    print("review   : essay/svg are graded by humans "
+          "(artifacts under results/<run>/artifacts/)")
+    print(f"\nverdict: {'READY' if models else 'READY (unverified)'} "
+          f"— run `wlb run` to start")
+
+
+def main(argv=None) -> None:
+    p = argparse.ArgumentParser(
+        prog="wlb",
+        description=f"{BRAND} — one provider, one model, "
+                    "interactive target input")
+    p.add_argument("--config", default=DEFAULT_CONFIG,
+                   help="optional run-parameter JSON "
+                        "(default config/bench.json; may not exist)")
+    p.add_argument("-V", "--version", action="version",
+                   version=f"{BRAND} {VERSION} (wl-benchmark)")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    r = sub.add_parser("run",
+                       help="ask for endpoint/key/model, then run all tasks")
+    r.add_argument("--endpoint", help="skip prompt: OpenAI-compatible base_url")
+    r.add_argument("--key", help="skip prompt: API key")
+    r.add_argument("--model", help="skip prompt: model name")
+    r.add_argument("--tasks", help=f"comma list of {TASK_TYPES}")
+    r.add_argument("--out", default="results")
+    r.add_argument("--no-upload", action="store_true",
+                   help="do not upload to the benchmark site")
+    r.add_argument("--keep", action="store_true",
+                   help="keep local run data even after a successful upload")
+    r.set_defaults(fn=cmd_run)
+
+    l = sub.add_parser("list-tasks", help="list discovered tasks")
+    l.set_defaults(fn=cmd_list_tasks)
+
+    pb = sub.add_parser("publish",
+                        help="upload a run dir to the benchmark site "
+                             "(then delete it locally)")
+    pb.add_argument("run_dir")
+    pb.add_argument("--keep", action="store_true",
+                    help="keep the local run dir after upload")
+    pb.set_defaults(fn=cmd_publish)
+
+    rp = sub.add_parser("report",
+                        help="rebuild summary.md + review.pdf for a run dir")
+    rp.add_argument("run_dir")
+    rp.set_defaults(fn=cmd_report)
+
+    d = sub.add_parser("doctor",
+                       help="connectivity check (interactive, runs nothing)")
+    d.add_argument("--endpoint")
+    d.add_argument("--key")
+    d.add_argument("--model")
+    d.set_defaults(fn=cmd_doctor)
+
+    args = p.parse_args(argv)
+    args.fn(args)
+
+
+if __name__ == "__main__":
+    main()

wl_benchmark/client.py

@@ -0,0 +1,166 @@
+"""OpenAI-compatible chat client (stdlib only, no external deps).
+
+Supports: tool calling, multimodal content (text / image_url / file),
+retry with backoff, latency & usage recording.
+"""
+from __future__ import annotations
+
+import base64
+import json
+import time
+import urllib.error
+import urllib.request
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class ChatResult:
+    content: Optional[str] = None
+    tool_calls: List[Dict[str, Any]] = field(default_factory=list)
+    finish_reason: Optional[str] = None
+    usage: Dict[str, Any] = field(default_factory=dict)
+    latency: float = 0.0
+    raw: Dict[str, Any] = field(default_factory=dict)
+    error: Optional[str] = None
+
+    @property
+    def ok(self) -> bool:
+        return self.error is None
+
+
+class ChatClient:
+    """Minimal OpenAI-compatible /v1/chat/completions client.
+
+    proxy: "direct" (default; bypass system/env proxies — typical endpoints
+           here are reachable directly)
+         | "system" (follow system/env proxies) | "http://host:port"
+    """
+
+    RETRYABLE = (429, 500, 502, 503, 504)
+
+    def __init__(self, base_url: str, api_key: str, timeout: int = 180,
+                 max_retries: int = 3, proxy: str = "direct"):
+        self.base_url = base_url.rstrip("/")
+        self.api_key = api_key
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.proxy = proxy
+        if proxy == "direct":
+            # bypass macOS system proxy / env-var proxies (urllib reads them by default)
+            self._opener = urllib.request.build_opener(
+                urllib.request.ProxyHandler({}))
+        elif proxy == "system":
+            self._opener = urllib.request.build_opener()
+        else:
+            self._opener = urllib.request.build_opener(
+                urllib.request.ProxyHandler({"http": proxy, "https": proxy}))
+
+    # -------------------------------------------------------------- helpers
+    @staticmethod
+    def image_part(path_or_dataurl: str) -> Dict[str, Any]:
+        """Build an image_url content part from a local file or data URL."""
+        if path_or_dataurl.startswith(("http://", "https://", "data:")):
+            url = path_or_dataurl
+        else:
+            with open(path_or_dataurl, "rb") as f:
+                b64 = base64.b64encode(f.read()).decode()
+            mime = "image/png" if path_or_dataurl.endswith(".png") else "image/jpeg"
+            url = f"data:{mime};base64,{b64}"
+        return {"type": "image_url", "image_url": {"url": url}}
+
+    @staticmethod
+    def file_part(path: str) -> Dict[str, Any]:
+        """Build a file content part (pdf etc.) as base64 data URL."""
+        import mimetypes
+        mime = mimetypes.guess_type(path)[0] or "application/octet-stream"
+        with open(path, "rb") as f:
+            b64 = base64.b64encode(f.read()).decode()
+        return {"type": "file", "file": {
+            "filename": path.split("/")[-1],
+            "file_data": f"data:{mime};base64,{b64}",
+        }}
+
+    # ------------------------------------------------------------------ api
+    def chat(self, model: str, messages: List[Dict[str, Any]],
+             tools: Optional[List[Dict[str, Any]]] = None,
+             tool_choice: str = "auto",
+             max_tokens: Optional[int] = None,
+             temperature: Optional[float] = None,
+             response_format: Optional[Dict[str, Any]] = None) -> ChatResult:
+        body: Dict[str, Any] = {"model": model, "messages": messages}
+        if tools:
+            body["tools"] = tools
+            body["tool_choice"] = tool_choice
+        if max_tokens:
+            body["max_tokens"] = max_tokens
+        if temperature is not None:
+            body["temperature"] = temperature
+        if response_format:
+            body["response_format"] = response_format
+
+        payload = json.dumps(body).encode()
+        last_err: Optional[str] = None
+
+        for attempt in range(self.max_retries):
+            t0 = time.time()
+            req = urllib.request.Request(
+                f"{self.base_url}/chat/completions", data=payload,
+                headers={"Authorization": f"Bearer {self.api_key}",
+                         "Content-Type": "application/json"})
+            try:
+                with self._opener.open(req, timeout=self.timeout) as r:
+                    data = json.loads(r.read().decode())
+                return self._parse(data, time.time() - t0)
+            except urllib.error.HTTPError as e:
+                detail = ""
+                try:
+                    detail = e.read().decode()[:300]
+                except Exception:
+                    pass
+                last_err = f"HTTP {e.code}: {detail}"
+                if e.code not in self.RETRYABLE:
+                    break
+            except Exception as e:  # noqa: BLE001
+                last_err = f"{type(e).__name__}: {e}"
+            time.sleep(2 * (attempt + 1))
+
+        return ChatResult(error=last_err, latency=time.time() - t0)
+
+    @staticmethod
+    def _parse(data: Dict[str, Any], latency: float) -> ChatResult:
+        if "choices" not in data:
+            return ChatResult(error=json.dumps(data, ensure_ascii=False)[:500],
+                              latency=latency, raw=data)
+        choice = data["choices"][0]
+        msg = choice.get("message", {})
+        return ChatResult(
+            content=msg.get("content"),
+            tool_calls=msg.get("tool_calls") or [],
+            finish_reason=choice.get("finish_reason"),
+            usage=data.get("usage", {}),
+            latency=latency,
+            raw=data,
+        )
+
+
+def parse_judge_json(text: str) -> Optional[Dict[str, Any]]:
+    """Extract a JSON object from a judge reply (tolerates code fences)."""
+    if not text:
+        return None
+    text = text.strip()
+    if "```" in text:
+        for seg in text.split("```"):
+            seg = seg.strip()
+            if seg.startswith("json"):
+                seg = seg[4:].strip()
+            if seg.startswith("{"):
+                text = seg
+                break
+    start, end = text.find("{"), text.rfind("}")
+    if start == -1 or end <= start:
+        return None
+    try:
+        return json.loads(text[start:end + 1])
+    except json.JSONDecodeError:
+        return None

wl_benchmark/publisher.py

@@ -0,0 +1,113 @@
+"""Publisher: upload a finished run to the WL-Benchmark platform
+(a Cloudflare Worker, see site/), then wipe the local run directory.
+
+Default flow of `wlb run`:
+  1. build one self-contained page for the new run (images inlined);
+  2. POST it to  <site>/api/runs  (Bearer upload token);
+  3. print the share link  <site>/r/<run-id>;
+  4. delete the local run directory — the platform is the single
+     source of truth.
+
+Configuration (config/site.json, gitignored; env vars win):
+  WL_BENCH_URL    e.g. https://benchmark.wulei.org
+  WL_BENCH_TOKEN  the platform's UPLOAD_TOKEN
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+import shutil
+import urllib.request
+
+from .site import build_run_page, manifest_entry
+
+SITE_CONFIG = os.path.join("config", "site.json")
+
+
+# --------------------------------------------------------------- config
+def load_site_config() -> dict | None:
+    cfg = {}
+    if os.path.exists(SITE_CONFIG):
+        try:
+            with open(SITE_CONFIG, encoding="utf-8") as f:
+                cfg = json.load(f)
+        except json.JSONDecodeError:
+            cfg = {}
+    cfg["url"] = os.environ.get("WL_BENCH_URL", cfg.get("url", "")).rstrip("/")
+    cfg["token"] = os.environ.get("WL_BENCH_TOKEN", cfg.get("token", ""))
+    if cfg["url"] and cfg["url"].startswith(("http://", "https://")):
+        return cfg
+    return None
+
+
+def prompt_site_config() -> dict | None:
+    """Interactive first-time setup; returns None if the user declines."""
+    print("\n[publish] the benchmark platform is not configured yet.")
+    print("           every run is uploaded to your own site and the local")
+    print("           data is deleted afterwards.")
+    ans = input("Configure it now? [Y/n] ").strip().lower()
+    if ans in ("n", "no"):
+        return None
+    url = ""
+    while not url.startswith(("http://", "https://")):
+        url = input("Platform URL (e.g. https://benchmark.wulei.org): "
+                    ).strip().rstrip("/")
+    token = input("Upload token (UPLOAD_TOKEN secret of the worker): ").strip()
+    if not token:
+        print("[publish] empty token — upload will likely be rejected (401)")
+    cfg = {"url": url, "token": token}
+    os.makedirs(os.path.dirname(SITE_CONFIG), exist_ok=True)
+    with open(SITE_CONFIG, "w", encoding="utf-8") as f:
+        json.dump(cfg, f, indent=1)
+    print(f"[publish] saved -> {SITE_CONFIG} (gitignored)")
+    return cfg
+
+
+# -------------------------------------------------------------- publish
+def publish_run(run_dir: str, cfg: dict) -> str:
+    """Upload one run; returns the share link."""
+    run_id = os.path.basename(run_dir.rstrip("/"))
+    with open(os.path.join(run_dir, "results.json"), encoding="utf-8") as f:
+        results = json.load(f)
+
+    page = build_run_page(results, run_id, run_dir)
+    payload = json.dumps({
+        "id": run_id,
+        "meta": manifest_entry(results, run_id),
+        "html": page,
+    }, ensure_ascii=False).encode()
+
+    req = urllib.request.Request(
+        cfg["url"] + "/api/runs", data=payload, method="POST",
+        headers={"Content-Type": "application/json; charset=utf-8",
+                 "Authorization": f"Bearer {cfg['token']}",
+                 # a custom UA: Cloudflare's Browser Integrity Check (error
+                 # 1010) blocks the default Python-urllib signature
+                 "User-Agent": "wl-benchmark/0.4"})
+    try:
+        with urllib.request.urlopen(req, timeout=120) as r:
+            resp = json.load(r)
+    except urllib.error.HTTPError as e:
+        detail = e.read().decode(errors="replace")[:200]
+        raise RuntimeError(f"upload rejected: HTTP {e.code} — {detail}") from e
+    if not resp.get("ok"):
+        raise RuntimeError(f"platform refused the run: {resp}")
+    return f"{cfg['url']}/r/{run_id}"
+
+
+# --------------------------------------------------------------- cleanup
+def cleanup_run(run_dir: str) -> bool:
+    """Delete a run directory (results/<YYYYMMDD-HHMMSS>) after a
+    successful upload. Refuses anything that does not look like a run."""
+    name = os.path.basename(run_dir.rstrip("/"))
+    parent = os.path.basename(os.path.dirname(run_dir.rstrip("/")))
+    if parent != "results" or not re.fullmatch(r"\d{8}-\d{6}", name):
+        print(f"[cleanup] refusing to delete {run_dir!r} — not a run dir")
+        return False
+    shutil.rmtree(run_dir, ignore_errors=True)
+    try:
+        os.rmdir(os.path.dirname(run_dir))   # drop empty results root
+    except OSError:
+        pass
+    return True

wl_benchmark/reporter.py

@@ -0,0 +1,61 @@
+"""Markdown reporter: aggregate results.json into a readable summary.
+
+essay / svg use human review: the score column shows "pending human review"
+and lists artifact paths for the reviewer.
+"""
+from __future__ import annotations
+
+import json
+import os
+from collections import defaultdict
+
+
+def summarize(results_path: str) -> str:
+    with open(results_path, encoding="utf-8") as f:
+        results = json.load(f)
+
+    lines = ["# WL-Benchmark Run Summary", ""]
+
+    by_model = defaultdict(list)
+    for r in results:
+        by_model[(r["provider"], r["model"])].append(r)
+
+    for (prov, model), rs in sorted(by_model.items()):
+        lines += [f"## {prov} / {model}", "",
+                  "| task | type | score | latency | artifacts | error |",
+                  "|---|---|---|---|---|---|"]
+        for r in rs:
+            if r["error"]:
+                score = "ERR"
+            elif r["score"] is None:
+                score = "pending human review"
+            else:
+                score = f"{r['score']:.2f}"
+            lat = f"{r['latency']}s" if r.get("latency") else "-"
+            arts = "; ".join(os.path.relpath(a) for a in r.get("artifacts", []))
+            arts = arts.replace("|", "\\|") or "-"
+            err = (r["error"] or "")[:60].replace("|", "\\|")
+            lines.append(f"| {r['task_id']} | {r['task_type']} | {score} "
+                         f"| {lat} | {arts} | {err} |")
+
+        scored = [r["score"] for r in rs
+                  if not r["error"] and r["score"] is not None]
+        manual = [r for r in rs if not r["error"] and r["score"] is None]
+        if scored:
+            avg = sum(scored) / len(scored)
+            lines += ["", f"**Auto-scored average: {avg:.3f} "
+                          f"({len(scored)} auto + {len(manual)} pending human "
+                          f"review)**"]
+        elif manual:
+            lines += ["", f"**All {len(manual)} tasks pending human review**"]
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def write_report(run_dir: str) -> str:
+    src = os.path.join(run_dir, "results.json")
+    dst = os.path.join(run_dir, "summary.md")
+    with open(dst, "w", encoding="utf-8") as f:
+        f.write(summarize(src))
+    return dst