benchscope 1.0.6__tar.gz → 1.1.0__tar.gz

{benchscope-1.0.6 → benchscope-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchscope
-Version: 1.0.6
+Version: 1.1.0
 Summary: LLM inference performance testing tool. Supports vLLM, SGLang, and any OpenAI-compatible API.
 Author-email: benchscope <labelnet@foxmail.com>
 License-Expression: Apache-2.0
@@ -29,8 +29,13 @@ Requires-Dist: openpyxl>=3.1
 Requires-Dist: pydantic>=2
 Requires-Dist: python-multipart>=0.0.9
 Requires-Dist: pyyaml>=6.0
+Requires-Dist: aiohttp>=3.9
 Provides-Extra: modelscope
 Requires-Dist: modelscope>=1.15; extra == "modelscope"
+Provides-Extra: accuracy-native
+Requires-Dist: torch>=2.0; extra == "accuracy-native"
+Requires-Dist: transformers>=4.40; extra == "accuracy-native"
+Requires-Dist: peft>=0.10; extra == "accuracy-native"
 Dynamic: license-file
 
 <div align="center">
@@ -41,9 +46,9 @@ Dynamic: license-file
 
 <p align="center"><strong>English</strong> | <a href="README.zh-CN.md">简体中文</a></p>
 
-BenchScope is an open‑source LLM inference benchmarking platform built with herness coding.
+BenchScope is an open‑source LLM inference benchmarking platform built with ~~Vibe Coding~~ Harness Coding.
 
-A visualization testing platform for LLM model **performance & accuracy**, supporting models deployed with vLLM / SGLang and any OpenAI-compatible inference service.
+A visualization testing platform for LLM model **performance & accuracy**, supporting models inference with vLLM / SGLang and any OpenAI‑compatible API.
 
 <div align="center">
   <img src="asserts/main-performance.png" width="72%" alt="BenchScope main performance screenshot" />
@@ -55,7 +60,7 @@ A visualization testing platform for LLM model **performance & accuracy**, suppo
 
 - **Easy to install** — `pip install` and one command starts the whole web platform.
 - **Performance testing dual mode** — Concurrency Mode (multi-level concurrency load) and Threshold Mode (auto-search the max concurrency meeting the threshold).
-- **Accuracy testing dual mode** — Online / offline testing (planned, v5.0).
+- **Accuracy testing dual mode** —  Native（local model weights）/ Serving（API）double model tasks。
 - **Real-time data feedback** — every concurrency result streams into tables, charts and progress in real time.
 - **Visualization curves** — multi-dimensional charts for throughput / TTFT / TPOT / ITL.
 - **Log cache & download** — run logs, mean/P99 summaries and Excel export with online preview & download.

{benchscope-1.0.6 → benchscope-1.1.0}/README.md

@@ -6,9 +6,9 @@
 
 <p align="center"><strong>English</strong> | <a href="README.zh-CN.md">简体中文</a></p>
 
-BenchScope is an open‑source LLM inference benchmarking platform built with herness coding.
+BenchScope is an open‑source LLM inference benchmarking platform built with ~~Vibe Coding~~ Harness Coding.
 
-A visualization testing platform for LLM model **performance & accuracy**, supporting models deployed with vLLM / SGLang and any OpenAI-compatible inference service.
+A visualization testing platform for LLM model **performance & accuracy**, supporting models inference with vLLM / SGLang and any OpenAI‑compatible API.
 
 <div align="center">
   <img src="asserts/main-performance.png" width="72%" alt="BenchScope main performance screenshot" />
@@ -20,7 +20,7 @@ A visualization testing platform for LLM model **performance & accuracy**, suppo
 
 - **Easy to install** — `pip install` and one command starts the whole web platform.
 - **Performance testing dual mode** — Concurrency Mode (multi-level concurrency load) and Threshold Mode (auto-search the max concurrency meeting the threshold).
-- **Accuracy testing dual mode** — Online / offline testing (planned, v5.0).
+- **Accuracy testing dual mode** —  Native（local model weights）/ Serving（API）double model tasks。
 - **Real-time data feedback** — every concurrency result streams into tables, charts and progress in real time.
 - **Visualization curves** — multi-dimensional charts for throughput / TTFT / TPOT / ITL.
 - **Log cache & download** — run logs, mean/P99 summaries and Excel export with online preview & download.

{benchscope-1.0.6 → benchscope-1.1.0}/benchscope/__init__.py

@@ -1,4 +1,4 @@
 """BenchScope - LLM inference performance testing tool."""
 
-# PEP 440：开发中带 .dev0 后缀，正式发布时改为 "1.0.6"
-__version__ = "1.0.6"
+# PEP 440：开发中带 .dev0 后缀，正式发布时改为 "1.0.7"
+__version__ = "1.1.0"

benchscope-1.1.0/benchscope/accuracy/__init__.py

@@ -0,0 +1,15 @@
+"""精度测试模块（Accuracy）：与性能模块彻底解耦的独立评测闭环。
+
+子模块：
+  - engines.py      精度引擎适配（bench 引擎注册表 eval 能力过滤 + 环境校验）
+  - datasets.py     评测数据集（datasets.yaml 注册表 + 下载/标准化/抽样/统计）
+  - executor.py     推理执行器（Serving/Native/Mock 统一接口；benchscope eval 实现体）
+  - metrics.py      指标汇总（accuracy/pass_rate/分学科/Token 统计/结论）
+  - estimator.py    Token 预估（常量表 > 实测统计 > 字符估算）
+  - baselines.py    开源基线库与对标计算
+  - task_manager.py EvalTaskManager（独立调度 + 三件套落库 + WS 推送）
+  - compare.py      多任务对比 / Native vs Serving 一致性差值
+  - scorers/        判分器（choice / math / code / judge）
+
+设计文档：docs/rules/AccuracyEngine.md
+"""

benchscope-1.1.0/benchscope/accuracy/baselines.py

@@ -0,0 +1,160 @@
+"""开源模型基准对标（平台核心新增能力）。
+
+每次评测结束后自动横向对标内置基线池（configs/baselines.yaml，静态固化公开测评
+分数 + 来源标注），生成：
+  1. 同数据集差值百分点（当前模型 vs 最接近尺寸基线 / 最优基线）
+  2. 能力档位评级 S / A / B / C（相对最优基线：≥0 → S；≥-5pp → A；≥-15pp → B；否则 C）
+  3. 同尺寸段排名百分比（段内基线中被当前模型超越的比例）
+  4. 能力雷达（知识 / 数学 / 代码 / 对话维度，单数据集任务取该维度得分）
+  5. 自动结论：优于同尺寸开源基线 / 持平基线 / 明显劣于基线（风险预警）
+"""
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+import yaml
+
+log = logging.getLogger("benchscope.accuracy.baselines")
+
+CONFIGS_DIR = Path(__file__).resolve().parent.parent / "configs"
+BASELINES_YAML = CONFIGS_DIR / "baselines.yaml"
+
+# 同尺寸段划分（B）：[0, 4) / [4, 15) / [15, 40) / [40, +∞)
+_SIZE_SEGMENTS = ((0, 4), (4, 15), (15, 40), (40, float("inf")))
+
+# 档位阈值（相对最优基线差值，百分点）
+_GRADE_RULES = (
+    (0.0, "S"),
+    (-5.0, "A"),
+    (-15.0, "B"),
+)
+
+RADAR_DIMS = ("知识", "数学", "代码", "对话")
+_DIM_BY_CATEGORY = {
+    "accuracy-knowledge": "知识",
+    "accuracy-math": "数学",
+    "accuracy-code": "代码",
+    "accuracy-chat": "对话",
+    "accuracy-mix": "综合",
+}
+
+
+def load_baselines() -> dict:
+    """读取基线库（原文 + 解析结果，供 API 展示与更新）。"""
+    content = BASELINES_YAML.read_text(encoding="utf-8") if BASELINES_YAML.exists() else ""
+    try:
+        data = yaml.safe_load(content) or {}
+    except Exception:
+        log.exception("解析 baselines.yaml 失败")
+        data = {}
+    return {
+        "version": data.get("version") or "",
+        "models": data.get("models") or [],
+        "content": content,
+    }
+
+
+def save_baselines(content: str) -> dict:
+    """管理员更新基线库（YAML 校验后写盘）。"""
+    data = yaml.safe_load(content or "")
+    if not isinstance(data, dict) or not isinstance(data.get("models"), list) or not data["models"]:
+        raise ValueError("基线库必须是含非空 models 列表的对象")
+    for m in data["models"]:
+        if not isinstance(m, dict) or not m.get("name") or not isinstance(m.get("scores"), dict):
+            raise ValueError("基线模型必须含 name 与 scores 字典")
+    BASELINES_YAML.write_text(content, encoding="utf-8")
+    return load_baselines()
+
+
+def _segment(params_b: float) -> int:
+    for i, (lo, hi) in enumerate(_SIZE_SEGMENTS):
+        if lo <= params_b < hi:
+            return i
+    return len(_SIZE_SEGMENTS) - 1
+
+
+def _grade(diff_vs_best: float) -> str:
+    for threshold, grade in _GRADE_RULES:
+        if diff_vs_best >= threshold:
+            return grade
+    return "C"
+
+
+def _conclusion(diff_vs_segment: float) -> str:
+    if diff_vs_segment >= 0:
+        return "优于同尺寸开源基线"
+    if diff_vs_segment >= -5.0:
+        return "持平基线"
+    return "明显劣于基线（风险预警）"
+
+
+def compute_benchmark(meta: dict, result: dict) -> dict | None:
+    """由数据集元数据与评测结果计算基线对标。
+
+    返回 None 表示该数据集无任何基线分数可对标。
+    """
+    dataset_id = meta.get("id") or ""
+    library = load_baselines()
+    models = [m for m in library["models"] if isinstance(m.get("scores"), dict)]
+
+    # 当前模型该数据集主指标（百分制）
+    main_metric = (meta.get("eval") or {}).get("metrics") or "accuracy"
+    score = None
+    if main_metric == "accuracy":
+        score = result.get("accuracy")
+    elif main_metric == "exact_match":
+        score = (result.get("dataset_metrics") or {}).get("exact_match")
+    elif main_metric == "pass@1":
+        score = (result.get("dataset_metrics") or {}).get("pass_at_1")
+    elif main_metric == "mt_bench":
+        score = (result.get("dataset_metrics") or {}).get("mt_bench_score")
+    if score is None:
+        return None
+
+    scored = [
+        {"name": m.get("name"), "params_b": float(m.get("params_b") or 0),
+         "group": m.get("group") or "general", "source": m.get("source") or "",
+         "score": float(m["scores"][dataset_id])}
+        for m in models if dataset_id in m["scores"]
+    ]
+    if not scored:
+        return None
+
+    best = max(scored, key=lambda m: m["score"])
+    # 尺寸最接近的基线（参数量差最小）
+    nearest = min(scored, key=lambda m: abs(m["params_b"] - float(result.get("_params_b") or 0) or 0))
+    current_seg = _segment(nearest["params_b"])
+    segment_models = [m for m in scored if _segment(m["params_b"]) == current_seg]
+    seg_best = max(segment_models, key=lambda m: m["score"])
+    better_count = sum(1 for m in segment_models if score >= m["score"])
+
+    diff_best = round(score - best["score"], 2)
+    diff_nearest = round(score - nearest["score"], 2)
+    diff_segment = round(score - seg_best["score"], 2)
+
+    # 能力雷达：单数据集任务仅该维度有值（综合类全维空）
+    dim = _DIM_BY_CATEGORY.get(meta.get("category") or "")
+    radar = {d: None for d in RADAR_DIMS}
+    if dim in radar:
+        radar[dim] = round(float(score), 2)
+
+    return {
+        "dataset_id": dataset_id,
+        "metric": main_metric,
+        "score": round(float(score), 2),
+        "baselines": scored,
+        "baseline_used": {"name": nearest["name"], "score": nearest["score"], "params_b": nearest["params_b"],
+                          "source": nearest["source"]},
+        "diff_pp": diff_nearest,
+        "diff_vs_best_pp": diff_best,
+        "best_baseline": {"name": best["name"], "score": best["score"]},
+        "grade": _grade(diff_best),
+        "rank_pct": round(better_count / len(segment_models) * 100.0, 1) if segment_models else None,
+        "segment": {"range_b": [segment_models[0]["params_b"], segment_models[-1]["params_b"]] if segment_models else [],
+                    "count": len(segment_models), "best": seg_best["name"], "best_score": seg_best["score"]},
+        "diff_vs_segment_pp": diff_segment,
+        "radar": radar,
+        "conclusion": _conclusion(diff_segment),
+        "baseline_version": library.get("version") or "",
+    }

benchscope-1.1.0/benchscope/accuracy/compare.py

@@ -0,0 +1,112 @@
+"""多任务对比 / Native vs Serving 训推一致性差值（P15）。
+
+对比口径：
+  - 横向对比：多任务按「模型+数据集」分组，逐指标（accuracy / exact_match / pass@1 /
+    mt_bench_score）列出；
+  - 一致性：同组内同时存在 native 与 serving 任务时计算逐指标差值
+    （|diff| ≤ 2pp 训推一致；≤ 5pp 存在偏差；> 5pp 显著偏差）。
+"""
+from __future__ import annotations
+
+CONSISTENT = "训推一致"
+DEVIATED = "存在偏差"
+SIGNIFICANT = "显著偏差"
+
+# 参与对比的主指标（按数据集判分器取值）
+_MAIN_METRICS = (
+    ("accuracy", "accuracy"),
+    ("exact_match", "dataset_metrics.exact_match"),
+    ("pass@1", "dataset_metrics.pass_at_1"),
+    ("mt_bench_score", "dataset_metrics.mt_bench_score"),
+)
+
+
+def _main_metric(result: dict | None) -> tuple[str, float | None]:
+    """任务的主指标取值（按数据集判分器选择）。"""
+    if not result:
+        return "accuracy", None
+    dm = result.get("dataset_metrics") or {}
+    if dm.get("pass_at_1") is not None:
+        return "pass@1", dm.get("pass_at_1")
+    if dm.get("exact_match") is not None:
+        return "exact_match", dm.get("exact_match")
+    if dm.get("mt_bench_score") is not None:
+        return "mt_bench_score", dm.get("mt_bench_score")
+    return "accuracy", result.get("accuracy")
+
+
+def _deviation_label(diff_pp: float) -> str:
+    diff = abs(diff_pp)
+    if diff <= 2.0:
+        return CONSISTENT
+    if diff <= 5.0:
+        return DEVIATED
+    return SIGNIFICANT
+
+
+def compare_tasks(task_snapshots: list[dict]) -> dict:
+    """对比多任务结果。
+
+    task_snapshots：[{task_id, name, mode, model, lora_name/lora_path, dataset_id,
+                      dataset_name, engine_id, result}]
+    返回 {items, groups}：items 为任务主指标表，groups 按「模型+数据集」分组，
+    组内含 native vs serving 一致性差值。
+    """
+    items = []
+    for t in task_snapshots:
+        result = t.get("result") or {}
+        metric_name, metric_value = _main_metric(result)
+        items.append({
+            "task_id": t.get("task_id"),
+            "name": t.get("name") or t.get("task_id"),
+            "mode": t.get("mode"),
+            "engine_id": t.get("engine_id"),
+            "model": t.get("model"),
+            "lora_name": t.get("lora_name") or "",
+            "lora_path": t.get("lora_path") or "",
+            "dataset_id": t.get("dataset_id"),
+            "dataset_name": t.get("dataset_name"),
+            "status": t.get("status"),
+            "metric": metric_name,
+            "metric_value": metric_value,
+            "total_samples": result.get("total_samples"),
+            "correct_samples": result.get("correct_samples"),
+            "invalid_samples": result.get("invalid_samples"),
+            "pass_rate": result.get("pass_rate"),
+            "conclusion": result.get("conclusion"),
+        })
+
+    # 按「模型 + 数据集」分组，组内做模式对比
+    groups: dict[str, list[dict]] = {}
+    for item in items:
+        key = f"{item.get('model')}|{item.get('lora_path') or ''}|{item.get('dataset_id')}"
+        groups.setdefault(key, []).append(item)
+
+    group_outputs = []
+    for key, members in groups.items():
+        native = next((m for m in members if m.get("mode") == "native"), None)
+        serving = next((m for m in members if m.get("mode") == "serving"), None)
+        consistency = None
+        if native and serving and native.get("metric_value") is not None and serving.get("metric_value") is not None:
+            diff = round(float(native["metric_value"]) - float(serving["metric_value"]), 2)
+            consistency = {
+                "native_task_id": native["task_id"],
+                "serving_task_id": serving["task_id"],
+                "metric": native["metric"],
+                "native_value": native["metric_value"],
+                "serving_value": serving["metric_value"],
+                "diff_pp": diff,
+                "conclusion": _deviation_label(diff),
+            }
+        group_outputs.append({
+            "key": key,
+            "model": members[0].get("model"),
+            "lora_path": members[0].get("lora_path") or "",
+            "dataset_id": members[0].get("dataset_id"),
+            "tasks": [m["task_id"] for m in members],
+            "items": members,
+            "consistency": consistency,
+        })
+
+    group_outputs.sort(key=lambda g: g["key"])
+    return {"items": items, "groups": group_outputs}