benchscope 1.0.5__tar.gz → 1.0.6__tar.gz

{benchscope-1.0.5 → benchscope-1.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchscope
-Version: 1.0.5
+Version: 1.0.6
 Summary: LLM inference performance testing tool. Supports vLLM, SGLang, and any OpenAI-compatible API.
 Author-email: benchscope <labelnet@foxmail.com>
 License-Expression: Apache-2.0
@@ -28,18 +28,25 @@ Requires-Dist: requests>=2.31
 Requires-Dist: openpyxl>=3.1
 Requires-Dist: pydantic>=2
 Requires-Dist: python-multipart>=0.0.9
+Requires-Dist: pyyaml>=6.0
 Provides-Extra: modelscope
 Requires-Dist: modelscope>=1.15; extra == "modelscope"
 Dynamic: license-file
 
-# benchscope
+<div align="center">
+  <img src="asserts/black_logo.png" width="120" height="120" alt="BenchScope logo" />
+</div>
+
+<h1 align="center" style="font-size: 48px; margin-top: 12px;">BenchScope</h1>
 
-**English** | [简体中文](README.zh-CN.md)
+<p align="center"><strong>English</strong> | <a href="README.zh-CN.md">简体中文</a></p>
 
-> A visualization testing platform for LLM model **performance & accuracy**, supporting models deployed with vLLM / SGLang and any OpenAI-compatible inference service.
+BenchScope is an open‑source LLM inference benchmarking platform built with herness coding.
+
+A visualization testing platform for LLM model **performance & accuracy**, supporting models deployed with vLLM / SGLang and any OpenAI-compatible inference service.
 
 <div align="center">
-  <img src="asserts/main-performance.png" width="72%" alt="benchscope main performance screenshot" />
+  <img src="asserts/main-performance.png" width="72%" alt="BenchScope main performance screenshot" />
 </div>
 
 ---
@@ -66,6 +73,10 @@ benchscope
 benchscope --port 8080 --no-browser
 ```
 
+## Development
+
+See [docs/Readme.md](docs/Readme.md).
+
 ## Open Source
 
 - **License** — [Apache License 2.0](LICENSE)

{benchscope-1.0.5 → benchscope-1.0.6}/README.md

@@ -1,11 +1,17 @@
-# benchscope
+<div align="center">
+  <img src="asserts/black_logo.png" width="120" height="120" alt="BenchScope logo" />
+</div>
+
+<h1 align="center" style="font-size: 48px; margin-top: 12px;">BenchScope</h1>
 
-**English** | [简体中文](README.zh-CN.md)
+<p align="center"><strong>English</strong> | <a href="README.zh-CN.md">简体中文</a></p>
 
-> A visualization testing platform for LLM model **performance & accuracy**, supporting models deployed with vLLM / SGLang and any OpenAI-compatible inference service.
+BenchScope is an open‑source LLM inference benchmarking platform built with herness coding.
+
+A visualization testing platform for LLM model **performance & accuracy**, supporting models deployed with vLLM / SGLang and any OpenAI-compatible inference service.
 
 <div align="center">
-  <img src="asserts/main-performance.png" width="72%" alt="benchscope main performance screenshot" />
+  <img src="asserts/main-performance.png" width="72%" alt="BenchScope main performance screenshot" />
 </div>
 
 ---
@@ -32,6 +38,10 @@ benchscope
 benchscope --port 8080 --no-browser
 ```
 
+## Development
+
+See [docs/Readme.md](docs/Readme.md).
+
 ## Open Source
 
 - **License** — [Apache License 2.0](LICENSE)

benchscope-1.0.6/benchscope/__init__.py

@@ -0,0 +1,4 @@
+"""BenchScope - LLM inference performance testing tool."""
+
+# PEP 440：开发中带 .dev0 后缀，正式发布时改为 "1.0.6"
+__version__ = "1.0.6"

benchscope-1.0.6/benchscope/builtin_datasets.py

@@ -0,0 +1,103 @@
+"""内置数据集模块：读取 configs/datasets.yaml，支持下载并缓存到 datasets_dir。"""
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Optional
+
+import yaml
+
+from benchscope.datasets import download_file, list_dataset_files
+
+log = logging.getLogger("benchscope.builtin_datasets")
+
+CONFIGS_DIR = Path(__file__).resolve().parent / "configs"
+DATASETS_YAML = CONFIGS_DIR / "datasets.yaml"
+
+
+def load_builtin_datasets() -> list[dict]:
+    """读取内置数据集定义（configs/datasets.yaml）。"""
+    if not DATASETS_YAML.exists():
+        log.warning("datasets.yaml 不存在: %s", DATASETS_YAML)
+        return []
+    try:
+        data = yaml.safe_load(DATASETS_YAML.read_text(encoding="utf-8"))
+        datasets = data.get("datasets", []) if isinstance(data, dict) else []
+        return [d for d in datasets if isinstance(d, dict) and d.get("id")]
+    except Exception:
+        log.exception("解析 datasets.yaml 失败")
+        return []
+
+
+def _cache_root(datasets_dir: Path) -> Path:
+    """内置数据集缓存根目录：datasets_dir（Settings/Datasets 管理的数据集下载目录）。"""
+    return datasets_dir
+
+
+def _ds_cache_dir(cache_root: Path, ds_id: str) -> Path:
+    return cache_root / ds_id
+
+
+def dataset_status(ds: dict, cache_root: Path) -> dict:
+    """返回数据集缓存状态：cached（文件列表）/ not_cached。"""
+    d = _ds_cache_dir(cache_root, ds.get("id", ""))
+    files: list[str] = []
+    if d.is_dir():
+        files = sorted(p.name for p in d.iterdir() if p.is_file())
+    return {"id": ds.get("id"), "cached": bool(files), "files": files}
+
+
+def download_builtin_dataset(ds: dict, data_dir: Path, timeout: int = 300) -> dict:
+    """下载内置数据集到缓存目录（data_dir/datasets/{id}/），返回下载结果。"""
+    ds_id = ds.get("id", "")
+    if not ds_id:
+        raise ValueError("数据集缺少 id")
+    source = ds.get("source") or {}
+    stype = source.get("type", "")
+    dest_dir = _ds_cache_dir(_cache_root(data_dir), ds_id)
+    dest_dir.mkdir(parents=True, exist_ok=True)
+
+    if stype == "modelscope":
+        dataset_id = source.get("dataset_id")
+        if not dataset_id:
+            raise ValueError(f"数据集 {ds_id} 缺少 modelscope dataset_id")
+        files = list_dataset_files(dataset_id, timeout=timeout)
+        cands = [
+            (f.get("Path"), f.get("Size") or 0)
+            for f in files
+            if (f.get("Path") or "").endswith((".json", ".jsonl", ".parquet", ".csv"))
+        ]
+        if not cands:
+            raise FileNotFoundError(f"数据集 {dataset_id} 中未找到可下载文件")
+        wanted = source.get("file")
+        if wanted:
+            matched = [c for c in cands if c[0].endswith(wanted)]
+            if matched:
+                cands = matched
+        # 优先真正的数据文件：按大小降序（排除 dataset_infos.json 等小元数据）
+        cands.sort(key=lambda x: -x[1])
+        target = cands[0][0]
+        dest = dest_dir / Path(target).name
+        if not dest.exists() or dest.stat().st_size == 0:
+            log.info("下载内置数据集 %s 文件 %s ...", ds_id, target)
+            download_file(dataset_id, target, dest, timeout=timeout)
+        return {"id": ds_id, "path": str(dest), "size": dest.stat().st_size}
+    elif stype == "url":
+        url = source.get("url")
+        if not url:
+            raise ValueError(f"数据集 {ds_id} 缺少 url")
+        import requests
+
+        dest = dest_dir / Path(url.split("?")[0]).name
+        if not dest.exists() or dest.stat().st_size == 0:
+            log.info("下载内置数据集 %s（%s）...", ds_id, url)
+            tmp = dest.with_suffix(dest.suffix + ".part")
+            with requests.get(url, stream=True, timeout=timeout) as resp:
+                resp.raise_for_status()
+                with open(tmp, "wb") as f:
+                    for chunk in resp.iter_content(chunk_size=1 << 20):
+                        f.write(chunk)
+            tmp.replace(dest)
+        return {"id": ds_id, "path": str(dest), "size": dest.stat().st_size}
+    else:
+        raise ValueError(f"数据集 {ds_id} 不支持的类型: {stype!r}")

{benchscope-1.0.5 → benchscope-1.0.6}/benchscope/cli.py

@@ -6,6 +6,25 @@ import logging
 import sys
 import webbrowser
 import threading
+from datetime import datetime
+
+
+def _setup_runtime_logging() -> None:
+    """将服务运行日志落盘到 logs 目录：runtime_年月日.log。"""
+    try:
+        from benchscope.server.state import state
+
+        logs_dir = state.config.logs_dir
+        logs_dir.mkdir(parents=True, exist_ok=True)
+        rt_file = logs_dir / f"runtime_{datetime.now().strftime('%Y%m%d')}.log"
+        fh = logging.FileHandler(rt_file, encoding="utf-8")
+        fh.setFormatter(
+            logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s")
+        )
+        logging.getLogger().addHandler(fh)
+        logging.getLogger("benchscope").info("runtime 日志已落盘: %s", rt_file)
+    except Exception:
+        logging.getLogger("benchscope").exception("runtime 日志配置失败")
 
 
 def main(argv=None) -> int:
@@ -29,6 +48,7 @@ def main(argv=None) -> int:
     from benchscope.server.app import create_app
 
     app = create_app()
+    _setup_runtime_logging()
     url = f"http://127.0.0.1:{args.port}"
 
     if not args.no_browser:

benchscope-1.0.6/benchscope/config.py

@@ -0,0 +1,234 @@
+"""配置持久化：settings.json 读写与运行时配置单例。"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+from copy import deepcopy
+from pathlib import Path
+
+from benchscope.constants import DEFAULT_CONFIG
+
+log = logging.getLogger("benchscope.config")
+
+
+def _data_root() -> Path:
+    """数据根目录：默认 ~/.benchscope；可通过环境变量 BENCHSCOPE_DATA_DIR 覆盖（测试隔离）。"""
+    env = os.environ.get("BENCHSCOPE_DATA_DIR")
+    if env:
+        return Path(os.path.expanduser(env)).resolve()
+    return Path.home() / ".benchscope"
+
+
+DATA_ROOT = _data_root()
+
+# 设置持久化文件：~/.benchscope/settings.json
+DEFAULT_CONFIG_PATH = DATA_ROOT / "settings.json"
+# 旧版配置（兼容迁移）
+LEGACY_CONFIG_PATH = DATA_ROOT / "config.json"
+# 旧版默认路径 -> 新目录体系（迁移归一化）
+LEGACY_DEFAULT_PATHS = {
+    "./logs": str(DATA_ROOT / "logs"),
+    "./datasets": str(DATA_ROOT / "datasets"),
+}
+
+# data_dir 下的默认子目录映射：配置 key -> 子目录名
+DATA_SUBDIRS = {
+    "perfs_dir": "perfs",
+    "evals_dir": "evals",
+    "analysis_dir": "analysys",
+    "logs_dir": "logs",
+    "sessions_dir": "sessions",
+    "models_dir": "models",
+    "datasets_dir": "datasets",
+    "plugins_dir": "plugins",
+}
+
+
+class ConfigManager:
+    """线程安全的配置管理。"""
+
+    def __init__(self, path: Path | str | None = None):
+        self.path = Path(path) if path else DEFAULT_CONFIG_PATH
+        # 旧版 config.json 与 settings.json 同目录（默认均为 ~/.benchscope）
+        self.legacy_path = self.path.parent / "config.json"
+        self._lock = threading.RLock()
+        self._data: dict = deepcopy(DEFAULT_CONFIG)
+        self._redirect_default_dirs()
+        self.load()
+
+    def _redirect_default_dirs(self) -> None:
+        """测试隔离：设置 BENCHSCOPE_DATA_DIR 时，把默认缓存目录重定向到该根目录下。
+
+        仅作用于默认值；settings.json 中已持久化的自定义目录仍以持久化值为准。
+        """
+        env = os.environ.get("BENCHSCOPE_DATA_DIR")
+        if not env:
+            return
+        root = Path(os.path.expanduser(env)).resolve()
+        self._data["data_dir"] = str(root)
+        for key, sub in DATA_SUBDIRS.items():
+            self._data[key] = str(root / sub)
+
+    # ---------- 持久化 ----------
+    def load(self) -> None:
+        with self._lock:
+            # 优先读 settings.json；若不存在则尝试旧版 config.json（兼容迁移）
+            source = self.path if self.path.exists() else self.legacy_path
+            if source and source.exists():
+                try:
+                    loaded = json.loads(source.read_text(encoding="utf-8"))
+                    self._merge(self._data, loaded)
+                    self._normalize_legacy_defaults()
+                except Exception:
+                    log.exception("配置加载失败，使用默认配置: %s", source)
+                # 若读取的是旧版 config.json，则一次性落盘 settings.json 完成迁移
+                if source != self.path:
+                    self.save()
+            else:
+                # 首次启动：落盘默认配置，确保 settings.json 存在
+                self.save()
+            # 启动时检查目录是否齐全（缺失则创建）
+            self.ensure_dirs()
+
+    def _normalize_legacy_defaults(self) -> None:
+        """旧版默认相对路径（./logs、./datasets）归一化为新目录体系默认值。"""
+        for old, new in LEGACY_DEFAULT_PATHS.items():
+            for key in ("logs_dir", "datasets_dir"):
+                if self._data.get(key) == old:
+                    self._data[key] = new
+
+    def save(self) -> None:
+        with self._lock:
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            self.path.write_text(
+                json.dumps(self._data, ensure_ascii=False, indent=2),
+                encoding="utf-8",
+            )
+
+    @staticmethod
+    def _merge(base: dict, overlay: dict) -> None:
+        for key, value in overlay.items():
+            if isinstance(value, dict) and isinstance(base.get(key), dict):
+                ConfigManager._merge(base[key], value)
+            else:
+                base[key] = value
+
+    # ---------- 访问 ----------
+    def get(self, key: str, default=None):
+        with self._lock:
+            return self._data.get(key, default)
+
+    def set(self, key: str, value) -> None:
+        with self._lock:
+            self._data[key] = value
+            self.save()
+
+    def update(self, patch: dict) -> dict:
+        with self._lock:
+            # data_dir 联动：当数据根目录变化时，未自定义的子目录跟随新 data_dir
+            new_data = patch.get("data_dir")
+            if new_data:
+                old_data = self._data.get("data_dir", DEFAULT_CONFIG["data_dir"])
+                old_root = str(Path(os.path.expanduser(old_data)).resolve())
+                new_root = str(Path(os.path.expanduser(new_data)).resolve())
+                for key, sub in DATA_SUBDIRS.items():
+                    if key in patch:  # 本次显式指定，不联动
+                        continue
+                    cur = self._data.get(key)
+                    if cur:
+                        cur_resolved = str(Path(os.path.expanduser(cur)).resolve())
+                        if cur_resolved == str(Path(old_root) / sub):
+                            patch[key] = str(Path(new_root) / sub)
+            self._merge(self._data, patch)
+            self.save()
+            self.ensure_dirs()
+            return deepcopy(self._data)
+
+    def snapshot(self) -> dict:
+        with self._lock:
+            return deepcopy(self._data)
+
+    def resolve_dir(self, key: str) -> Path:
+        """解析配置中的目录（展开 ~ 并 resolve）。"""
+        raw = self._data.get(key) or DEFAULT_CONFIG.get(key, "")
+        return Path(os.path.expanduser(raw)).resolve()
+
+    def ensure_dirs(self) -> None:
+        """检查并创建所有缓存目录（启动时调用）。"""
+        with self._lock:
+            for key in ("data_dir", "perfs_dir", "evals_dir", "analysis_dir",
+                        "logs_dir", "sessions_dir", "models_dir", "datasets_dir", "plugins_dir"):
+                raw = self._data.get(key, DEFAULT_CONFIG.get(key, ""))
+                try:
+                    Path(os.path.expanduser(raw)).mkdir(parents=True, exist_ok=True)
+                except Exception:
+                    log.exception("目录创建失败: %s=%s", key, raw)
+
+    # ---------- 常用辅助 ----------
+    @property
+    def api(self) -> dict:
+        return self.get("api", {})
+
+    @property
+    def data_dir(self) -> Path:
+        """数据根目录，默认 ~/.benchscope。"""
+        raw = self.get("data_dir", DEFAULT_CONFIG["data_dir"])
+        return Path(os.path.expanduser(raw)).resolve()
+
+    @property
+    def perfs_dir(self) -> Path:
+        """性能测试任务目录，默认 ~/.benchscope/perfs。"""
+        raw = self.get("perfs_dir", DEFAULT_CONFIG["perfs_dir"])
+        return Path(os.path.expanduser(raw)).resolve()
+
+    @property
+    def evals_dir(self) -> Path:
+        """精度测试任务目录，默认 ~/.benchscope/evals。"""
+        raw = self.get("evals_dir", DEFAULT_CONFIG["evals_dir"])
+        return Path(os.path.expanduser(raw)).resolve()
+
+    @property
+    def analysis_dir(self) -> Path:
+        """数据分析目录，默认 ~/.benchscope/analysys（联动 Datas）。"""
+        raw = self.get("analysis_dir", DEFAULT_CONFIG["analysis_dir"])
+        return Path(os.path.expanduser(raw)).resolve()
+
+    @property
+    def logs_dir(self) -> Path:
+        """日志目录（runtime 日志 + 任务终端输出），默认 ~/.benchscope/logs。"""
+        raw = self.get("logs_dir", DEFAULT_CONFIG["logs_dir"])
+        return Path(os.path.expanduser(raw)).resolve()
+
+    @property
+    def sessions_dir(self) -> Path:
+        """会话缓存目录，默认 ~/.benchscope/sessions。"""
+        raw = self.get("sessions_dir", DEFAULT_CONFIG["sessions_dir"])
+        return Path(os.path.expanduser(raw)).resolve()
+
+    @property
+    def datasets_dir(self) -> Path:
+        """数据集下载目录，默认 ~/.benchscope/datasets。"""
+        raw = self.get("datasets_dir", DEFAULT_CONFIG["datasets_dir"])
+        return Path(os.path.expanduser(raw)).resolve()
+
+    @property
+    def models_dir(self) -> Path:
+        """模型下载缓存目录，默认 ~/.benchscope/models。"""
+        raw = self.get("models_dir", DEFAULT_CONFIG["models_dir"])
+        return Path(os.path.expanduser(raw)).resolve()
+
+    @property
+    def plugins_dir(self) -> Path:
+        """插件安装加载目录，默认 ~/.benchscope/plugins。"""
+        raw = self.get("plugins_dir", DEFAULT_CONFIG["plugins_dir"])
+        return Path(os.path.expanduser(raw)).resolve()
+
+    def set_api(self, patch: dict) -> dict:
+        with self._lock:
+            api = deepcopy(self._data.setdefault("api", {}))
+            api.update(patch)
+            self._data["api"] = api
+            self.save()
+            return deepcopy(api)

{benchscope-1.0.5 → benchscope-1.0.6}/benchscope/constants.py

@@ -38,9 +38,16 @@ DEFAULT_CONFIG = {
         "extra_headers": {},
     },
     "gpu": {"auto": True, "name": "", "count": 8},
-    "logs_dir": "./logs",
-    "datasets_dir": "./datasets",
-    "data_dir": "~/.benchscope",  # 服务端数据持久化目录（任务 / 会话等）
+    # 缓存目录体系（数据根目录为 data_dir，其余为各功能目录）
+    "data_dir": "~/.benchscope",  # 数据根目录（服务端数据持久化 / 任务 / 会话等）
+    "perfs_dir": "~/.benchscope/perfs",  # 性能测试任务目录
+    "evals_dir": "~/.benchscope/evals",  # 精度测试任务目录
+    "analysis_dir": "~/.benchscope/analysys",  # 数据分析目录（联动 Datas）
+    "logs_dir": "~/.benchscope/logs",  # 日志目录（runtime 日志 + 任务终端输出）
+    "sessions_dir": "~/.benchscope/sessions",  # 会话缓存目录
+    "models_dir": "~/.benchscope/models",  # 模型下载缓存目录（联动 Settings/Models）
+    "datasets_dir": "~/.benchscope/datasets",  # 数据集下载目录（联动 Settings/Datasets）
+    "plugins_dir": "~/.benchscope/plugins",  # 插件安装加载目录（联动 Settings/Plugins）
     "tpot_threshold_ms": 100,
     "request_rate": "inf",  # inf 或数字
     "bench_commands": {

{benchscope-1.0.5 → benchscope-1.0.6}/benchscope/parser.py

@@ -73,7 +73,7 @@ def parse_metrics(output: str) -> dict:
     返回结构:
     {
       "concurrency": int,
-      "output": float, "peakoutput": float, "total": float,
+      "output_mean": float, "peakoutput_mean": float, "total_mean": float,
       "ttft_mean": float, "tpot_mean": float, "itl_mean": float,
       "ttft_p99": float, "tpot_p99": float, "itl_p99": float,
       "req_per_s": float, "raw": str(完整原始输出)