vlmeter 0.2.0__py3-none-any.whl

vlmeter/__init__.py

@@ -0,0 +1,3 @@
+"""vlmeter — 多模态大模型准确率 × 成本计量流水线（vision-language meter）."""
+
+__version__ = "0.2.0"

vlmeter/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

vlmeter/cli.py

@@ -0,0 +1,135 @@
+"""Command-line interface: run / report / list-models."""
+from __future__ import annotations
+
+import argparse
+import asyncio
+import os
+import sys
+from pathlib import Path
+
+from . import __version__
+from .models import load_registry
+from .runner import RESULTS_DIR, RunOptions, run_eval
+
+
+def _load_dotenv() -> None:
+    env = Path(".env")
+    if not env.exists():
+        return
+    for line in env.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        k, _, v = line.partition("=")
+        os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'"))
+
+
+def _common_args(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--registry", type=Path, default=None, help="models.yaml 路径（默认 config/models.yaml）")
+    p.add_argument("--results-dir", type=Path, default=RESULTS_DIR)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="vlmeter", description="多模态大模型准确率评测流水线"
+    )
+    parser.add_argument("--version", action="version", version=f"vlmeter {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    run = sub.add_parser("run", help="跑评测（支持断点续跑）")
+    run.add_argument("--models", required=True, help="逗号分隔的模型名，见 list-models")
+    run.add_argument("--data", required=True, type=Path, help="数据集路径（.jsonl / .csv）")
+    run.add_argument("--judge", default=None, help="judge 裁判模型名（VQA 语义判分）")
+    run.add_argument("--limit", type=int, default=None, help="只评测前 N 题")
+    run.add_argument("--fresh", action="store_true", help="忽略已缓存结果，全部重跑")
+    _common_args(run)
+
+    rep = sub.add_parser("report", help="由已有结果生成对比报告")
+    rep.add_argument("--data", required=True, type=Path)
+    rep.add_argument("--models", default=None, help="逗号分隔；缺省为有结果文件的所有注册模型")
+    _common_args(rep)
+
+    sub.add_parser("list-models", help="列出模型注册表").add_argument(
+        "--registry", type=Path, default=None
+    )
+    return parser
+
+
+def _resolve_model_names(args) -> list[str]:
+    registry = load_registry(args.registry)
+    if getattr(args, "models", None):
+        names = [m.strip() for m in args.models.split(",") if m.strip()]
+        unknown = [m for m in names if m not in registry]
+        if unknown:
+            sys.exit(f"未知模型: {unknown}，可选: {sorted(registry)}")
+        return names
+    # report without --models: any registered model that has a result file
+    from .dataset import load_dataset
+    from .runner import result_path
+
+    ds = load_dataset(args.data)
+    names = [
+        n for n in registry
+        if result_path(n, ds, args.results_dir).exists()
+    ]
+    if not names:
+        sys.exit("没有找到任何结果文件；请先运行 vlmeter run")
+    return names
+
+
+def main(argv: list[str] | None = None) -> None:
+    _load_dotenv()
+    args = build_parser().parse_args(argv)
+
+    if args.command == "list-models":
+        registry = load_registry(args.registry)
+        print(f"{'名称':<28}{'provider':<12}{'模型ID':<30}{'单价(入/出 ¥/M)':<18}密钥环境变量")
+        print("-" * 110)
+        for name, cfg in registry.items():
+            key = cfg.api_key_env or "-"
+            price = (
+                f"{cfg.price_input}/{cfg.price_output}" if cfg.price_input is not None else "-"
+            )
+            missing = (
+                f"（未设置 {key}！）" if cfg.api_key_env and not os.environ.get(cfg.api_key_env)
+                else ""
+            )
+            print(f"{name:<28}{cfg.provider:<12}{cfg.model:<30}{price:<18}{key}{missing}")
+        return
+
+    if args.command == "run":
+        opts = RunOptions(
+            models=[m.strip() for m in args.models.split(",") if m.strip()],
+            data=args.data,
+            judge=args.judge,
+            limit=args.limit,
+            fresh=args.fresh,
+            registry_path=args.registry,
+            results_dir=args.results_dir,
+        )
+        summaries = asyncio.run(run_eval(opts))
+        print("\n运行完成：")
+        for s in summaries:
+            print(
+                f"  {s.model}: 新作答 {s.answered}，跳过已缓存 {s.skipped_no_change}，"
+                f"失败 {s.errors}，未判定 {s.unscored}"
+            )
+        # fall through to report generation below with resolved models
+        args.models = ",".join(opts.models)
+
+    from .dataset import load_dataset
+    from .report import console_summary, generate_report, report_path
+
+    registry = load_registry(args.registry)
+    dataset = load_dataset(args.data)
+    names = _resolve_model_names(args)
+    args.results_dir.mkdir(parents=True, exist_ok=True)
+    console_summary(dataset, names, args.results_dir, registry)
+    md = generate_report(dataset, names, args.results_dir, registry=registry)
+    out = report_path(dataset, args.results_dir)
+    out.write_text(md, encoding="utf-8")
+    print(f"\n报告已生成: {out}")
+
+
+if __name__ == "__main__":
+    main()

vlmeter/dataset.py

@@ -0,0 +1,188 @@
+"""Dataset loading and validation.
+
+A dataset is a JSONL file (one item per line) or a CSV with the same columns:
+
+  id          unique id (auto-generated from row index if missing)
+  image       path to the image, relative to the dataset file
+  task        "classification" | "vqa" | "checklist" (default: vqa)
+  question    prompt shown to the model (not needed for checklist)
+  choices     classification: option list, e.g. ["猫", "狗", "鸟"]
+  answer      ground truth: option text or its letter (classification), or the
+              reference answer text (vqa)
+  category    optional, used for per-category breakdown in the report
+  match       optional vqa scoring strategy: exact | contains | numeric | judge
+  tolerance   optional numeric tolerance for match=numeric
+  checks      checklist only: [{"name": "有自行车", "answer": true,
+              "criteria": "可见完整的自行车或其主要部分，含部分遮挡"}] —
+              one image, multiple yes/no scene checks, answered in ONE call;
+              name 短名用于报告，criteria 判定标准只进提示词（可选）
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+
+VALID_TASKS = {"classification", "vqa", "checklist"}
+VALID_MATCH = {"exact", "contains", "numeric", "judge"}
+
+
+@dataclass
+class Check:
+    name: str      # 短名：用于报告列名与结果 ID
+    answer: bool
+    criteria: str = ""  # 判定标准：写进提示词，对齐模型与标注员的主观标准
+
+
+@dataclass
+class Item:
+    id: str
+    image: str
+    task: str = "vqa"
+    question: str = ""
+    choices: list[str] = field(default_factory=list)
+    answer: str = ""
+    category: str = "未分类"
+    match: str | None = None
+    tolerance: float = 0.0
+    checks: list[Check] = field(default_factory=list)
+
+
+def check_record_id(item_id: str, check_name: str) -> str:
+    return f"{item_id}::{check_name}"
+
+
+def item_record_ids(item: Item) -> list[str]:
+    """Result-record ids an item expands to (checklist items yield one per check)."""
+    if item.task == "checklist":
+        return [check_record_id(item.id, c.name) for c in item.checks]
+    return [item.id]
+
+
+@dataclass
+class Dataset:
+    items: list[Item]
+    path: Path
+    fingerprint: str  # md5 of the raw file, used to key result caching
+
+    def __len__(self) -> int:
+        return len(self.items)
+
+    @property
+    def total_records(self) -> int:
+        return sum(len(item_record_ids(it)) for it in self.items)
+
+
+def load_dataset(path: str | Path) -> Dataset:
+    p = Path(path).resolve()
+    if not p.exists():
+        raise FileNotFoundError(f"dataset not found: {p}")
+    fingerprint = hashlib.md5(p.read_bytes()).hexdigest()
+
+    rows: list[dict] = []
+    if p.suffix.lower() == ".csv":
+        import pandas as pd
+
+        df = pd.read_csv(p, dtype=str, keep_default_na=False)
+        for _, row in df.iterrows():
+            rows.append(dict(row))
+    else:
+        for i, line in enumerate(p.read_text(encoding="utf-8").splitlines(), 1):
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            try:
+                rows.append(json.loads(line))
+            except json.JSONDecodeError as e:
+                raise ValueError(f"{p.name} line {i}: invalid JSON ({e})") from e
+
+    items: list[Item] = []
+    errors: list[str] = []
+    for i, row in enumerate(rows):
+        item, err = _build_item(row, i, p.parent)
+        if err:
+            errors.append(err)
+        else:
+            items.append(item)
+    if errors:
+        raise ValueError("dataset validation failed:\n  " + "\n  ".join(errors))
+    if not items:
+        raise ValueError(f"dataset {p.name} has no items")
+    return Dataset(items=items, path=p, fingerprint=fingerprint)
+
+
+def _build_item(row: dict, index: int, base_dir: Path) -> tuple[Item | None, str | None]:
+    where = f"row {index}" + (f" (id={row.get('id')})" if row.get("id") else "")
+
+    image = str(row.get("image") or "").strip()
+    if not image:
+        return None, f"{where}: missing 'image'"
+    image_path = Path(image)
+    if not image_path.is_absolute():
+        image_path = (base_dir / image_path).resolve()
+    if not image_path.exists():
+        return None, f"{where}: image not found: {image_path}"
+
+    task = str(row.get("task") or "vqa").strip().lower()
+    if task not in VALID_TASKS:
+        return None, f"{where}: unknown task '{task}' (expected {sorted(VALID_TASKS)})"
+
+    question = str(row.get("question") or "").strip()
+    if not question and task != "checklist":
+        return None, f"{where}: missing 'question'"
+
+    checks: list[Check] = []
+    if task == "checklist":
+        raw_checks = row.get("checks")
+        if not isinstance(raw_checks, list) or not raw_checks:
+            return None, f"{where}: checklist needs a non-empty 'checks' list"
+        for j, c in enumerate(raw_checks):
+            if not isinstance(c, dict) or not str(c.get("name") or "").strip():
+                return None, f"{where}: checks[{j}] needs a 'name'"
+            ans = c.get("answer")
+            if not isinstance(ans, bool):
+                return None, f"{where}: checks[{j}].answer must be true/false"
+            checks.append(
+                Check(
+                    name=str(c["name"]).strip(),
+                    answer=ans,
+                    criteria=str(c.get("criteria") or "").strip(),
+                )
+            )
+    else:
+        answer = str(row.get("answer") if row.get("answer") is not None else "").strip()
+        if not answer:
+            return None, f"{where}: missing 'answer'"
+
+    choices_raw = row.get("choices")
+    choices = [str(c).strip() for c in choices_raw if str(c).strip()] if choices_raw else []
+    if task == "classification":
+        if len(choices) < 2:
+            return None, f"{where}: classification needs >=2 choices"
+        gold = answer if answer in choices else (
+            choices[ord(answer.upper()[0]) - 65] if len(answer) == 1 and "A" <= answer.upper()[0] <= "Z"
+            and ord(answer.upper()[0]) - 65 < len(choices) else None
+        )
+        if gold is None:
+            return None, f"{where}: answer '{answer}' not in choices {choices}"
+
+    match = row.get("match")
+    if match is not None:
+        match = str(match).strip().lower()
+        if match not in VALID_MATCH:
+            return None, f"{where}: unknown match '{match}' (expected {sorted(VALID_MATCH)})"
+
+    item = Item(
+        id=str(row.get("id") or f"q{index:04d}"),
+        image=str(image_path),
+        task=task,
+        question=question,
+        choices=choices,
+        answer=answer if task != "checklist" else "",
+        category=str(row.get("category") or "未分类"),
+        match=match,
+        tolerance=float(row.get("tolerance") or 0.0),
+        checks=checks,
+    )
+    return item, None

vlmeter/images.py

@@ -0,0 +1,56 @@
+"""Image loading, optional downscaling, and base64 encoding for each provider."""
+from __future__ import annotations
+
+import base64
+import io
+from pathlib import Path
+
+MIME_BY_EXT = {
+    ".png": "image/png",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".webp": "image/webp",
+    ".gif": "image/gif",
+}
+
+# Providers differ; keep requests lean by capping the longest side.
+MAX_SIDE = 1568
+MAX_BYTES = 4 * 1024 * 1024
+
+
+def _load_bytes(path: str | Path) -> tuple[bytes, str]:
+    p = Path(path)
+    if not p.exists():
+        raise FileNotFoundError(f"image not found: {p}")
+    mime = MIME_BY_EXT.get(p.suffix.lower(), "image/png")
+    return p.read_bytes(), mime
+
+
+def prepare_image(path: str | Path) -> tuple[bytes, str]:
+    """Return (raw_bytes, mime). Downscale/re-encode oversized images."""
+    raw, mime = _load_bytes(path)
+    if len(raw) <= MAX_BYTES:
+        return raw, mime
+
+    from PIL import Image  # imported lazily so non-image runs need no Pillow
+
+    img = Image.open(io.BytesIO(raw))
+    if img.mode not in ("RGB", "L"):
+        img = img.convert("RGB")
+    if max(img.size) > MAX_SIDE:
+        img.thumbnail((MAX_SIDE, MAX_SIDE))
+    buf = io.BytesIO()
+    img.save(buf, format="JPEG", quality=90)
+    return buf.getvalue(), "image/jpeg"
+
+
+def b64_image(path: str | Path) -> tuple[str, str]:
+    """Return (base64_no_prefix, mime)."""
+    raw, mime = prepare_image(path)
+    return base64.b64encode(raw).decode("ascii"), mime
+
+
+def data_url(path: str | Path) -> str:
+    """Return a data: URL, accepted by OpenAI-compatible chat endpoints."""
+    b64, mime = b64_image(path)
+    return f"data:{mime};base64,{b64}"

vlmeter/judge.py

@@ -0,0 +1,75 @@
+"""LLM-as-judge for open-ended VQA answers (text-only model call)."""
+from __future__ import annotations
+
+import json
+import re
+
+from .models.base import BaseVLModel
+from .retry import retry_call
+
+_JUDGE_PROMPT = """你是评测裁判。根据问题、标准答案和模型回答，判断模型回答是否正确。
+判定标准：数值必须相等（单位不同但数值等价算对，如 1.5kg 与 1500g）；文字答案允许表述差异，但含义必须与标准答案一致；回答中包含正确答案且无矛盾信息即算对。
+
+问题：{question}
+标准答案：{gold}
+模型回答：{prediction}
+
+只输出一个 JSON 对象，格式：{{"correct": true}} 或 {{"correct": false}}，不要输出其他内容。"""
+
+
+_CHOOSE_PROMPT = """你是评测裁判。模型本应从选项中选一个并回答字母，但它输出了一段描述。
+请判断模型的回答最符合哪个选项。
+
+问题：{question}
+选项：
+{options}
+模型回答：{prediction}
+
+只输出一个 JSON 对象，格式：{{"choice": "A"}}（字母为最符合的选项），不要输出其他内容。如果模型回答与任何选项都不符，输出 {{"choice": null}}。"""
+
+
+class Judge:
+    def __init__(self, model: BaseVLModel, attempts: int = 3):
+        self.model = model
+        self.attempts = attempts
+
+    async def _ask(self, prompt: str) -> str | None:
+        async def _call():
+            return await self.model.chat(None, prompt)
+
+        try:
+            resp = await retry_call(_call, attempts=self.attempts)
+        except Exception as e:  # noqa: BLE001
+            return None
+        m = re.search(r"\{[^{}]*\}", resp.text, re.S)
+        if not m:
+            return None
+        try:
+            return json.loads(m.group(0))
+        except json.JSONDecodeError:
+            return None
+
+    async def verify(self, question: str, gold: str, prediction: str) -> tuple[bool | None, str]:
+        """Return (correct, reason). correct=None means judging failed."""
+        prompt = _JUDGE_PROMPT.format(question=question, gold=gold, prediction=prediction[:500])
+        verdict = await self._ask(prompt)
+        if isinstance(verdict, dict) and isinstance(verdict.get("correct"), bool):
+            return verdict["correct"], ""
+        return None, "judge_parse_fail"
+
+    async def choose(self, question: str, choices: list[str], prediction: str) -> int | None:
+        """Map a free-form classification answer back to a choice index."""
+        from .prompting import LETTERS
+
+        options = "\n".join(f"{LETTERS[i]}. {c}" for i, c in enumerate(choices))
+        prompt = _CHOOSE_PROMPT.format(
+            question=question, options=options, prediction=prediction[:500]
+        )
+        verdict = await self._ask(prompt)
+        if isinstance(verdict, dict):
+            choice = verdict.get("choice")
+            if isinstance(choice, str) and len(choice) == 1 and "A" <= choice.upper() <= "H":
+                idx = ord(choice.upper()[0]) - 65
+                if idx < len(choices):
+                    return idx
+        return None

vlmeter/models/__init__.py

@@ -0,0 +1,61 @@
+"""Model registry: load config/models.yaml and build adapter instances."""
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+from .base import BaseVLModel, ModelConfig
+from .mock import MockModel
+
+PROVIDERS: dict[str, type[BaseVLModel]] = {}
+
+
+def _register(cls: type[BaseVLModel]) -> type[BaseVLModel]:
+    PROVIDERS[cls.provider_name] = cls
+    return cls
+
+
+def _build_providers() -> None:
+    from .anthropic_client import AnthropicModel
+    from .gemini_client import GeminiModel
+    from .ollama_client import OllamaModel
+    from .openai_client import OpenAICompatModel
+
+    for cls in (OpenAICompatModel, AnthropicModel, GeminiModel, OllamaModel, MockModel):
+        if cls.provider_name not in PROVIDERS:
+            PROVIDERS[cls.provider_name] = cls
+
+
+def load_registry(path: str | Path | None = None) -> dict[str, ModelConfig]:
+    _build_providers()
+    if path is None:
+        path = Path(__file__).resolve().parents[2] / "config" / "models.yaml"
+    path = Path(path)
+    if not path.exists():
+        raise FileNotFoundError(f"model registry not found: {path}")
+    raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    registry: dict[str, ModelConfig] = {}
+    for name, spec in raw.get("models", {}).items():
+        if spec is None:
+            continue
+        if spec.get("enabled") is False:
+            continue
+        spec = {k: v for k, v in spec.items() if k != "enabled"}
+        provider = spec.get("provider")
+        if provider not in PROVIDERS:
+            raise ValueError(
+                f"model '{name}': unknown provider '{provider}', "
+                f"expected one of {sorted(PROVIDERS)}"
+            )
+        cfg = ModelConfig(name=name, **spec)
+        registry[name] = cfg
+    return registry
+
+
+def build_model(name: str, registry: dict[str, ModelConfig]) -> BaseVLModel:
+    _build_providers()
+    if name not in registry:
+        raise KeyError(f"model '{name}' not in registry; available: {sorted(registry)}")
+    cfg = registry[name]
+    return PROVIDERS[cfg.provider](cfg)

vlmeter/models/anthropic_client.py

@@ -0,0 +1,51 @@
+"""Anthropic Messages API adapter (Claude models), called via httpx."""
+from __future__ import annotations
+
+import time
+
+import httpx
+
+from ..images import b64_image
+from .base import BaseVLModel, ModelResponse
+
+
+class AnthropicModel(BaseVLModel):
+    provider_name = "anthropic"
+
+    async def chat(self, image_path: str | None, prompt: str) -> ModelResponse:
+        blocks: list[dict] = []
+        if image_path:
+            b64, mime = b64_image(image_path)
+            blocks.append(
+                {"type": "image", "source": {"type": "base64", "media_type": mime, "data": b64}}
+            )
+        blocks.append({"type": "text", "text": prompt})
+        base = (self.cfg.resolve_base_url() or "https://api.anthropic.com").rstrip("/")
+        start = time.perf_counter()
+        async with httpx.AsyncClient(timeout=self.cfg.timeout) as client:
+            resp = await client.post(
+                f"{base}/v1/messages",
+                headers={
+                    "x-api-key": self.cfg.resolve_api_key() or "",
+                    "anthropic-version": "2023-06-01",
+                    "content-type": "application/json",
+                },
+                json={
+                    "model": self.cfg.model,
+                    "max_tokens": self.cfg.max_tokens,
+                    "temperature": self.cfg.temperature,
+                    "messages": [{"role": "user", "content": blocks}],
+                },
+            )
+            resp.raise_for_status()
+            data = resp.json()
+        latency = int((time.perf_counter() - start) * 1000)
+        text = "".join(b.get("text", "") for b in data.get("content", [])).strip()
+        usage = data.get("usage") or {}
+        in_tok = usage.get("input_tokens")
+        out_tok = usage.get("output_tokens")
+        tokens = (in_tok or 0) + (out_tok or 0) or None
+        return ModelResponse(
+            text=text, latency_ms=latency, tokens=tokens,
+            input_tokens=in_tok, output_tokens=out_tok,
+        )

vlmeter/models/base.py

@@ -0,0 +1,72 @@
+"""Model adapter interface and shared call-result type."""
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class ModelResponse:
+    text: str
+    latency_ms: int = 0
+    tokens: int | None = None              # total (fallback when no split)
+    input_tokens: int | None = None
+    output_tokens: int | None = None
+    finish_reason: str | None = None
+    raw: Any = field(default=None, repr=False)
+
+    @property
+    def ok(self) -> bool:
+        return bool(self.text.strip())
+
+
+@dataclass
+class ModelConfig:
+    name: str                       # registry key / display name
+    provider: str                   # openai | anthropic | gemini | ollama | mock
+    model: str                      # provider-side model id
+    base_url: str | None = None
+    base_url_env: str | None = None  # read base_url from this env var (overrides base_url)
+    api_key_env: str | None = None
+    api_key_default: str = "EMPTY"   # used when the env var is unset (vLLM ignores keys)
+    max_tokens: int = 2048
+    temperature: float = 0.0
+    timeout: float = 120.0
+    concurrency: int = 4
+    price_input: float | None = None   # 元 / 每百万输入 tokens（按官网价核对）
+    price_output: float | None = None  # 元 / 每百万输出 tokens
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    def estimate_cost(self, input_tokens: int, output_tokens: int) -> float | None:
+        """估算单次调用成本（元）；未配置单价返回 None。"""
+        if self.price_input is None or self.price_output is None:
+            return None
+        return (input_tokens * self.price_input + output_tokens * self.price_output) / 1_000_000
+
+    def resolve_base_url(self) -> str | None:
+        if self.base_url_env:
+            return os.environ.get(self.base_url_env) or self.base_url
+        return self.base_url
+
+    def resolve_api_key(self) -> str | None:
+        if self.api_key_env:
+            return os.environ.get(self.api_key_env) or self.api_key_default
+        return self.api_key_default
+
+
+class BaseVLModel:
+    """A vision-language chat model. Implementations must be safe to share
+    across asyncio tasks (create per-call clients or reuse an httpx client)."""
+
+    provider_name: str = "base"
+
+    def __init__(self, cfg: ModelConfig):
+        self.cfg = cfg
+
+    @property
+    def name(self) -> str:
+        return self.cfg.name
+
+    async def chat(self, image_path: str | None, prompt: str) -> ModelResponse:
+        raise NotImplementedError