sothstan 0.1.0rc1__py3-none-any.whl

sothstan/__init__.py

@@ -0,0 +1,29 @@
+"""Sothstan — LLM API 模型验真。
+
+一个端点声称自己在服务模型 X，它真的在服务 X 吗？
+sothstan 用探测（probe）提取指纹信号（signal），与官方模型指纹基线（baseline）
+做混淆集似然比比较（confusion-set likelihood ratio），给出统计判决（verdict）。
+"""
+
+from sothstan._version import __version__
+from sothstan.baseline import Baseline, load_baseline, save_baseline
+from sothstan.http import ApiClient, ApiError, BudgetExceeded
+from sothstan.runner import collect_baseline_signals, verify
+from sothstan.types import Signal, VerdictLabel
+from sothstan.verdict import Verdict, decide
+
+__all__ = [
+    "ApiError",
+    "ApiClient",
+    "Baseline",
+    "BudgetExceeded",
+    "Signal",
+    "Verdict",
+    "VerdictLabel",
+    "collect_baseline_signals",
+    "decide",
+    "load_baseline",
+    "save_baseline",
+    "verify",
+    "__version__",
+]

sothstan/_version.py

@@ -0,0 +1,3 @@
+"""版本号单一定义点（避免循环导入）。"""
+
+__version__ = "0.1.0rc1"

sothstan/audit.py

@@ -0,0 +1,210 @@
+"""批量审计：一个 CSV 目标清单 → 受控并发验真 → 可发布的汇总报告。
+
+CSV 格式（utf-8-sig，含表头）：name,base_url,model,key_env
+- key_env 可为空（匿名请求）
+- 同一 model 的多行共享注册表基线；混淆集 = 注册表中除该模型外的全部基线
+  （与 `sothstan check` 同规则：demo 基线默认排除）
+
+退出码约定（cmd_audit）：0=全部 AUTHENTIC；1=含 SUSPICIOUS；2=含 MISMATCH；
+3=全部 INCONCLUSIVE/失败；4=基础设施错误（文件/表头/基线全缺）。
+"""
+
+from __future__ import annotations
+
+import csv
+import os
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+from sothstan._version import __version__
+from sothstan.baseline import Baseline, load_registry
+from sothstan.report import HONESTY_APPENDIX, result_to_dict
+from sothstan.runner import verify
+from sothstan.verdict import VerdictLabel
+
+REQUIRED_COLUMNS = {"name", "base_url", "model", "key_env"}
+
+
+@dataclass
+class AuditTarget:
+    name: str
+    base_url: str
+    model: str
+    key_env: str = ""
+
+
+@dataclass
+class AuditReport:
+    rows: list[dict] = field(default_factory=list)
+    created_at: str = ""
+    seed: int = 0
+    summary: dict = field(default_factory=dict)
+
+
+def load_targets(path: str | Path) -> tuple[list[AuditTarget], list[str]]:
+    """解析目标 CSV，返回 (targets, problems)。缺列/空行/缺值如实收集。"""
+    problems: list[str] = []
+    targets: list[AuditTarget] = []
+    with open(path, encoding="utf-8-sig", newline="") as f:
+        reader = csv.DictReader(f)
+        if reader.fieldnames is None or not REQUIRED_COLUMNS.issubset(set(reader.fieldnames)):
+            return [], [f"表头缺列：需要 {sorted(REQUIRED_COLUMNS)}，实际 {reader.fieldnames}"]
+        for i, row in enumerate(reader, start=2):
+            name = (row.get("name") or "").strip()
+            base_url = (row.get("base_url") or "").strip()
+            model = (row.get("model") or "").strip()
+            key_env = (row.get("key_env") or "").strip()
+            if not name and not base_url and not model:
+                continue  # 整行空白，跳过
+            if not name or not base_url or not model:
+                problems.append(f"第 {i} 行：name/base_url/model 均不能为空")
+                continue
+            targets.append(AuditTarget(name=name, base_url=base_url, model=model, key_env=key_env))
+    if not targets and not problems:
+        problems.append("清单为空：没有任何目标")
+    return targets, problems
+
+
+def _rivals_for(model: str, registry: dict[str, Baseline], include_demo: bool) -> list[Baseline]:
+    return [
+        b for mid, b in registry.items()
+        if mid != model and (include_demo or not b.is_demo)
+    ]
+
+
+def audit_one(
+    target: AuditTarget,
+    registry: dict[str, Baseline],
+    include_demo: bool,
+    seed: int,
+    timeout: float,
+    retries: int,
+    max_requests: int,
+    max_prompt_tokens: int,
+    adversarial: bool,
+) -> dict:
+    """验真单个目标，返回一行报告数据（不抛异常——失败也是一行）。"""
+    row: dict = {
+        "name": target.name,
+        "base_url": target.base_url,
+        "claimed_model": target.model,
+        "error": None,
+        "report": None,
+    }
+    baseline = registry.get(target.model)
+    if baseline is None:
+        row["error"] = f"注册表中没有 {target.model} 的基线"
+        return row
+    api_key = os.environ.get(target.key_env) if target.key_env else None
+    result = verify(
+        base_url=target.base_url,
+        model=target.model,
+        baseline=baseline,
+        rivals=_rivals_for(target.model, registry, include_demo),
+        api_key=api_key,
+        seed=seed,
+        timeout=timeout,
+        retries=retries,
+        max_requests=max_requests,
+        max_prompt_tokens=max_prompt_tokens,
+        options={"adversarial": True} if adversarial else None,
+    )
+    row["report"] = result_to_dict(result, target.base_url, target.model, seed)
+    return row
+
+
+def run_audit(
+    targets: list[AuditTarget],
+    registry_dir: str | Path | None = None,
+    include_demo: bool = False,
+    seed: int = 20260928,
+    max_workers: int = 4,
+    timeout: float = 30.0,
+    retries: int = 2,
+    max_requests: int = 64,
+    max_prompt_tokens: int = 200_000,
+    adversarial: bool = False,
+) -> AuditReport:
+    """受控并发跑批。注册表只加载一次，逐目标共享。"""
+    registry = load_registry(registry_dir)
+    report = AuditReport(
+        created_at=datetime.now(timezone.utc).isoformat(timespec="seconds"),
+        seed=seed,
+    )
+    if not registry:
+        report.summary = {"error": "注册表为空：没有任何可用基线"}
+        return report
+
+    def _one(t: AuditTarget) -> dict:
+        return audit_one(
+            t, registry, include_demo, seed, timeout, retries,
+            max_requests, max_prompt_tokens, adversarial,
+        )
+
+    with ThreadPoolExecutor(max_workers=max(1, max_workers)) as pool:
+        report.rows = list(pool.map(_one, targets))
+
+    labels = [
+        r["report"]["verdict"]["label"] for r in report.rows if r["report"] is not None
+    ]
+    report.summary = {
+        "total": len(report.rows),
+        "AUTHENTIC": labels.count(VerdictLabel.AUTHENTIC.value),
+        "SUSPICIOUS": labels.count(VerdictLabel.SUSPICIOUS.value),
+        "MISMATCH": labels.count(VerdictLabel.MISMATCH.value),
+        "INCONCLUSIVE": labels.count(VerdictLabel.INCONCLUSIVE.value),
+        "errors": sum(1 for r in report.rows if r["error"]),
+        "registry_models": sorted(registry),
+    }
+    return report
+
+
+def render_audit_markdown(report: AuditReport) -> str:
+    """可发布的汇总报告：摘要表 + 声明。逐端点完整证据在 JSON。"""
+    s = report.summary
+    lines = [
+        "# Sothstan批量审计报告",
+        "",
+        f"- 生成时间：{report.created_at}（sothstan {__version__}）",
+        f"- seed：{report.seed}（同 seed 同探测集，证据链可复现）",
+        f"- 目标：{s.get('total', 0)} 个｜"
+        f"AUTHENTIC {s.get('AUTHENTIC', 0)}｜SUSPICIOUS {s.get('SUSPICIOUS', 0)}｜"
+        f"MISMATCH {s.get('MISMATCH', 0)}｜INCONCLUSIVE {s.get('INCONCLUSIVE', 0)}｜"
+        f"失败 {s.get('errors', 0)}",
+        f"- 混淆集基线：{', '.join(s.get('registry_models', [])) or '（无）'}",
+        "",
+        "| 目标 | 声称模型 | 判决 | margin | 最强竞争解释 | 证据链 |",
+        "|---|---|---|---|---|---|",
+    ]
+    for r in report.rows:
+        rep = r["report"]
+        if rep is None:
+            lines.append(f"| {r['name']} | {r['claimed_model']} | 失败 | — | — | {r['error']} |")
+            continue
+        v = rep["verdict"]
+        lines.append(
+            f"| {r['name']} | {v['claimed_model']} | **{v['label']}** "
+            f"| {v['margin_total']:+.1f} | {v['runner_up'] or '—'} "
+            f"| `{rep['transcript_sha256'][:8]}…` |"
+        )
+    lines += [
+        "",
+        "> 逐端点完整信号证据表见同报告的 JSON（含每个信号的观测值/基线值/竞争者得分）。",
+        "",
+        HONESTY_APPENDIX,
+        "",
+    ]
+    return "\n".join(lines)
+
+
+def audit_exit_code(report: AuditReport) -> int:
+    s = report.summary
+    if s.get("MISMATCH"):
+        return 2
+    if s.get("SUSPICIOUS"):
+        return 1
+    if s.get("total") and s.get("AUTHENTIC"):
+        return 0
+    return 3  # 全部 INCONCLUSIVE/失败/空

sothstan/baseline.py

@@ -0,0 +1,165 @@
+"""基线：官方模型指纹的版本化存储。
+
+一个 Baseline = 一个官方模型在某一时刻的完整指纹快照。
+注册表（registry 目录）里的全部基线互为"混淆集"候选。
+
+诚实边界：基线必须用 `sothstan collect` 对**官方端点**采集；
+仓库内置的 `_demo_*.json` 描述的是本库自带 mock 人格，仅用于离线
+测试与 selftest，绝不可当作任何真实模型的指纹使用。
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from sothstan.types import SIGNAL_KINDS
+
+SCHEMA_VERSION = 1
+PACKAGE_REGISTRY = Path(__file__).parent / "baselines"
+
+
+@dataclass
+class SignalSpec:
+    name: str
+    kind: str
+    value: object
+    weight: float = 1.0
+    meta: dict = field(default_factory=dict)
+
+
+@dataclass
+class Baseline:
+    model_id: str
+    family: str
+    signals: dict[str, SignalSpec]
+    collected_at: str = ""
+    collector: str = ""
+    endpoint_hint: str = ""
+    tags: list[str] = field(default_factory=list)
+    honesty_note: str = ""
+
+    def get_signal(self, name: str) -> SignalSpec | None:
+        return self.signals.get(name)
+
+    @property
+    def is_demo(self) -> bool:
+        return "demo" in self.tags
+
+    def to_dict(self) -> dict:
+        return {
+            "schema_version": SCHEMA_VERSION,
+            "model_id": self.model_id,
+            "family": self.family,
+            "tags": self.tags,
+            "collected_at": self.collected_at,
+            "collector": self.collector,
+            "endpoint_hint": self.endpoint_hint,
+            "honesty_note": self.honesty_note,
+            "signals": {
+                name: {
+                    "kind": s.kind,
+                    "value": s.value,
+                    "weight": s.weight,
+                    "meta": s.meta,
+                }
+                for name, s in sorted(self.signals.items())
+            },
+        }
+
+
+def validate_baseline_dict(data: dict) -> list[str]:
+    """返回问题列表；空列表 = 合法。"""
+    problems: list[str] = []
+    if not isinstance(data, dict):
+        return ["顶层必须是对象"]
+    if data.get("schema_version") != SCHEMA_VERSION:
+        problems.append(f"schema_version 必须为 {SCHEMA_VERSION}")
+    if not data.get("model_id") or not isinstance(data.get("model_id"), str):
+        problems.append("model_id 缺失或非字符串")
+    if not isinstance(data.get("family"), str) or not data.get("family"):
+        problems.append("family 缺失")
+    signals = data.get("signals")
+    if not isinstance(signals, dict) or not signals:
+        problems.append("signals 必须为非空对象")
+        return problems
+    for name, spec in signals.items():
+        if not isinstance(spec, dict):
+            problems.append(f"{name}: 必须为对象")
+            continue
+        weight = spec.get("weight", 1.0)
+        if not isinstance(weight, (int, float)) or isinstance(weight, bool):
+            problems.append(f"{name}: weight 必须为数值")
+        kind = spec.get("kind")
+        if kind not in SIGNAL_KINDS:
+            problems.append(f"{name}: 非法 kind {kind!r}")
+            continue
+        value = spec.get("value")
+        if kind == "int_vector" and not (isinstance(value, list) and value):
+            problems.append(f"{name}: int_vector 需要非空数组")
+        if kind == "int" and not isinstance(value, int):
+            problems.append(f"{name}: int 需要整数")
+        if kind == "categorical" and not isinstance(value, str):
+            problems.append(f"{name}: categorical 需要字符串")
+        if kind == "bool" and not isinstance(value, bool):
+            problems.append(f"{name}: bool 需要布尔值")
+    return problems
+
+
+def baseline_from_dict(data: dict) -> Baseline:
+    problems = validate_baseline_dict(data)
+    if problems:
+        raise ValueError("基线不合法: " + "; ".join(problems))
+    signals = {
+        name: SignalSpec(
+            name=name,
+            kind=spec["kind"],
+            value=spec["value"],
+            weight=float(spec.get("weight", 1.0)),
+            meta=spec.get("meta", {}),
+        )
+        for name, spec in data["signals"].items()
+    }
+    return Baseline(
+        model_id=data["model_id"],
+        family=data["family"],
+        signals=signals,
+        collected_at=data.get("collected_at", ""),
+        collector=data.get("collector", ""),
+        endpoint_hint=data.get("endpoint_hint", ""),
+        tags=list(data.get("tags", [])),
+        honesty_note=data.get("honesty_note", ""),
+    )
+
+
+def load_baseline(path: str | Path) -> Baseline:
+    p = Path(path)
+    data = json.loads(p.read_text(encoding="utf-8"))
+    return baseline_from_dict(data)
+
+
+def save_baseline(baseline: Baseline, path: str | Path) -> Path:
+    p = Path(path)
+    p.parent.mkdir(parents=True, exist_ok=True)
+    p.write_text(
+        json.dumps(baseline.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
+    )
+    return p
+
+
+def load_registry(directory: str | Path | None = None) -> dict[str, Baseline]:
+    """加载目录下全部 *.json 基线，按 model_id 索引。坏文件如实跳过并收集错误。"""
+    out: dict[str, Baseline] = {}
+    errors: list[str] = []
+    d = Path(directory) if directory is not None else PACKAGE_REGISTRY
+    if not d.exists():
+        return out
+    for f in sorted(d.glob("*.json")):
+        try:
+            b = baseline_from_dict(json.loads(f.read_text(encoding="utf-8")))
+        except (ValueError, json.JSONDecodeError) as exc:
+            errors.append(f"{f.name}: {exc}")
+            continue
+        out[b.model_id] = b
+    return out

sothstan/baselines/_demo_aqua-70b.json

@@ -0,0 +1,163 @@
+{
+  "schema_version": 1,
+  "model_id": "aqua-70b",
+  "family": "aqua",
+  "tags": [
+    "demo"
+  ],
+  "collected_at": "2026-09-28T13:34:11+00:00",
+  "collector": "sothstan 0.1.0rc1",
+  "endpoint_hint": "",
+  "honesty_note": "内置 mock 人格的指纹，仅用于离线测试与 selftest，不代表任何真实模型。",
+  "signals": {
+    "errors.empty_messages": {
+      "kind": "categorical",
+      "value": "400|invalid_parameter__messages_must",
+      "weight": 1.0,
+      "meta": {}
+    },
+    "errors.invalid_role": {
+      "kind": "categorical",
+      "value": "400|invalid_parameter__invalid_role",
+      "weight": 1.0,
+      "meta": {}
+    },
+    "errors.temperature_out_of_range": {
+      "kind": "categorical",
+      "value": "400|invalid_parameter__temperature_m",
+      "weight": 1.0,
+      "meta": {}
+    },
+    "errors.unknown_param": {
+      "kind": "categorical",
+      "value": "400|invalid_parameter__unknown_param",
+      "weight": 1.0,
+      "meta": {}
+    },
+    "limits.logprobs_support": {
+      "kind": "bool",
+      "value": true,
+      "weight": 1.0,
+      "meta": {}
+    },
+    "limits.max_tokens_huge": {
+      "kind": "categorical",
+      "value": "accepted",
+      "weight": 1.0,
+      "meta": {}
+    },
+    "limits.model_field_echo": {
+      "kind": "categorical",
+      "value": "aqua-70b",
+      "weight": 0.0,
+      "meta": {}
+    },
+    "limits.n_param_support": {
+      "kind": "bool",
+      "value": true,
+      "weight": 1.0,
+      "meta": {}
+    },
+    "reasoning.completion_per_char": {
+      "kind": "int",
+      "value": 493,
+      "weight": 1.0,
+      "meta": {}
+    },
+    "reasoning.completion_tokens": {
+      "kind": "int",
+      "value": 39,
+      "weight": 1.0,
+      "meta": {}
+    },
+    "reasoning.reasoning_field": {
+      "kind": "bool",
+      "value": false,
+      "weight": 1.0,
+      "meta": {}
+    },
+    "reasoning.style_marker": {
+      "kind": "categorical",
+      "value": "steps_with_final",
+      "weight": 1.0,
+      "meta": {}
+    },
+    "template.empty_user_total": {
+      "kind": "int",
+      "value": 4,
+      "weight": 1.5,
+      "meta": {}
+    },
+    "template.pair_growth": {
+      "kind": "int",
+      "value": 48,
+      "weight": 1.0,
+      "meta": {}
+    },
+    "template.system_overhead": {
+      "kind": "int",
+      "value": 17,
+      "weight": 1.0,
+      "meta": {}
+    },
+    "token_count.completion_int": {
+      "kind": "int",
+      "value": 7,
+      "weight": 1.0,
+      "meta": {
+        "finish_reason": "stop"
+      }
+    },
+    "token_count.prompt_curve": {
+      "kind": "int_vector",
+      "value": [
+        11,
+        17,
+        41,
+        48,
+        103,
+        91,
+        28,
+        31,
+        26,
+        22,
+        33,
+        33,
+        32,
+        9,
+        23,
+        34,
+        33,
+        27,
+        41,
+        43
+      ],
+      "weight": 2.0,
+      "meta": {
+        "canon_ids": [
+          "zh_short_1",
+          "zh_short_2",
+          "zh_mid_1",
+          "zh_mid_2",
+          "zh_long_1",
+          "zh_long_2",
+          "mix_zh_en_1",
+          "mix_zh_en_2",
+          "numbers_1",
+          "numbers_2",
+          "fullwidth_1",
+          "fullwidth_2",
+          "emoji_1",
+          "emoji_2",
+          "rare_cjk_1",
+          "rare_cjk_2",
+          "code_1",
+          "code_2",
+          "repeat_1",
+          "mixed_all"
+        ],
+        "mode": "canon"
+      }
+    }
+  }
+}