modelcheck 1.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modelcheck/__init__.py +4 -0
- modelcheck/__main__.py +3 -0
- modelcheck/checks/__init__.py +12 -0
- modelcheck/checks/capabilities/__init__.py +1 -0
- modelcheck/checks/capabilities/check.py +64 -0
- modelcheck/checks/forty_questions/__init__.py +0 -0
- modelcheck/checks/forty_questions/check.py +79 -0
- modelcheck/checks/hidden_prompt/__init__.py +0 -0
- modelcheck/checks/hidden_prompt/check.py +52 -0
- modelcheck/checks/identity/__init__.py +1 -0
- modelcheck/checks/identity/check.py +118 -0
- modelcheck/checks/late_turns/__init__.py +0 -0
- modelcheck/checks/late_turns/check.py +71 -0
- modelcheck/checks/model_field/__init__.py +1 -0
- modelcheck/checks/model_field/check.py +67 -0
- modelcheck/checks/random_number/__init__.py +1 -0
- modelcheck/checks/random_number/check.py +70 -0
- modelcheck/checks/reasoning/__init__.py +0 -0
- modelcheck/checks/reasoning/check.py +104 -0
- modelcheck/checks/response_shape/__init__.py +1 -0
- modelcheck/checks/response_shape/check.py +89 -0
- modelcheck/checks/thinking_signature/__init__.py +1 -0
- modelcheck/checks/thinking_signature/check.py +66 -0
- modelcheck/checks/tokenizer/__init__.py +1 -0
- modelcheck/checks/tokenizer/check.py +89 -0
- modelcheck/cli.py +207 -0
- modelcheck/core/__init__.py +1 -0
- modelcheck/core/baselines.py +60 -0
- modelcheck/core/check.py +30 -0
- modelcheck/core/client.py +170 -0
- modelcheck/core/engine.py +97 -0
- modelcheck/core/rand.py +52 -0
- modelcheck/core/types.py +79 -0
- modelcheck/makers/__init__.py +69 -0
- modelcheck/makers/alibaba/__init__.py +14 -0
- modelcheck/makers/amazon/__init__.py +9 -0
- modelcheck/makers/anthropic/__init__.py +15 -0
- modelcheck/makers/bytedance/__init__.py +12 -0
- modelcheck/makers/cohere/__init__.py +9 -0
- modelcheck/makers/deepseek/__init__.py +14 -0
- modelcheck/makers/google/__init__.py +14 -0
- modelcheck/makers/meta/__init__.py +12 -0
- modelcheck/makers/microsoft/__init__.py +11 -0
- modelcheck/makers/minimax/__init__.py +12 -0
- modelcheck/makers/mistral/__init__.py +12 -0
- modelcheck/makers/moonshot/__init__.py +14 -0
- modelcheck/makers/nvidia/__init__.py +10 -0
- modelcheck/makers/openai/__init__.py +14 -0
- modelcheck/makers/sber/__init__.py +10 -0
- modelcheck/makers/xai/__init__.py +13 -0
- modelcheck/makers/yandex/__init__.py +10 -0
- modelcheck/makers/zhipu/__init__.py +14 -0
- modelcheck/report/__init__.py +1 -0
- modelcheck/report/render.py +98 -0
- modelcheck-1.0.1.dist-info/METADATA +87 -0
- modelcheck-1.0.1.dist-info/RECORD +59 -0
- modelcheck-1.0.1.dist-info/WHEEL +4 -0
- modelcheck-1.0.1.dist-info/entry_points.txt +2 -0
- modelcheck-1.0.1.dist-info/licenses/LICENSE +9 -0
modelcheck/core/check.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Базовый класс проверки и реестр всех проверок."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from abc import ABC, abstractmethod
|
|
5
|
+
|
|
6
|
+
from .client import Client
|
|
7
|
+
from .types import Finding, Target
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Check(ABC):
|
|
11
|
+
name: str = "" # id проверки (папка)
|
|
12
|
+
title: str = "" # по-русски для отчёта
|
|
13
|
+
protocols: tuple[str, ...] = ("anthropic", "openai")
|
|
14
|
+
makers: tuple[str, ...] = () # пусто = для всех производителей
|
|
15
|
+
level: int = 1 # 1 быстрая, 2 стандарт, 3 полная (дороже)
|
|
16
|
+
|
|
17
|
+
def applies(self, target: Target, maker: str) -> bool:
|
|
18
|
+
return target.protocol in self.protocols and (not self.makers or maker in self.makers)
|
|
19
|
+
|
|
20
|
+
@abstractmethod
|
|
21
|
+
def run(self, client: Client, target: Target, maker: str) -> list[Finding]:
|
|
22
|
+
...
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
REGISTRY: list[type[Check]] = []
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def register(cls: type[Check]) -> type[Check]:
|
|
29
|
+
REGISTRY.append(cls)
|
|
30
|
+
return cls
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
"""HTTP-клиенты для двух протоколов. Проверки не знают про URL — только про Reply."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import time
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
import httpx
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class Reply:
|
|
12
|
+
status: int
|
|
13
|
+
text: str = "" # итоговый текст ответа
|
|
14
|
+
model: str = "" # поле model из ответа
|
|
15
|
+
id: str = "" # id ответа
|
|
16
|
+
usage: dict = field(default_factory=dict)
|
|
17
|
+
thinking: list[dict] = field(default_factory=list) # блоки размышлений (anthropic)
|
|
18
|
+
tool_calls: list[dict] = field(default_factory=list)
|
|
19
|
+
extra: dict = field(default_factory=dict) # нестандартные поля (reasoning_content, system_fingerprint)
|
|
20
|
+
headers: dict = field(default_factory=dict)
|
|
21
|
+
raw: dict = field(default_factory=dict)
|
|
22
|
+
error: str = ""
|
|
23
|
+
latency: float = 0.0
|
|
24
|
+
|
|
25
|
+
@property
|
|
26
|
+
def ok(self) -> bool:
|
|
27
|
+
return 200 <= self.status < 300 and not self.error
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class Client:
|
|
31
|
+
"""Базовый клиент: считает запросы, ловит сетевые ошибки."""
|
|
32
|
+
|
|
33
|
+
def __init__(self, base_url: str, api_key: str, timeout: float = 45, retries: int = 2):
|
|
34
|
+
self.base = base_url.rstrip("/")
|
|
35
|
+
if self.base.endswith("/v1"):
|
|
36
|
+
self.base = self.base[:-3]
|
|
37
|
+
self.key = api_key
|
|
38
|
+
self.http = httpx.Client(timeout=timeout, headers={"user-agent": "modelcheck"})
|
|
39
|
+
self.retries = retries
|
|
40
|
+
self.count = 0
|
|
41
|
+
|
|
42
|
+
def _post(self, path: str, headers: dict, body: dict) -> tuple[int, dict, dict, float, str]:
|
|
43
|
+
"""POST с повтором на временные сбои (429, 5xx, обрыв): флапающий сервис ≠ подмена."""
|
|
44
|
+
for attempt in range(self.retries + 1):
|
|
45
|
+
self.count += 1
|
|
46
|
+
t = time.perf_counter()
|
|
47
|
+
try:
|
|
48
|
+
r = self.http.post(self.base + path, headers=headers, json=body)
|
|
49
|
+
except httpx.HTTPError as e:
|
|
50
|
+
if attempt < self.retries:
|
|
51
|
+
time.sleep(1.5 * (attempt + 1))
|
|
52
|
+
continue
|
|
53
|
+
return 0, {}, {}, time.perf_counter() - t, f"{type(e).__name__}: {e}"
|
|
54
|
+
dt = time.perf_counter() - t
|
|
55
|
+
if (r.status_code == 429 or r.status_code >= 500) and attempt < self.retries:
|
|
56
|
+
time.sleep(1.5 * (attempt + 1))
|
|
57
|
+
continue
|
|
58
|
+
try:
|
|
59
|
+
return r.status_code, r.json(), dict(r.headers), dt, ""
|
|
60
|
+
except ValueError:
|
|
61
|
+
return r.status_code, {}, dict(r.headers), dt, f"не JSON: {r.text[:200]}"
|
|
62
|
+
return 0, {}, {}, 0.0, "нет ответа"
|
|
63
|
+
|
|
64
|
+
def models(self) -> tuple[int, dict]:
|
|
65
|
+
self.count += 1
|
|
66
|
+
try:
|
|
67
|
+
r = self.http.get(self.base + "/v1/models", headers=self._auth())
|
|
68
|
+
return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {})
|
|
69
|
+
except (httpx.HTTPError, ValueError):
|
|
70
|
+
return 0, {}
|
|
71
|
+
|
|
72
|
+
def _auth(self) -> dict:
|
|
73
|
+
raise NotImplementedError
|
|
74
|
+
|
|
75
|
+
def chat(self, model: str, messages: list[dict], **kw) -> Reply:
|
|
76
|
+
raise NotImplementedError
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class AnthropicClient(Client):
|
|
80
|
+
def _auth(self) -> dict:
|
|
81
|
+
return {"x-api-key": self.key, "anthropic-version": "2023-06-01", "content-type": "application/json"}
|
|
82
|
+
|
|
83
|
+
def chat(self, model, messages, max_tokens=300, system=None, thinking=None, tools=None, temperature=None, **_) -> Reply:
|
|
84
|
+
body = {"model": model, "max_tokens": max_tokens, "messages": messages}
|
|
85
|
+
if system:
|
|
86
|
+
body["system"] = system
|
|
87
|
+
if thinking:
|
|
88
|
+
body["thinking"] = {"type": "enabled", "budget_tokens": thinking}
|
|
89
|
+
if tools:
|
|
90
|
+
body["tools"] = tools
|
|
91
|
+
if temperature is not None and not thinking:
|
|
92
|
+
body["temperature"] = temperature
|
|
93
|
+
st, d, h, dt, err = self._post("/v1/messages", self._auth(), body)
|
|
94
|
+
rep = Reply(status=st, raw=d, headers=h, latency=dt, error=err)
|
|
95
|
+
if err or "content" not in d:
|
|
96
|
+
rep.error = rep.error or _err(d)
|
|
97
|
+
return rep
|
|
98
|
+
for b in d.get("content", []):
|
|
99
|
+
if b.get("type") == "text":
|
|
100
|
+
rep.text += b.get("text", "")
|
|
101
|
+
elif b.get("type") in ("thinking", "redacted_thinking"):
|
|
102
|
+
rep.thinking.append(b)
|
|
103
|
+
elif b.get("type") == "tool_use":
|
|
104
|
+
rep.tool_calls.append(b)
|
|
105
|
+
rep.model, rep.id, rep.usage = d.get("model", ""), d.get("id", ""), d.get("usage") or {}
|
|
106
|
+
return rep
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class OpenAIClient(Client):
|
|
110
|
+
def _auth(self) -> dict:
|
|
111
|
+
return {"Authorization": f"Bearer {self.key}", "content-type": "application/json"}
|
|
112
|
+
|
|
113
|
+
def chat(self, model, messages, max_tokens=300, system=None, thinking=None, tools=None, temperature=None, **_) -> Reply:
|
|
114
|
+
msgs = ([{"role": "system", "content": system}] if system else []) + messages
|
|
115
|
+
body = {"model": model, "max_tokens": max_tokens, "messages": msgs}
|
|
116
|
+
if tools:
|
|
117
|
+
body["tools"] = [{"type": "function", "function": {"name": t["name"], "description": t["description"],
|
|
118
|
+
"parameters": t["input_schema"]}} for t in tools]
|
|
119
|
+
if temperature is not None:
|
|
120
|
+
body["temperature"] = temperature
|
|
121
|
+
st, d, h, dt, err = self._post("/v1/chat/completions", self._auth(), body)
|
|
122
|
+
rep = Reply(status=st, raw=d, headers=h, latency=dt, error=err)
|
|
123
|
+
if err or not d.get("choices"):
|
|
124
|
+
rep.error = rep.error or _err(d)
|
|
125
|
+
return rep
|
|
126
|
+
msg = d["choices"][0].get("message") or {}
|
|
127
|
+
rep.text = msg.get("content") or ""
|
|
128
|
+
rep.tool_calls = msg.get("tool_calls") or []
|
|
129
|
+
rep.model, rep.id, rep.usage = d.get("model", ""), d.get("id", ""), d.get("usage") or {}
|
|
130
|
+
for k in ("reasoning_content", "reasoning"):
|
|
131
|
+
if msg.get(k):
|
|
132
|
+
rep.extra[k] = msg[k]
|
|
133
|
+
if "system_fingerprint" in d:
|
|
134
|
+
rep.extra["system_fingerprint"] = d["system_fingerprint"]
|
|
135
|
+
return rep
|
|
136
|
+
|
|
137
|
+
def responses(self, model, text, include_encrypted=False, previous_items=None) -> Reply:
|
|
138
|
+
"""OpenAI / xAI Responses API (/v1/responses) — нужен для проверки зашифрованных размышлений."""
|
|
139
|
+
inp = list(previous_items or []) + [{"role": "user", "content": text}]
|
|
140
|
+
body = {"model": model, "input": inp, "store": False, "max_output_tokens": 2000}
|
|
141
|
+
if include_encrypted:
|
|
142
|
+
body["include"] = ["reasoning.encrypted_content"]
|
|
143
|
+
body["reasoning"] = {"effort": "low"}
|
|
144
|
+
st, d, h, dt, err = self._post("/v1/responses", self._auth(), body)
|
|
145
|
+
rep = Reply(status=st, raw=d, headers=h, latency=dt, error=err)
|
|
146
|
+
if err or "output" not in d:
|
|
147
|
+
rep.error = rep.error or _err(d)
|
|
148
|
+
return rep
|
|
149
|
+
items = d.get("output") or []
|
|
150
|
+
rep.extra["output_items"] = items
|
|
151
|
+
for it in items:
|
|
152
|
+
if it.get("type") == "reasoning" and it.get("encrypted_content"):
|
|
153
|
+
rep.extra["encrypted_reasoning"] = it["encrypted_content"]
|
|
154
|
+
if it.get("type") == "message":
|
|
155
|
+
for c in it.get("content") or []:
|
|
156
|
+
rep.text += c.get("text", "")
|
|
157
|
+
rep.model, rep.id, rep.usage = d.get("model", ""), d.get("id", ""), d.get("usage") or {}
|
|
158
|
+
return rep
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _err(d: dict) -> str:
|
|
162
|
+
e = d.get("error") if isinstance(d, dict) else None
|
|
163
|
+
if isinstance(e, dict):
|
|
164
|
+
return e.get("message") or e.get("type") or str(e)
|
|
165
|
+
return str(e or d)[:200]
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def make_client(protocol: str, base_url: str, api_key: str, timeout: float = 45) -> Client:
|
|
169
|
+
cls = AnthropicClient if protocol == "anthropic" else OpenAIClient
|
|
170
|
+
return cls(base_url, api_key, timeout=timeout)
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""Запуск проверок и итоговый вердикт по двум осям.
|
|
2
|
+
|
|
3
|
+
Ось MODEL — «настоящая ли модель». «Подмена» только при сильной улике (critical)
|
|
4
|
+
или двух независимых провалах; одиночный слабый сигнал даёт «подозрительно».
|
|
5
|
+
Ось SERVICE — «честно ли обслуживают»: скрытый промпт, игнор параметров, урезания.
|
|
6
|
+
Недоступный сервис (502, таймаут) — «нет данных», а не обвинение.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import time
|
|
11
|
+
|
|
12
|
+
from ..makers import maker_of
|
|
13
|
+
from . import rand
|
|
14
|
+
from .check import REGISTRY
|
|
15
|
+
from .client import make_client
|
|
16
|
+
from .types import Axis, Finding, Report, ServiceVerdict, Status, Target, Verdict
|
|
17
|
+
|
|
18
|
+
WEIGHT = {Status.PASS: 1.0, Status.WARN: 0.4, Status.FAIL: 0.0}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def detect_protocol(model: str, base_url: str) -> str:
|
|
22
|
+
return "anthropic" if "anthropic.com" in base_url else "openai"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def decide(findings: list[Finding]) -> tuple[Verdict, int]:
|
|
26
|
+
if any(f.evidence.get("unavailable") for f in findings):
|
|
27
|
+
return Verdict.UNKNOWN, 0
|
|
28
|
+
scored = [f for f in findings if f.status in WEIGHT and f.axis is Axis.MODEL]
|
|
29
|
+
if not scored:
|
|
30
|
+
return Verdict.UNKNOWN, 0
|
|
31
|
+
score = round(100 * sum(WEIGHT[f.status] for f in scored) / len(scored))
|
|
32
|
+
fails = [f for f in scored if f.status is Status.FAIL]
|
|
33
|
+
if any(f.critical for f in fails) or len({f.check for f in fails}) >= 2:
|
|
34
|
+
return Verdict.FAKE, min(score, 30)
|
|
35
|
+
if fails or sum(f.status is Status.WARN for f in scored) >= 3:
|
|
36
|
+
return Verdict.SUSPICIOUS, min(score, 70)
|
|
37
|
+
return Verdict.GENUINE, score
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def decide_service(findings: list[Finding]) -> ServiceVerdict:
|
|
41
|
+
svc = [f for f in findings if f.axis is Axis.SERVICE and f.status in WEIGHT]
|
|
42
|
+
if not svc:
|
|
43
|
+
return ServiceVerdict.UNKNOWN
|
|
44
|
+
return ServiceVerdict.INTERFERES if any(f.status in (Status.FAIL, Status.WARN) for f in svc) else ServiceVerdict.CLEAN
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def run(target: Target, level: int = 2, only: list[str] | None = None, on_step=None,
|
|
48
|
+
seed: int | None = None, timeout: float = 45) -> Report:
|
|
49
|
+
used_seed = rand.seed(seed)
|
|
50
|
+
maker = maker_of(target.model)
|
|
51
|
+
client = make_client(target.protocol, target.base_url, target.api_key, timeout=timeout)
|
|
52
|
+
findings: list[Finding] = []
|
|
53
|
+
t = time.perf_counter()
|
|
54
|
+
for cls in REGISTRY:
|
|
55
|
+
chk = cls()
|
|
56
|
+
if only and chk.name not in only:
|
|
57
|
+
continue
|
|
58
|
+
if not only and chk.level > level:
|
|
59
|
+
continue
|
|
60
|
+
if not chk.applies(target, maker):
|
|
61
|
+
continue
|
|
62
|
+
if on_step:
|
|
63
|
+
on_step(chk.title)
|
|
64
|
+
try:
|
|
65
|
+
findings += chk.run(client, target, maker)
|
|
66
|
+
except Exception as e: # одна сломанная проверка не валит весь отчёт
|
|
67
|
+
findings.append(Finding(chk.name, Status.SKIP, chk.title, f"Ошибка проверки: {type(e).__name__}: {e}"))
|
|
68
|
+
if any(f.evidence.get("unavailable") for f in findings):
|
|
69
|
+
break # сервис лежит — дальше только тратить запросы
|
|
70
|
+
verdict, score = decide(findings)
|
|
71
|
+
return Report(target, maker, findings, verdict, score, time.perf_counter() - t, client.count,
|
|
72
|
+
service=decide_service(findings), seed=used_seed)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def run_many(target: Target, times: int, **kw) -> Report:
|
|
76
|
+
"""Несколько прогонов подряд с разными сидами (у пула аккаунтов запросы уходят на разные бэкенды).
|
|
77
|
+
|
|
78
|
+
Итог — худший вердикт по модели; у каждого провала видно, в скольких прогонах он повторился.
|
|
79
|
+
"""
|
|
80
|
+
kw.pop("seed", None)
|
|
81
|
+
reports = [run(target, **kw) for _ in range(times)]
|
|
82
|
+
order = [Verdict.FAKE, Verdict.SUSPICIOUS, Verdict.GENUINE, Verdict.UNKNOWN]
|
|
83
|
+
worst = min(reports, key=lambda r: order.index(r.verdict))
|
|
84
|
+
seen: dict[tuple, int] = {}
|
|
85
|
+
for r in reports:
|
|
86
|
+
for key in {(f.check, f.title, f.status) for f in r.findings}:
|
|
87
|
+
seen[key] = seen.get(key, 0) + 1
|
|
88
|
+
if times > 1:
|
|
89
|
+
for f in worst.findings:
|
|
90
|
+
if f.status in (Status.FAIL, Status.WARN):
|
|
91
|
+
f.detail += f" [в {seen.get((f.check, f.title, f.status), 1)} из {times} прогонов]"
|
|
92
|
+
worst.runs = times
|
|
93
|
+
worst.requests = sum(r.requests for r in reports)
|
|
94
|
+
worst.elapsed = sum(r.elapsed for r in reports)
|
|
95
|
+
if any(r.service is ServiceVerdict.INTERFERES for r in reports):
|
|
96
|
+
worst.service = ServiceVerdict.INTERFERES
|
|
97
|
+
return worst
|
modelcheck/core/rand.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Рандомизация проверочных запросов.
|
|
2
|
+
|
|
3
|
+
Если посредник выучит наши тестовые фразы, он сможет отправлять именно их на настоящую
|
|
4
|
+
модель, а остальной трафик — на дешёвую. Поэтому каждый прогон собирает фразы заново:
|
|
5
|
+
случайная формулировка + случайная «обычная» обёртка, так что запрос похож на рабочий.
|
|
6
|
+
Сид печатается в отчёте — прогон можно повторить один в один (--seed).
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import random
|
|
11
|
+
|
|
12
|
+
_rng = random.Random()
|
|
13
|
+
|
|
14
|
+
# обычные рабочие фразы, в которые прячем проверку
|
|
15
|
+
COVERS = [
|
|
16
|
+
"", "", "", # иногда без обёртки
|
|
17
|
+
"Работаю над ботом, вопрос по ходу. ",
|
|
18
|
+
"Небольшой вопрос между делом: ",
|
|
19
|
+
"Пишу заметку, уточни, пожалуйста. ",
|
|
20
|
+
"Quick one while I'm debugging: ",
|
|
21
|
+
"Сейчас разбираюсь с API. ",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def seed(value: int | None) -> int:
|
|
26
|
+
s = value if value is not None else random.randrange(1, 10**9)
|
|
27
|
+
_rng.seed(s)
|
|
28
|
+
return s
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def pick(options: list[str]) -> str:
|
|
32
|
+
return _rng.choice(options)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def cover(text: str) -> str:
|
|
36
|
+
"""Обернуть проверочный вопрос в «обычный» контекст."""
|
|
37
|
+
return _rng.choice(COVERS) + text
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def word(n: int = 5) -> str:
|
|
41
|
+
"""Случайное кодовое слово (для канареек)."""
|
|
42
|
+
return "".join(_rng.choice("АБВГДЕЖЗИКЛМНОПРСТУФХ") for _ in range(n)) + "-" + str(_rng.randint(1000, 9999))
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def number(a: int, b: int) -> int:
|
|
46
|
+
return _rng.randint(a, b)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def shuffle(seq: list) -> list:
|
|
50
|
+
out = list(seq)
|
|
51
|
+
_rng.shuffle(out)
|
|
52
|
+
return out
|
modelcheck/core/types.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Общие типы: цель проверки, результат одной проверки, итоговый отчёт.
|
|
2
|
+
|
|
3
|
+
Вердикт двухосевой (как в api-scam-hunter): «настоящая ли модель» и «честно ли
|
|
4
|
+
обслуживают». Скрытый промпт на 4000 токенов — это переплата и вмешательство,
|
|
5
|
+
но модель внутри при этом может быть настоящей. Это разные вопросы.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from enum import Enum
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Status(str, Enum):
|
|
14
|
+
PASS = "pass" # признак настоящей модели / честного сервиса
|
|
15
|
+
WARN = "warn" # подозрительно, но не доказательство
|
|
16
|
+
FAIL = "fail" # признак подмены / нарушения
|
|
17
|
+
INFO = "info" # просто факт (посредник, скорость)
|
|
18
|
+
SKIP = "skip" # проверка неприменима или не удалась
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Axis(str, Enum):
|
|
22
|
+
MODEL = "model" # что за модель отвечает
|
|
23
|
+
SERVICE = "service" # честно ли обслуживает посредник: скрытый промпт, параметры, урезания
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class Verdict(str, Enum):
|
|
27
|
+
GENUINE = "genuine"
|
|
28
|
+
SUSPICIOUS = "suspicious"
|
|
29
|
+
FAKE = "fake"
|
|
30
|
+
UNKNOWN = "unknown"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class ServiceVerdict(str, Enum):
|
|
34
|
+
CLEAN = "clean"
|
|
35
|
+
INTERFERES = "interferes" # вмешивается: скрытый промпт, игнор параметров, урезанные возможности
|
|
36
|
+
UNKNOWN = "unknown"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass
|
|
40
|
+
class Finding:
|
|
41
|
+
check: str
|
|
42
|
+
status: Status
|
|
43
|
+
title: str
|
|
44
|
+
detail: str = ""
|
|
45
|
+
critical: bool = False # одного такого FAIL достаточно для «подмены»
|
|
46
|
+
evidence: dict = field(default_factory=dict)
|
|
47
|
+
axis: Axis = Axis.MODEL
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass
|
|
51
|
+
class Target:
|
|
52
|
+
base_url: str
|
|
53
|
+
api_key: str
|
|
54
|
+
model: str
|
|
55
|
+
protocol: str # "anthropic" | "openai"
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def masked_key(self) -> str:
|
|
59
|
+
k = self.api_key
|
|
60
|
+
return k[:4] + "…" + k[-4:] if len(k) > 12 else "***"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass
|
|
64
|
+
class Report:
|
|
65
|
+
target: Target
|
|
66
|
+
maker: str
|
|
67
|
+
findings: list[Finding]
|
|
68
|
+
verdict: Verdict
|
|
69
|
+
score: int
|
|
70
|
+
elapsed: float
|
|
71
|
+
requests: int
|
|
72
|
+
service: ServiceVerdict = ServiceVerdict.UNKNOWN
|
|
73
|
+
seed: int = 0
|
|
74
|
+
runs: int = 1
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def svc(check: str, status: Status, title: str, detail: str = "", **kw) -> Finding:
|
|
78
|
+
"""Находка про честность обслуживания (ось SERVICE)."""
|
|
79
|
+
return Finding(check, status, title, detail, axis=Axis.SERVICE, **kw)
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Профили производителей. Каждый — в своей папке: __init__.py (PROFILE) + RESEARCH.md (как проверять, источники).
|
|
2
|
+
|
|
3
|
+
Профиль описывает «почерк» производителя: как модель себя называет, какие служебные
|
|
4
|
+
токены у её токенизатора, какие поля она отдаёт, есть ли криптоподпись размышлений,
|
|
5
|
+
сколько скрытых токенов добавляет сам официальный API.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import importlib
|
|
10
|
+
import pkgutil
|
|
11
|
+
import re
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class Profile:
|
|
17
|
+
name: str # id: anthropic, openai, …
|
|
18
|
+
title: str # для отчёта
|
|
19
|
+
model_re: str # как узнать по имени модели
|
|
20
|
+
self_words: tuple[str, ...] # как модель/компания называется в ответах
|
|
21
|
+
native_protocol: str = "openai" # anthropic | openai | gemini
|
|
22
|
+
tiers: tuple[str, ...] = () # дорогой -> дешёвый
|
|
23
|
+
signature: str | None = None # anthropic_thinking | openai_encrypted | gemini_thought | xai_encrypted
|
|
24
|
+
reasoning_field: str | None = None # поле с размышлениями в chat completions (reasoning_content)
|
|
25
|
+
special_tokens: tuple[str, ...] = () # служебные токены токенизатора
|
|
26
|
+
id_prefixes: tuple[str, ...] = () # префиксы id на родном протоколе
|
|
27
|
+
input_overhead: int = 40 # сколько входных токенов на «x» даёт ОФИЦИАЛЬНЫЙ API
|
|
28
|
+
cjk_native: bool = False # учится на китайском: иероглифы не улика
|
|
29
|
+
open_weights: bool = False # открытые веса (можно поднять у себя → частая подмена)
|
|
30
|
+
notes: tuple[str, ...] = field(default_factory=tuple)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
PROFILES: dict[str, Profile] = {}
|
|
34
|
+
|
|
35
|
+
for _m in pkgutil.iter_modules(__path__):
|
|
36
|
+
if _m.ispkg:
|
|
37
|
+
_p = importlib.import_module(f"{__name__}.{_m.name}").PROFILE
|
|
38
|
+
PROFILES[_p.name] = _p
|
|
39
|
+
|
|
40
|
+
# порядок важен: более специфичные шаблоны раньше (gemma — google, а не meta)
|
|
41
|
+
_ORDER = sorted(PROFILES.values(), key=lambda p: -len(p.model_re))
|
|
42
|
+
|
|
43
|
+
UNKNOWN = Profile("unknown", "Неизвестный производитель", r"$^", ())
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def maker_of(model: str) -> str:
|
|
47
|
+
m = model.lower()
|
|
48
|
+
for p in _ORDER:
|
|
49
|
+
if re.search(p.model_re, m):
|
|
50
|
+
return p.name
|
|
51
|
+
return "unknown"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def profile(name: str) -> Profile:
|
|
55
|
+
return PROFILES.get(name, UNKNOWN)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def named_makers(text: str) -> set[str]:
|
|
59
|
+
t = text.lower()
|
|
60
|
+
return {p.name for p in PROFILES.values()
|
|
61
|
+
if any(re.search(rf"(?<!\w){re.escape(w)}(?!\w)", t) for w in p.self_words)}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def tier_of(model: str, maker: str) -> str | None:
|
|
65
|
+
m = model.lower()
|
|
66
|
+
for t in profile(maker).tiers:
|
|
67
|
+
if re.search(rf"(?<![a-z]){t}(?![a-z])", m):
|
|
68
|
+
return t
|
|
69
|
+
return None
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Профиль: Alibaba (Qwen). Исследование и источники — RESEARCH.md рядом."""
|
|
2
|
+
from .. import Profile
|
|
3
|
+
|
|
4
|
+
PROFILE = Profile(
|
|
5
|
+
name="alibaba",
|
|
6
|
+
title="Alibaba (Qwen)",
|
|
7
|
+
model_re=r"qwen|qwq|alibaba|tongyi|dashscope",
|
|
8
|
+
self_words=('alibaba', 'qwen', 'tongyi', '通义'),
|
|
9
|
+
tiers=('max', 'plus', 'turbo', 'flash'),
|
|
10
|
+
reasoning_field="reasoning_content",
|
|
11
|
+
special_tokens=('<|im_end|>', '<|endoftext|>'),
|
|
12
|
+
cjk_native=True,
|
|
13
|
+
open_weights=True,
|
|
14
|
+
)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Профиль: Anthropic (Claude). Исследование и источники — RESEARCH.md рядом."""
|
|
2
|
+
from .. import Profile
|
|
3
|
+
|
|
4
|
+
PROFILE = Profile(
|
|
5
|
+
name="anthropic",
|
|
6
|
+
title="Anthropic (Claude)",
|
|
7
|
+
model_re=r"claude|anthropic",
|
|
8
|
+
self_words=('anthropic', 'claude'),
|
|
9
|
+
native_protocol="anthropic",
|
|
10
|
+
tiers=('opus', 'sonnet', 'haiku'),
|
|
11
|
+
signature="anthropic_thinking",
|
|
12
|
+
id_prefixes=('msg_',),
|
|
13
|
+
input_overhead=40,
|
|
14
|
+
notes=('Самая сильная проверка в мире: подпись блока thinking, подделать нельзя.',),
|
|
15
|
+
)
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Профиль: ByteDance (Doubao, Seed). Исследование и источники — RESEARCH.md рядом."""
|
|
2
|
+
from .. import Profile
|
|
3
|
+
|
|
4
|
+
PROFILE = Profile(
|
|
5
|
+
name="bytedance",
|
|
6
|
+
title="ByteDance (Doubao, Seed)",
|
|
7
|
+
model_re=r"doubao|seed-|bytedance|volc",
|
|
8
|
+
self_words=('bytedance', 'doubao', '豆包', '字节'),
|
|
9
|
+
tiers=('pro', 'turbo', 'lite'),
|
|
10
|
+
reasoning_field="reasoning_content",
|
|
11
|
+
cjk_native=True,
|
|
12
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Профиль: DeepSeek. Исследование и источники — RESEARCH.md рядом."""
|
|
2
|
+
from .. import Profile
|
|
3
|
+
|
|
4
|
+
PROFILE = Profile(
|
|
5
|
+
name="deepseek",
|
|
6
|
+
title="DeepSeek",
|
|
7
|
+
model_re=r"deepseek",
|
|
8
|
+
self_words=('deepseek',),
|
|
9
|
+
tiers=('reasoner', 'pro', 'chat', 'flash'),
|
|
10
|
+
reasoning_field="reasoning_content",
|
|
11
|
+
special_tokens=('<|end▁of▁sentence|>', '<|begin▁of▁sentence|>'),
|
|
12
|
+
cjk_native=True,
|
|
13
|
+
open_weights=True,
|
|
14
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Профиль: Google (Gemini, Gemma). Исследование и источники — RESEARCH.md рядом."""
|
|
2
|
+
from .. import Profile
|
|
3
|
+
|
|
4
|
+
PROFILE = Profile(
|
|
5
|
+
name="google",
|
|
6
|
+
title="Google (Gemini, Gemma)",
|
|
7
|
+
model_re=r"gemini|gemma|google|palm",
|
|
8
|
+
self_words=('google', 'gemini', 'deepmind', 'gemma'),
|
|
9
|
+
native_protocol="gemini",
|
|
10
|
+
tiers=('ultra', 'pro', 'argon', 'carbon', 'flash', 'lite'),
|
|
11
|
+
signature="gemini_thought",
|
|
12
|
+
input_overhead=40,
|
|
13
|
+
notes=('Gemini 3+ думает по умолчанию: отсутствие следов thinking — признак переходника.',),
|
|
14
|
+
)
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Профиль: Meta (Llama). Исследование и источники — RESEARCH.md рядом."""
|
|
2
|
+
from .. import Profile
|
|
3
|
+
|
|
4
|
+
PROFILE = Profile(
|
|
5
|
+
name="meta",
|
|
6
|
+
title="Meta (Llama)",
|
|
7
|
+
model_re=r"llama|meta-llama|(?<![a-z])meta(?![a-z])",
|
|
8
|
+
self_words=('meta', 'llama'),
|
|
9
|
+
tiers=('maverick', 'scout', '70b', '8b'),
|
|
10
|
+
special_tokens=('<|eot_id|>', '<|end_of_text|>', '<|start_header_id|>'),
|
|
11
|
+
open_weights=True,
|
|
12
|
+
)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Профиль: Microsoft (Phi, MAI). Исследование и источники — RESEARCH.md рядом."""
|
|
2
|
+
from .. import Profile
|
|
3
|
+
|
|
4
|
+
PROFILE = Profile(
|
|
5
|
+
name="microsoft",
|
|
6
|
+
title="Microsoft (Phi, MAI)",
|
|
7
|
+
model_re=r"(?<![a-z])phi-|microsoft|mai-",
|
|
8
|
+
self_words=('microsoft', 'phi'),
|
|
9
|
+
special_tokens=('<|end|>', '<|endoftext|>'),
|
|
10
|
+
open_weights=True,
|
|
11
|
+
)
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Профиль: MiniMax. Исследование и источники — RESEARCH.md рядом."""
|
|
2
|
+
from .. import Profile
|
|
3
|
+
|
|
4
|
+
PROFILE = Profile(
|
|
5
|
+
name="minimax",
|
|
6
|
+
title="MiniMax",
|
|
7
|
+
model_re=r"minimax|abab",
|
|
8
|
+
self_words=('minimax',),
|
|
9
|
+
reasoning_field="reasoning_content",
|
|
10
|
+
cjk_native=True,
|
|
11
|
+
open_weights=True,
|
|
12
|
+
)
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Профиль: Mistral. Исследование и источники — RESEARCH.md рядом."""
|
|
2
|
+
from .. import Profile
|
|
3
|
+
|
|
4
|
+
PROFILE = Profile(
|
|
5
|
+
name="mistral",
|
|
6
|
+
title="Mistral",
|
|
7
|
+
model_re=r"mistral|mixtral|codestral|magistral|ministral|pixtral|devstral",
|
|
8
|
+
self_words=('mistral',),
|
|
9
|
+
tiers=('large', 'medium', 'small', 'ministral'),
|
|
10
|
+
special_tokens=('</s>', '[/INST]'),
|
|
11
|
+
open_weights=True,
|
|
12
|
+
)
|