quantdiff 0.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- quantdiff/__init__.py +53 -0
- quantdiff/__main__.py +5 -0
- quantdiff/_http.py +151 -0
- quantdiff/_text.py +13 -0
- quantdiff/_version.py +1 -0
- quantdiff/api.py +340 -0
- quantdiff/backends/__init__.py +28 -0
- quantdiff/backends/_common.py +342 -0
- quantdiff/backends/base.py +91 -0
- quantdiff/backends/llamacpp.py +428 -0
- quantdiff/backends/ollama.py +359 -0
- quantdiff/backends/openai_compat.py +338 -0
- quantdiff/cache.py +240 -0
- quantdiff/card.py +1664 -0
- quantdiff/cli.py +377 -0
- quantdiff/discover.py +488 -0
- quantdiff/errors.py +45 -0
- quantdiff/metrics/__init__.py +36 -0
- quantdiff/metrics/codeexec.py +428 -0
- quantdiff/metrics/jsonschema.py +610 -0
- quantdiff/metrics/logit.py +214 -0
- quantdiff/metrics/tasks.py +114 -0
- quantdiff/metrics/textsim.py +66 -0
- quantdiff/metrics/toolcheck.py +99 -0
- quantdiff/png.py +360 -0
- quantdiff/preflight.py +365 -0
- quantdiff/progress.py +283 -0
- quantdiff/py.typed +0 -0
- quantdiff/report.py +780 -0
- quantdiff/runner.py +492 -0
- quantdiff/spec.py +154 -0
- quantdiff/stats.py +226 -0
- quantdiff/suites/__init__.py +462 -0
- quantdiff/suites/data/chat.jsonl +22 -0
- quantdiff/suites/data/code.jsonl +32 -0
- quantdiff/suites/data/json.jsonl +34 -0
- quantdiff/suites/data/scoring.jsonl +41 -0
- quantdiff/suites/data/tools.jsonl +32 -0
- quantdiff/types.py +322 -0
- quantdiff/verdict.py +1513 -0
- quantdiff-0.1.0rc1.dist-info/METADATA +514 -0
- quantdiff-0.1.0rc1.dist-info/RECORD +45 -0
- quantdiff-0.1.0rc1.dist-info/WHEEL +4 -0
- quantdiff-0.1.0rc1.dist-info/entry_points.txt +2 -0
- quantdiff-0.1.0rc1.dist-info/licenses/LICENSE +202 -0
quantdiff/png.py
ADDED
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
"""PNG export of the HTML scorecard through an installed Chromium-family browser.
|
|
2
|
+
|
|
3
|
+
A headless browser renders the exact HTML card, so the PNG is pixel-identical to what the
|
|
4
|
+
card looks like when opened, and quantdiff keeps zero Python dependencies.
|
|
5
|
+
|
|
6
|
+
The browser cannot size a screenshot to its content, so rendering takes two launches:
|
|
7
|
+
|
|
8
|
+
1. Measure: a wrapper page loads the card into an iframe of the target width and, once it
|
|
9
|
+
has loaded, writes the card's full height into a meta tag. ``--dump-dom`` prints the
|
|
10
|
+
resulting DOM and the height is parsed from it. The iframe pins the layout width
|
|
11
|
+
exactly, which a top-level page cannot do because ``--dump-dom`` subtracts window
|
|
12
|
+
decorations from ``--window-size``. The card itself stays free of scripts.
|
|
13
|
+
2. Screenshot: the card is captured with a window of exactly that width and height.
|
|
14
|
+
|
|
15
|
+
Every launch uses a throwaway profile in a private temporary directory and forces the
|
|
16
|
+
light color scheme so the output does not depend on the user's settings.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import logging
|
|
22
|
+
import os
|
|
23
|
+
import re
|
|
24
|
+
import shutil
|
|
25
|
+
import subprocess
|
|
26
|
+
import sys
|
|
27
|
+
import tempfile
|
|
28
|
+
import time
|
|
29
|
+
from collections.abc import Mapping, Sequence
|
|
30
|
+
from html import escape
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
from typing import Final
|
|
33
|
+
|
|
34
|
+
from quantdiff.errors import RenderError
|
|
35
|
+
|
|
36
|
+
__all__ = [
|
|
37
|
+
"BROWSER_ENV",
|
|
38
|
+
"find_browser",
|
|
39
|
+
"max_height",
|
|
40
|
+
"measure_command",
|
|
41
|
+
"measure_page",
|
|
42
|
+
"parse_height",
|
|
43
|
+
"platform_candidates",
|
|
44
|
+
"render_png",
|
|
45
|
+
"screenshot_command",
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
logger = logging.getLogger(__name__)
|
|
49
|
+
|
|
50
|
+
BROWSER_ENV: Final = "QUANTDIFF_BROWSER"
|
|
51
|
+
MIN_WIDTH: Final = 320
|
|
52
|
+
MAX_WIDTH: Final = 4096
|
|
53
|
+
MIN_SCALE: Final = 0.5
|
|
54
|
+
MAX_SCALE: Final = 4.0
|
|
55
|
+
MAX_HEIGHT: Final = 20_000
|
|
56
|
+
"""Tallest card, in CSS pixels, that is screenshotted; anything taller is clipped."""
|
|
57
|
+
MAX_PNG_PIXELS: Final = 100_000_000
|
|
58
|
+
"""Largest image area, in device pixels, so a huge width and scale cannot exhaust memory."""
|
|
59
|
+
_STDERR_TAIL_CHARS: Final = 600
|
|
60
|
+
# --dump-dom shrinks the viewport by the window frame; the slack keeps the iframe unclipped.
|
|
61
|
+
_MEASURE_WINDOW_SLACK: Final = 120
|
|
62
|
+
_MEASURE_WINDOW_HEIGHT: Final = 200
|
|
63
|
+
_PNG_SIGNATURE: Final = b"\x89PNG\r\n\x1a\n"
|
|
64
|
+
_HEIGHT_META: Final = re.compile(r'<meta name="quantdiff-height" content="(\d+)">')
|
|
65
|
+
_NO_BROWSER: Final = (
|
|
66
|
+
"no Chromium-based browser found (Chrome, Edge, Brave or Chromium); "
|
|
67
|
+
f"install one or set {BROWSER_ENV} to its path"
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
_WINDOWS_BROWSERS: Final = (
|
|
71
|
+
("Microsoft", "Edge", "Application", "msedge.exe"),
|
|
72
|
+
("Google", "Chrome", "Application", "chrome.exe"),
|
|
73
|
+
("BraveSoftware", "Brave-Browser", "Application", "brave.exe"),
|
|
74
|
+
("Chromium", "Application", "chrome.exe"),
|
|
75
|
+
)
|
|
76
|
+
_WINDOWS_ROOTS: Final = ("ProgramFiles", "ProgramFiles(x86)", "LocalAppData")
|
|
77
|
+
_MACOS_BROWSERS: Final = (
|
|
78
|
+
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
|
79
|
+
"/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
|
|
80
|
+
"/Applications/Brave Browser.app/Contents/MacOS/Brave Browser",
|
|
81
|
+
"/Applications/Chromium.app/Contents/MacOS/Chromium",
|
|
82
|
+
)
|
|
83
|
+
_LINUX_BROWSERS: Final = (
|
|
84
|
+
"google-chrome",
|
|
85
|
+
"google-chrome-stable",
|
|
86
|
+
"chromium",
|
|
87
|
+
"chromium-browser",
|
|
88
|
+
"microsoft-edge",
|
|
89
|
+
"brave-browser",
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
_MEASURE_SCRIPT: Final = """
|
|
93
|
+
window.addEventListener("load", function () {
|
|
94
|
+
var card = document.getElementById("card").contentDocument.documentElement;
|
|
95
|
+
var meta = document.createElement("meta");
|
|
96
|
+
meta.name = "quantdiff-height";
|
|
97
|
+
meta.content = String(Math.ceil(card.scrollHeight));
|
|
98
|
+
document.head.appendChild(meta);
|
|
99
|
+
});
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
# Browser discovery ------------------------------------------------------------------------
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def find_browser() -> Path | None:
|
|
107
|
+
"""Locate a Chromium-family browser, or return None when none is installed.
|
|
108
|
+
|
|
109
|
+
The QUANTDIFF_BROWSER environment variable wins when set and must name an existing
|
|
110
|
+
file. Otherwise the standard install locations for the current platform are probed.
|
|
111
|
+
"""
|
|
112
|
+
explicit = os.environ.get(BROWSER_ENV)
|
|
113
|
+
if explicit:
|
|
114
|
+
path = Path(explicit).expanduser()
|
|
115
|
+
if not path.is_file():
|
|
116
|
+
raise RenderError(f"{BROWSER_ENV} is set to {explicit!r}, which is not a file")
|
|
117
|
+
return path
|
|
118
|
+
for candidate in platform_candidates(sys.platform, os.environ):
|
|
119
|
+
if candidate.is_file():
|
|
120
|
+
return candidate
|
|
121
|
+
return None
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def platform_candidates(platform: str, environ: Mapping[str, str]) -> list[Path]:
|
|
125
|
+
"""List the usual browser install paths for `platform`, most preferred first."""
|
|
126
|
+
if platform == "win32":
|
|
127
|
+
return _windows_candidates(environ)
|
|
128
|
+
if platform == "darwin":
|
|
129
|
+
return [Path(path) for path in _MACOS_BROWSERS]
|
|
130
|
+
found = (shutil.which(name) for name in _LINUX_BROWSERS)
|
|
131
|
+
return [Path(path) for path in found if path]
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _windows_candidates(environ: Mapping[str, str]) -> list[Path]:
|
|
135
|
+
roots = [Path(environ[name]) for name in _WINDOWS_ROOTS if environ.get(name)]
|
|
136
|
+
return [root.joinpath(*parts) for parts in _WINDOWS_BROWSERS for root in roots]
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
# Commands and pages -----------------------------------------------------------------------
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _common_flags(profile_dir: Path, scale: float) -> list[str]:
|
|
143
|
+
return [
|
|
144
|
+
"--headless=new",
|
|
145
|
+
# A private profile keeps the user's browser profile, cookies and extensions out.
|
|
146
|
+
f"--user-data-dir={profile_dir}",
|
|
147
|
+
"--no-first-run",
|
|
148
|
+
"--no-default-browser-check",
|
|
149
|
+
"--disable-extensions",
|
|
150
|
+
"--disable-default-apps",
|
|
151
|
+
"--disable-gpu",
|
|
152
|
+
"--disable-background-networking",
|
|
153
|
+
"--disable-sync",
|
|
154
|
+
"--disable-component-update",
|
|
155
|
+
"--mute-audio",
|
|
156
|
+
"--hide-scrollbars",
|
|
157
|
+
# 1 is Blink's light scheme; the card follows prefers-color-scheme otherwise.
|
|
158
|
+
"--blink-settings=preferredColorScheme=1",
|
|
159
|
+
f"--force-device-scale-factor={scale:g}",
|
|
160
|
+
]
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def measure_command(
|
|
164
|
+
browser: Path, page: Path, profile_dir: Path, *, width: int, scale: float
|
|
165
|
+
) -> list[str]:
|
|
166
|
+
"""Build the argv that prints the DOM of the measure page after it has loaded."""
|
|
167
|
+
return [
|
|
168
|
+
str(browser),
|
|
169
|
+
*_common_flags(profile_dir, scale),
|
|
170
|
+
f"--window-size={width + _MEASURE_WINDOW_SLACK},{_MEASURE_WINDOW_HEIGHT}",
|
|
171
|
+
"--dump-dom",
|
|
172
|
+
page.as_uri(),
|
|
173
|
+
]
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def screenshot_command(
|
|
177
|
+
browser: Path,
|
|
178
|
+
page: Path,
|
|
179
|
+
output: Path,
|
|
180
|
+
profile_dir: Path,
|
|
181
|
+
*,
|
|
182
|
+
width: int,
|
|
183
|
+
height: int,
|
|
184
|
+
scale: float,
|
|
185
|
+
) -> list[str]:
|
|
186
|
+
"""Build the argv that screenshots `page` into `output` at exactly width x height."""
|
|
187
|
+
return [
|
|
188
|
+
str(browser),
|
|
189
|
+
*_common_flags(profile_dir, scale),
|
|
190
|
+
f"--window-size={width},{height}",
|
|
191
|
+
f"--screenshot={output}",
|
|
192
|
+
page.as_uri(),
|
|
193
|
+
]
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def measure_page(card_html: str, width: int) -> str:
|
|
197
|
+
"""Wrap the card in an iframe `width` CSS pixels wide plus a script that records its height.
|
|
198
|
+
|
|
199
|
+
The iframe is 1px tall so the card's scrollHeight is its content height, not the
|
|
200
|
+
viewport's.
|
|
201
|
+
"""
|
|
202
|
+
return "\n".join(
|
|
203
|
+
[
|
|
204
|
+
"<!doctype html>",
|
|
205
|
+
'<html lang="en">',
|
|
206
|
+
"<head>",
|
|
207
|
+
'<meta charset="utf-8">',
|
|
208
|
+
"<style>html, body { margin: 0; } "
|
|
209
|
+
f"iframe {{ display: block; border: 0; width: {width}px; height: 1px; }}</style>",
|
|
210
|
+
"</head>",
|
|
211
|
+
"<body>",
|
|
212
|
+
f'<iframe id="card" srcdoc="{escape(card_html, quote=True)}"></iframe>',
|
|
213
|
+
f"<script>{_MEASURE_SCRIPT}</script>",
|
|
214
|
+
"</body>",
|
|
215
|
+
"</html>",
|
|
216
|
+
"",
|
|
217
|
+
]
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def parse_height(dom: str) -> int:
|
|
222
|
+
"""Read the card height that the measure page wrote into its DOM."""
|
|
223
|
+
match = _HEIGHT_META.search(dom)
|
|
224
|
+
if match is None:
|
|
225
|
+
raise RenderError("the browser did not report the card height; is it Chromium-based?")
|
|
226
|
+
height = int(match.group(1))
|
|
227
|
+
if height <= 0:
|
|
228
|
+
raise RenderError(f"the browser reported a card height of {height}px")
|
|
229
|
+
return height
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
# Rendering --------------------------------------------------------------------------------
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def render_png(
|
|
236
|
+
html: str,
|
|
237
|
+
out_path: Path,
|
|
238
|
+
*,
|
|
239
|
+
width: int = 1040,
|
|
240
|
+
scale: float = 2.0,
|
|
241
|
+
browser: Path | None = None,
|
|
242
|
+
timeout_seconds: float = 60.0,
|
|
243
|
+
) -> Path:
|
|
244
|
+
"""Render `html` to a PNG at `out_path` and return that path.
|
|
245
|
+
|
|
246
|
+
The image is `width * scale` pixels wide and exactly as tall as the page content
|
|
247
|
+
(capped by max_height()). `browser` defaults to find_browser(). Raises
|
|
248
|
+
RenderError when no browser is available, the browser fails or times out, or the
|
|
249
|
+
output is not a PNG.
|
|
250
|
+
"""
|
|
251
|
+
_check_size(width, scale)
|
|
252
|
+
executable = browser if browser is not None else find_browser()
|
|
253
|
+
if executable is None:
|
|
254
|
+
raise RenderError(_NO_BROWSER)
|
|
255
|
+
started = time.monotonic()
|
|
256
|
+
with tempfile.TemporaryDirectory(prefix="quantdiff-png-", ignore_cleanup_errors=True) as tmp:
|
|
257
|
+
workdir = Path(tmp)
|
|
258
|
+
profile = workdir / "profile"
|
|
259
|
+
measure = workdir / "measure.html"
|
|
260
|
+
card = workdir / "card.html"
|
|
261
|
+
shot = workdir / "card.png"
|
|
262
|
+
measure.write_text(measure_page(html, width), encoding="utf-8")
|
|
263
|
+
card.write_text(html, encoding="utf-8")
|
|
264
|
+
|
|
265
|
+
command = measure_command(executable, measure, profile, width=width, scale=scale)
|
|
266
|
+
height = parse_height(_run_browser(command, workdir, timeout_seconds))
|
|
267
|
+
limit = max_height(width, scale)
|
|
268
|
+
if height > limit:
|
|
269
|
+
logger.warning("card is %dpx tall; clipping the PNG to %dpx", height, limit)
|
|
270
|
+
height = limit
|
|
271
|
+
command = screenshot_command(
|
|
272
|
+
executable, card, shot, profile, width=width, height=height, scale=scale
|
|
273
|
+
)
|
|
274
|
+
_run_browser(command, workdir, timeout_seconds)
|
|
275
|
+
_check_png(shot)
|
|
276
|
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
277
|
+
shutil.copyfile(shot, out_path)
|
|
278
|
+
logger.debug("rendered %s in %.1fs", out_path, time.monotonic() - started)
|
|
279
|
+
return out_path
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _check_size(width: int, scale: float) -> None:
|
|
283
|
+
if not MIN_WIDTH <= width <= MAX_WIDTH:
|
|
284
|
+
raise RenderError(f"PNG width must be {MIN_WIDTH} to {MAX_WIDTH} CSS pixels, got {width}")
|
|
285
|
+
if not MIN_SCALE <= scale <= MAX_SCALE:
|
|
286
|
+
raise RenderError(f"PNG scale must be {MIN_SCALE:g} to {MAX_SCALE:g}, got {scale:g}")
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def max_height(width: int, scale: float) -> int:
|
|
290
|
+
"""Tallest screenshot, in CSS pixels, that stays within MAX_HEIGHT and MAX_PNG_PIXELS."""
|
|
291
|
+
return min(MAX_HEIGHT, int(MAX_PNG_PIXELS / (width * scale * scale)))
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _run_browser(command: Sequence[str], workdir: Path, timeout_seconds: float) -> str:
|
|
295
|
+
"""Run the browser and return its stdout. Output goes to files, not pipes, because
|
|
296
|
+
browser child processes can hold inherited pipes open after the parent is killed."""
|
|
297
|
+
stdout_path = workdir / "stdout.txt"
|
|
298
|
+
stderr_path = workdir / "stderr.txt"
|
|
299
|
+
with stdout_path.open("wb") as stdout, stderr_path.open("wb") as stderr:
|
|
300
|
+
try:
|
|
301
|
+
process = subprocess.Popen( # noqa: S603 - argv list, no shell; browser path comes from find_browser or the caller
|
|
302
|
+
command,
|
|
303
|
+
cwd=workdir,
|
|
304
|
+
stdin=subprocess.DEVNULL,
|
|
305
|
+
stdout=stdout,
|
|
306
|
+
stderr=stderr,
|
|
307
|
+
start_new_session=_NEW_SESSION,
|
|
308
|
+
)
|
|
309
|
+
except OSError as exc:
|
|
310
|
+
raise RenderError(f"could not start the browser {command[0]}: {exc}") from exc
|
|
311
|
+
try:
|
|
312
|
+
returncode = process.wait(timeout=timeout_seconds)
|
|
313
|
+
except subprocess.TimeoutExpired:
|
|
314
|
+
_kill(process)
|
|
315
|
+
process.wait()
|
|
316
|
+
raise RenderError(
|
|
317
|
+
f"the browser did not finish within {timeout_seconds:g}s{_stderr_tail(stderr_path)}"
|
|
318
|
+
) from None
|
|
319
|
+
if returncode != 0:
|
|
320
|
+
raise RenderError(f"the browser exited with code {returncode}{_stderr_tail(stderr_path)}")
|
|
321
|
+
return stdout_path.read_text(encoding="utf-8", errors="replace")
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _stderr_tail(path: Path) -> str:
|
|
325
|
+
try:
|
|
326
|
+
text = path.read_text(encoding="utf-8", errors="replace").strip()
|
|
327
|
+
except OSError:
|
|
328
|
+
return ""
|
|
329
|
+
if not text:
|
|
330
|
+
return ""
|
|
331
|
+
return f"; browser output ends with: {text[-_STDERR_TAIL_CHARS:]}"
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _check_png(path: Path) -> None:
|
|
335
|
+
try:
|
|
336
|
+
with path.open("rb") as handle:
|
|
337
|
+
header = handle.read(len(_PNG_SIGNATURE))
|
|
338
|
+
except OSError as exc:
|
|
339
|
+
raise RenderError(f"the browser did not write a screenshot: {exc}") from exc
|
|
340
|
+
if header != _PNG_SIGNATURE:
|
|
341
|
+
raise RenderError(f"the browser wrote {path.name}, but it is not a PNG image")
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
if sys.platform == "win32":
|
|
345
|
+
_NEW_SESSION: Final = False
|
|
346
|
+
|
|
347
|
+
def _kill(process: subprocess.Popen[bytes]) -> None:
|
|
348
|
+
# Chromium child processes exit on their own once the browser process is gone.
|
|
349
|
+
process.kill()
|
|
350
|
+
|
|
351
|
+
else:
|
|
352
|
+
import signal
|
|
353
|
+
|
|
354
|
+
_NEW_SESSION: Final = True
|
|
355
|
+
|
|
356
|
+
def _kill(process: subprocess.Popen[bytes]) -> None:
|
|
357
|
+
try:
|
|
358
|
+
os.killpg(process.pid, signal.SIGKILL)
|
|
359
|
+
except ProcessLookupError:
|
|
360
|
+
return
|