quantdiff 0.1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. quantdiff/__init__.py +53 -0
  2. quantdiff/__main__.py +5 -0
  3. quantdiff/_http.py +151 -0
  4. quantdiff/_text.py +13 -0
  5. quantdiff/_version.py +1 -0
  6. quantdiff/api.py +340 -0
  7. quantdiff/backends/__init__.py +28 -0
  8. quantdiff/backends/_common.py +342 -0
  9. quantdiff/backends/base.py +91 -0
  10. quantdiff/backends/llamacpp.py +428 -0
  11. quantdiff/backends/ollama.py +359 -0
  12. quantdiff/backends/openai_compat.py +338 -0
  13. quantdiff/cache.py +240 -0
  14. quantdiff/card.py +1664 -0
  15. quantdiff/cli.py +377 -0
  16. quantdiff/discover.py +488 -0
  17. quantdiff/errors.py +45 -0
  18. quantdiff/metrics/__init__.py +36 -0
  19. quantdiff/metrics/codeexec.py +428 -0
  20. quantdiff/metrics/jsonschema.py +610 -0
  21. quantdiff/metrics/logit.py +214 -0
  22. quantdiff/metrics/tasks.py +114 -0
  23. quantdiff/metrics/textsim.py +66 -0
  24. quantdiff/metrics/toolcheck.py +99 -0
  25. quantdiff/png.py +360 -0
  26. quantdiff/preflight.py +365 -0
  27. quantdiff/progress.py +283 -0
  28. quantdiff/py.typed +0 -0
  29. quantdiff/report.py +780 -0
  30. quantdiff/runner.py +492 -0
  31. quantdiff/spec.py +154 -0
  32. quantdiff/stats.py +226 -0
  33. quantdiff/suites/__init__.py +462 -0
  34. quantdiff/suites/data/chat.jsonl +22 -0
  35. quantdiff/suites/data/code.jsonl +32 -0
  36. quantdiff/suites/data/json.jsonl +34 -0
  37. quantdiff/suites/data/scoring.jsonl +41 -0
  38. quantdiff/suites/data/tools.jsonl +32 -0
  39. quantdiff/types.py +322 -0
  40. quantdiff/verdict.py +1513 -0
  41. quantdiff-0.1.0rc1.dist-info/METADATA +514 -0
  42. quantdiff-0.1.0rc1.dist-info/RECORD +45 -0
  43. quantdiff-0.1.0rc1.dist-info/WHEEL +4 -0
  44. quantdiff-0.1.0rc1.dist-info/entry_points.txt +2 -0
  45. quantdiff-0.1.0rc1.dist-info/licenses/LICENSE +202 -0
quantdiff/png.py ADDED
@@ -0,0 +1,360 @@
1
+ """PNG export of the HTML scorecard through an installed Chromium-family browser.
2
+
3
+ A headless browser renders the exact HTML card, so the PNG is pixel-identical to what the
4
+ card looks like when opened, and quantdiff keeps zero Python dependencies.
5
+
6
+ The browser cannot size a screenshot to its content, so rendering takes two launches:
7
+
8
+ 1. Measure: a wrapper page loads the card into an iframe of the target width and, once it
9
+ has loaded, writes the card's full height into a meta tag. ``--dump-dom`` prints the
10
+ resulting DOM and the height is parsed from it. The iframe pins the layout width
11
+ exactly, which a top-level page cannot do because ``--dump-dom`` subtracts window
12
+ decorations from ``--window-size``. The card itself stays free of scripts.
13
+ 2. Screenshot: the card is captured with a window of exactly that width and height.
14
+
15
+ Every launch uses a throwaway profile in a private temporary directory and forces the
16
+ light color scheme so the output does not depend on the user's settings.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import logging
22
+ import os
23
+ import re
24
+ import shutil
25
+ import subprocess
26
+ import sys
27
+ import tempfile
28
+ import time
29
+ from collections.abc import Mapping, Sequence
30
+ from html import escape
31
+ from pathlib import Path
32
+ from typing import Final
33
+
34
+ from quantdiff.errors import RenderError
35
+
36
+ __all__ = [
37
+ "BROWSER_ENV",
38
+ "find_browser",
39
+ "max_height",
40
+ "measure_command",
41
+ "measure_page",
42
+ "parse_height",
43
+ "platform_candidates",
44
+ "render_png",
45
+ "screenshot_command",
46
+ ]
47
+
48
+ logger = logging.getLogger(__name__)
49
+
50
+ BROWSER_ENV: Final = "QUANTDIFF_BROWSER"
51
+ MIN_WIDTH: Final = 320
52
+ MAX_WIDTH: Final = 4096
53
+ MIN_SCALE: Final = 0.5
54
+ MAX_SCALE: Final = 4.0
55
+ MAX_HEIGHT: Final = 20_000
56
+ """Tallest card, in CSS pixels, that is screenshotted; anything taller is clipped."""
57
+ MAX_PNG_PIXELS: Final = 100_000_000
58
+ """Largest image area, in device pixels, so a huge width and scale cannot exhaust memory."""
59
+ _STDERR_TAIL_CHARS: Final = 600
60
+ # --dump-dom shrinks the viewport by the window frame; the slack keeps the iframe unclipped.
61
+ _MEASURE_WINDOW_SLACK: Final = 120
62
+ _MEASURE_WINDOW_HEIGHT: Final = 200
63
+ _PNG_SIGNATURE: Final = b"\x89PNG\r\n\x1a\n"
64
+ _HEIGHT_META: Final = re.compile(r'<meta name="quantdiff-height" content="(\d+)">')
65
+ _NO_BROWSER: Final = (
66
+ "no Chromium-based browser found (Chrome, Edge, Brave or Chromium); "
67
+ f"install one or set {BROWSER_ENV} to its path"
68
+ )
69
+
70
+ _WINDOWS_BROWSERS: Final = (
71
+ ("Microsoft", "Edge", "Application", "msedge.exe"),
72
+ ("Google", "Chrome", "Application", "chrome.exe"),
73
+ ("BraveSoftware", "Brave-Browser", "Application", "brave.exe"),
74
+ ("Chromium", "Application", "chrome.exe"),
75
+ )
76
+ _WINDOWS_ROOTS: Final = ("ProgramFiles", "ProgramFiles(x86)", "LocalAppData")
77
+ _MACOS_BROWSERS: Final = (
78
+ "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
79
+ "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
80
+ "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser",
81
+ "/Applications/Chromium.app/Contents/MacOS/Chromium",
82
+ )
83
+ _LINUX_BROWSERS: Final = (
84
+ "google-chrome",
85
+ "google-chrome-stable",
86
+ "chromium",
87
+ "chromium-browser",
88
+ "microsoft-edge",
89
+ "brave-browser",
90
+ )
91
+
92
+ _MEASURE_SCRIPT: Final = """
93
+ window.addEventListener("load", function () {
94
+ var card = document.getElementById("card").contentDocument.documentElement;
95
+ var meta = document.createElement("meta");
96
+ meta.name = "quantdiff-height";
97
+ meta.content = String(Math.ceil(card.scrollHeight));
98
+ document.head.appendChild(meta);
99
+ });
100
+ """
101
+
102
+
103
+ # Browser discovery ------------------------------------------------------------------------
104
+
105
+
106
+ def find_browser() -> Path | None:
107
+ """Locate a Chromium-family browser, or return None when none is installed.
108
+
109
+ The QUANTDIFF_BROWSER environment variable wins when set and must name an existing
110
+ file. Otherwise the standard install locations for the current platform are probed.
111
+ """
112
+ explicit = os.environ.get(BROWSER_ENV)
113
+ if explicit:
114
+ path = Path(explicit).expanduser()
115
+ if not path.is_file():
116
+ raise RenderError(f"{BROWSER_ENV} is set to {explicit!r}, which is not a file")
117
+ return path
118
+ for candidate in platform_candidates(sys.platform, os.environ):
119
+ if candidate.is_file():
120
+ return candidate
121
+ return None
122
+
123
+
124
+ def platform_candidates(platform: str, environ: Mapping[str, str]) -> list[Path]:
125
+ """List the usual browser install paths for `platform`, most preferred first."""
126
+ if platform == "win32":
127
+ return _windows_candidates(environ)
128
+ if platform == "darwin":
129
+ return [Path(path) for path in _MACOS_BROWSERS]
130
+ found = (shutil.which(name) for name in _LINUX_BROWSERS)
131
+ return [Path(path) for path in found if path]
132
+
133
+
134
+ def _windows_candidates(environ: Mapping[str, str]) -> list[Path]:
135
+ roots = [Path(environ[name]) for name in _WINDOWS_ROOTS if environ.get(name)]
136
+ return [root.joinpath(*parts) for parts in _WINDOWS_BROWSERS for root in roots]
137
+
138
+
139
+ # Commands and pages -----------------------------------------------------------------------
140
+
141
+
142
+ def _common_flags(profile_dir: Path, scale: float) -> list[str]:
143
+ return [
144
+ "--headless=new",
145
+ # A private profile keeps the user's browser profile, cookies and extensions out.
146
+ f"--user-data-dir={profile_dir}",
147
+ "--no-first-run",
148
+ "--no-default-browser-check",
149
+ "--disable-extensions",
150
+ "--disable-default-apps",
151
+ "--disable-gpu",
152
+ "--disable-background-networking",
153
+ "--disable-sync",
154
+ "--disable-component-update",
155
+ "--mute-audio",
156
+ "--hide-scrollbars",
157
+ # 1 is Blink's light scheme; the card follows prefers-color-scheme otherwise.
158
+ "--blink-settings=preferredColorScheme=1",
159
+ f"--force-device-scale-factor={scale:g}",
160
+ ]
161
+
162
+
163
+ def measure_command(
164
+ browser: Path, page: Path, profile_dir: Path, *, width: int, scale: float
165
+ ) -> list[str]:
166
+ """Build the argv that prints the DOM of the measure page after it has loaded."""
167
+ return [
168
+ str(browser),
169
+ *_common_flags(profile_dir, scale),
170
+ f"--window-size={width + _MEASURE_WINDOW_SLACK},{_MEASURE_WINDOW_HEIGHT}",
171
+ "--dump-dom",
172
+ page.as_uri(),
173
+ ]
174
+
175
+
176
+ def screenshot_command(
177
+ browser: Path,
178
+ page: Path,
179
+ output: Path,
180
+ profile_dir: Path,
181
+ *,
182
+ width: int,
183
+ height: int,
184
+ scale: float,
185
+ ) -> list[str]:
186
+ """Build the argv that screenshots `page` into `output` at exactly width x height."""
187
+ return [
188
+ str(browser),
189
+ *_common_flags(profile_dir, scale),
190
+ f"--window-size={width},{height}",
191
+ f"--screenshot={output}",
192
+ page.as_uri(),
193
+ ]
194
+
195
+
196
+ def measure_page(card_html: str, width: int) -> str:
197
+ """Wrap the card in an iframe `width` CSS pixels wide plus a script that records its height.
198
+
199
+ The iframe is 1px tall so the card's scrollHeight is its content height, not the
200
+ viewport's.
201
+ """
202
+ return "\n".join(
203
+ [
204
+ "<!doctype html>",
205
+ '<html lang="en">',
206
+ "<head>",
207
+ '<meta charset="utf-8">',
208
+ "<style>html, body { margin: 0; } "
209
+ f"iframe {{ display: block; border: 0; width: {width}px; height: 1px; }}</style>",
210
+ "</head>",
211
+ "<body>",
212
+ f'<iframe id="card" srcdoc="{escape(card_html, quote=True)}"></iframe>',
213
+ f"<script>{_MEASURE_SCRIPT}</script>",
214
+ "</body>",
215
+ "</html>",
216
+ "",
217
+ ]
218
+ )
219
+
220
+
221
+ def parse_height(dom: str) -> int:
222
+ """Read the card height that the measure page wrote into its DOM."""
223
+ match = _HEIGHT_META.search(dom)
224
+ if match is None:
225
+ raise RenderError("the browser did not report the card height; is it Chromium-based?")
226
+ height = int(match.group(1))
227
+ if height <= 0:
228
+ raise RenderError(f"the browser reported a card height of {height}px")
229
+ return height
230
+
231
+
232
+ # Rendering --------------------------------------------------------------------------------
233
+
234
+
235
+ def render_png(
236
+ html: str,
237
+ out_path: Path,
238
+ *,
239
+ width: int = 1040,
240
+ scale: float = 2.0,
241
+ browser: Path | None = None,
242
+ timeout_seconds: float = 60.0,
243
+ ) -> Path:
244
+ """Render `html` to a PNG at `out_path` and return that path.
245
+
246
+ The image is `width * scale` pixels wide and exactly as tall as the page content
247
+ (capped by max_height()). `browser` defaults to find_browser(). Raises
248
+ RenderError when no browser is available, the browser fails or times out, or the
249
+ output is not a PNG.
250
+ """
251
+ _check_size(width, scale)
252
+ executable = browser if browser is not None else find_browser()
253
+ if executable is None:
254
+ raise RenderError(_NO_BROWSER)
255
+ started = time.monotonic()
256
+ with tempfile.TemporaryDirectory(prefix="quantdiff-png-", ignore_cleanup_errors=True) as tmp:
257
+ workdir = Path(tmp)
258
+ profile = workdir / "profile"
259
+ measure = workdir / "measure.html"
260
+ card = workdir / "card.html"
261
+ shot = workdir / "card.png"
262
+ measure.write_text(measure_page(html, width), encoding="utf-8")
263
+ card.write_text(html, encoding="utf-8")
264
+
265
+ command = measure_command(executable, measure, profile, width=width, scale=scale)
266
+ height = parse_height(_run_browser(command, workdir, timeout_seconds))
267
+ limit = max_height(width, scale)
268
+ if height > limit:
269
+ logger.warning("card is %dpx tall; clipping the PNG to %dpx", height, limit)
270
+ height = limit
271
+ command = screenshot_command(
272
+ executable, card, shot, profile, width=width, height=height, scale=scale
273
+ )
274
+ _run_browser(command, workdir, timeout_seconds)
275
+ _check_png(shot)
276
+ out_path.parent.mkdir(parents=True, exist_ok=True)
277
+ shutil.copyfile(shot, out_path)
278
+ logger.debug("rendered %s in %.1fs", out_path, time.monotonic() - started)
279
+ return out_path
280
+
281
+
282
+ def _check_size(width: int, scale: float) -> None:
283
+ if not MIN_WIDTH <= width <= MAX_WIDTH:
284
+ raise RenderError(f"PNG width must be {MIN_WIDTH} to {MAX_WIDTH} CSS pixels, got {width}")
285
+ if not MIN_SCALE <= scale <= MAX_SCALE:
286
+ raise RenderError(f"PNG scale must be {MIN_SCALE:g} to {MAX_SCALE:g}, got {scale:g}")
287
+
288
+
289
+ def max_height(width: int, scale: float) -> int:
290
+ """Tallest screenshot, in CSS pixels, that stays within MAX_HEIGHT and MAX_PNG_PIXELS."""
291
+ return min(MAX_HEIGHT, int(MAX_PNG_PIXELS / (width * scale * scale)))
292
+
293
+
294
+ def _run_browser(command: Sequence[str], workdir: Path, timeout_seconds: float) -> str:
295
+ """Run the browser and return its stdout. Output goes to files, not pipes, because
296
+ browser child processes can hold inherited pipes open after the parent is killed."""
297
+ stdout_path = workdir / "stdout.txt"
298
+ stderr_path = workdir / "stderr.txt"
299
+ with stdout_path.open("wb") as stdout, stderr_path.open("wb") as stderr:
300
+ try:
301
+ process = subprocess.Popen( # noqa: S603 - argv list, no shell; browser path comes from find_browser or the caller
302
+ command,
303
+ cwd=workdir,
304
+ stdin=subprocess.DEVNULL,
305
+ stdout=stdout,
306
+ stderr=stderr,
307
+ start_new_session=_NEW_SESSION,
308
+ )
309
+ except OSError as exc:
310
+ raise RenderError(f"could not start the browser {command[0]}: {exc}") from exc
311
+ try:
312
+ returncode = process.wait(timeout=timeout_seconds)
313
+ except subprocess.TimeoutExpired:
314
+ _kill(process)
315
+ process.wait()
316
+ raise RenderError(
317
+ f"the browser did not finish within {timeout_seconds:g}s{_stderr_tail(stderr_path)}"
318
+ ) from None
319
+ if returncode != 0:
320
+ raise RenderError(f"the browser exited with code {returncode}{_stderr_tail(stderr_path)}")
321
+ return stdout_path.read_text(encoding="utf-8", errors="replace")
322
+
323
+
324
+ def _stderr_tail(path: Path) -> str:
325
+ try:
326
+ text = path.read_text(encoding="utf-8", errors="replace").strip()
327
+ except OSError:
328
+ return ""
329
+ if not text:
330
+ return ""
331
+ return f"; browser output ends with: {text[-_STDERR_TAIL_CHARS:]}"
332
+
333
+
334
+ def _check_png(path: Path) -> None:
335
+ try:
336
+ with path.open("rb") as handle:
337
+ header = handle.read(len(_PNG_SIGNATURE))
338
+ except OSError as exc:
339
+ raise RenderError(f"the browser did not write a screenshot: {exc}") from exc
340
+ if header != _PNG_SIGNATURE:
341
+ raise RenderError(f"the browser wrote {path.name}, but it is not a PNG image")
342
+
343
+
344
+ if sys.platform == "win32":
345
+ _NEW_SESSION: Final = False
346
+
347
+ def _kill(process: subprocess.Popen[bytes]) -> None:
348
+ # Chromium child processes exit on their own once the browser process is gone.
349
+ process.kill()
350
+
351
+ else:
352
+ import signal
353
+
354
+ _NEW_SESSION: Final = True
355
+
356
+ def _kill(process: subprocess.Popen[bytes]) -> None:
357
+ try:
358
+ os.killpg(process.pid, signal.SIGKILL)
359
+ except ProcessLookupError:
360
+ return