llmdyno 0.3.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. llmdyno/__init__.py +50 -0
  2. llmdyno/__main__.py +54 -0
  3. llmdyno/api/__init__.py +0 -0
  4. llmdyno/api/external_api.py +181 -0
  5. llmdyno/api/results_api.py +225 -0
  6. llmdyno/api/routes.py +1198 -0
  7. llmdyno/benchmarks/__init__.py +0 -0
  8. llmdyno/benchmarks/artificial_analysis.py +175 -0
  9. llmdyno/benchmarks/builtin/t0_smoke_cli/benchmark.json +46 -0
  10. llmdyno/benchmarks/builtin/t0_smoke_cli/grader/spec.json +8 -0
  11. llmdyno/benchmarks/builtin/t0_smoke_cli/grader/tests/test_hidden_wordfreq.py +39 -0
  12. llmdyno/benchmarks/builtin/t0_smoke_cli/prompts/stage1.md +12 -0
  13. llmdyno/benchmarks/builtin/t0_smoke_cli/reference/tests/test_wordfreq.py +13 -0
  14. llmdyno/benchmarks/builtin/t0_smoke_cli/reference/wordfreq.py +34 -0
  15. llmdyno/benchmarks/builtin/t1_flask_site/benchmark.json +49 -0
  16. llmdyno/benchmarks/builtin/t1_flask_site/grader/grade.py +24 -0
  17. llmdyno/benchmarks/builtin/t1_flask_site/grader/tests/test_hidden_flask.py +170 -0
  18. llmdyno/benchmarks/builtin/t1_flask_site/probes/test_probes.py +75 -0
  19. llmdyno/benchmarks/builtin/t1_flask_site/prompts/stage1.md +20 -0
  20. llmdyno/benchmarks/builtin/t1_flask_site/reference/README.md +3 -0
  21. llmdyno/benchmarks/builtin/t1_flask_site/reference/app.py +99 -0
  22. llmdyno/benchmarks/builtin/t1_flask_site/reference/requirements.txt +3 -0
  23. llmdyno/benchmarks/builtin/t1_flask_site/reference/templates/base.html +1 -0
  24. llmdyno/benchmarks/builtin/t1_flask_site/reference/tests/test_app.py +17 -0
  25. llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/benchmark.json +49 -0
  26. llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/grader/grade.py +56 -0
  27. llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/grader/tests/test_hidden_jobs.py +139 -0
  28. llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/probes/test_probes.py +89 -0
  29. llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/prompts/stage1.md +50 -0
  30. llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/README.md +5 -0
  31. llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/backend/__init__.py +0 -0
  32. llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/backend/db.py +18 -0
  33. llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/backend/main.py +70 -0
  34. llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/backend/tests/test_api.py +25 -0
  35. llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/frontend/index.html +8 -0
  36. llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/frontend/src/api.ts +31 -0
  37. llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/frontend/src/main.ts +16 -0
  38. llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/frontend/tsconfig.json +11 -0
  39. llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/requirements.txt +4 -0
  40. llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/worker/__init__.py +0 -0
  41. llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/worker/worker.py +49 -0
  42. llmdyno/benchmarks/builtin/t3_long_iterative/benchmark.json +69 -0
  43. llmdyno/benchmarks/builtin/t3_long_iterative/grader/grade.py +44 -0
  44. llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/helpers.py +11 -0
  45. llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h1.py +57 -0
  46. llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h1_jsonformat.py +11 -0
  47. llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h2.py +76 -0
  48. llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h2_jsonformat.py +14 -0
  49. llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h3.py +68 -0
  50. llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h4.py +85 -0
  51. llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h5.py +102 -0
  52. llmdyno/benchmarks/builtin/t3_long_iterative/probes/test_probes.py +47 -0
  53. llmdyno/benchmarks/builtin/t3_long_iterative/prompts/stage1.md +21 -0
  54. llmdyno/benchmarks/builtin/t3_long_iterative/prompts/stage2.md +15 -0
  55. llmdyno/benchmarks/builtin/t3_long_iterative/prompts/stage3.md +12 -0
  56. llmdyno/benchmarks/builtin/t3_long_iterative/prompts/stage4.md +11 -0
  57. llmdyno/benchmarks/builtin/t3_long_iterative/prompts/stage5.md +19 -0
  58. llmdyno/benchmarks/builtin/t3_long_iterative/reference/inventory/__init__.py +0 -0
  59. llmdyno/benchmarks/builtin/t3_long_iterative/reference/inventory/__main__.py +62 -0
  60. llmdyno/benchmarks/builtin/t3_long_iterative/reference/inventory/errors.py +14 -0
  61. llmdyno/benchmarks/builtin/t3_long_iterative/reference/inventory/models.py +28 -0
  62. llmdyno/benchmarks/builtin/t3_long_iterative/reference/inventory/store.py +114 -0
  63. llmdyno/benchmarks/builtin/t3_long_iterative/reference/tests/test_own.py +24 -0
  64. llmdyno/benchmarks/builtin/t3_long_iterative/stage_files/stage1/tests/test_step1.py +34 -0
  65. llmdyno/benchmarks/builtin/t3_long_iterative/stage_files/stage2/tests/test_step2.py +22 -0
  66. llmdyno/benchmarks/builtin/t3_long_iterative/stage_files/stage3/tests/test_step3.py +22 -0
  67. llmdyno/benchmarks/builtin/t3_long_iterative/stage_files/stage4/tests/data/legacy.json +12 -0
  68. llmdyno/benchmarks/builtin/t3_long_iterative/stage_files/stage4/tests/test_step4.py +27 -0
  69. llmdyno/benchmarks/builtin/t3_long_iterative/stage_files/stage5/tests/test_step5.py +24 -0
  70. llmdyno/benchmarks/builtin/t4_data_ml/benchmark.json +50 -0
  71. llmdyno/benchmarks/builtin/t4_data_ml/gen_data.py +203 -0
  72. llmdyno/benchmarks/builtin/t4_data_ml/grader/grade.py +161 -0
  73. llmdyno/benchmarks/builtin/t4_data_ml/grader/test_labels.csv +1201 -0
  74. llmdyno/benchmarks/builtin/t4_data_ml/grader/truth.json +22 -0
  75. llmdyno/benchmarks/builtin/t4_data_ml/grader/truth_features.parquet +0 -0
  76. llmdyno/benchmarks/builtin/t4_data_ml/probes/test_probes.py +87 -0
  77. llmdyno/benchmarks/builtin/t4_data_ml/prompts/stage1.md +43 -0
  78. llmdyno/benchmarks/builtin/t4_data_ml/reference/pipeline.py +77 -0
  79. llmdyno/benchmarks/builtin/t4_data_ml/workspace/data/customers.csv +4151 -0
  80. llmdyno/benchmarks/builtin/t4_data_ml/workspace/data/events.jsonl.gz +0 -0
  81. llmdyno/benchmarks/builtin/t4_data_ml/workspace/data/labels_train.csv +2801 -0
  82. llmdyno/benchmarks/builtin/t4_data_ml/workspace/data/test_ids.csv +1201 -0
  83. llmdyno/benchmarks/builtin/t5_bugfix_semver/benchmark.json +47 -0
  84. llmdyno/benchmarks/builtin/t5_bugfix_semver/grader/grade.py +31 -0
  85. llmdyno/benchmarks/builtin/t5_bugfix_semver/grader/tests/test_hidden_semver.py +107 -0
  86. llmdyno/benchmarks/builtin/t5_bugfix_semver/prompts/stage1.md +10 -0
  87. llmdyno/benchmarks/builtin/t5_bugfix_semver/reference/semverlite/version.py +87 -0
  88. llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/CHANGELOG.md +12 -0
  89. llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/ISSUE.md +17 -0
  90. llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/README.md +13 -0
  91. llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/semverlite/__init__.py +5 -0
  92. llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/semverlite/ranges.py +137 -0
  93. llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/semverlite/version.py +83 -0
  94. llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/tests/test_basic.py +60 -0
  95. llmdyno/benchmarks/common/grading.py +61 -0
  96. llmdyno/benchmarks/common/polyglot_grade.py +107 -0
  97. llmdyno/benchmarks/common/spec_grade.py +42 -0
  98. llmdyno/benchmarks/envs.py +147 -0
  99. llmdyno/benchmarks/polyglot.py +276 -0
  100. llmdyno/benchmarks/registry.py +270 -0
  101. llmdyno/benchmarks/swebench.py +361 -0
  102. llmdyno/benchmarks/swebench_shim.py +28 -0
  103. llmdyno/benchmarks/validate.py +87 -0
  104. llmdyno/config.py +107 -0
  105. llmdyno/db.py +220 -0
  106. llmdyno/docker/polyglot-test.sh +18 -0
  107. llmdyno/docker/polyglot.Dockerfile +44 -0
  108. llmdyno/docker/quality.Dockerfile +7 -0
  109. llmdyno/docker/sandbox.Dockerfile +33 -0
  110. llmdyno/events.py +59 -0
  111. llmdyno/harnesses/__init__.py +0 -0
  112. llmdyno/harnesses/catalog.py +424 -0
  113. llmdyno/harnesses/images.py +143 -0
  114. llmdyno/harnesses/runtime.py +112 -0
  115. llmdyno/hub/__init__.py +0 -0
  116. llmdyno/hub/gguf.py +270 -0
  117. llmdyno/hub/hf.py +238 -0
  118. llmdyno/inference/__init__.py +0 -0
  119. llmdyno/inference/llamacpp.py +450 -0
  120. llmdyno/jobs.py +153 -0
  121. llmdyno/judge/__init__.py +0 -0
  122. llmdyno/judge/judge.py +271 -0
  123. llmdyno/lib.py +995 -0
  124. llmdyno/llm.py +229 -0
  125. llmdyno/memory.py +310 -0
  126. llmdyno/profiles.py +103 -0
  127. llmdyno/providers/__init__.py +0 -0
  128. llmdyno/providers/registry.py +156 -0
  129. llmdyno/proxy/llm_proxy.py +630 -0
  130. llmdyno/py.typed +0 -0
  131. llmdyno/quality/__init__.py +0 -0
  132. llmdyno/quality/analyze.py +361 -0
  133. llmdyno/runner/__init__.py +0 -0
  134. llmdyno/runner/executor.py +770 -0
  135. llmdyno/runner/metrics.py +188 -0
  136. llmdyno/runner/sandbox.py +275 -0
  137. llmdyno/safety.py +120 -0
  138. llmdyno/sample/round1.json +1 -0
  139. llmdyno/scoring.py +202 -0
  140. llmdyno/secrets.py +94 -0
  141. llmdyno/server.py +148 -0
  142. llmdyno/smoke.py +107 -0
  143. llmdyno/system/__init__.py +0 -0
  144. llmdyno/system/apple.py +156 -0
  145. llmdyno/system/deps.py +237 -0
  146. llmdyno/system/specs.py +434 -0
  147. llmdyno/web/static/assets/index-DDIFnEmp.css +1 -0
  148. llmdyno/web/static/assets/index-DOnMvXno.js +612 -0
  149. llmdyno/web/static/assets/inter-cyrillic-ext-wght-normal-B2xhLi22.woff2 +0 -0
  150. llmdyno/web/static/assets/inter-cyrillic-wght-normal-CMZtQduZ.woff2 +0 -0
  151. llmdyno/web/static/assets/inter-greek-ext-wght-normal-CGAr0uHJ.woff2 +0 -0
  152. llmdyno/web/static/assets/inter-greek-wght-normal-CaVNZxsx.woff2 +0 -0
  153. llmdyno/web/static/assets/inter-latin-ext-wght-normal-CFHvXkgd.woff2 +0 -0
  154. llmdyno/web/static/assets/inter-latin-wght-normal-C2S99t-D.woff2 +0 -0
  155. llmdyno/web/static/assets/inter-vietnamese-wght-normal-CBcvBZtf.woff2 +0 -0
  156. llmdyno/web/static/assets/jetbrains-mono-cyrillic-wght-normal-BptiBwAt.woff2 +0 -0
  157. llmdyno/web/static/assets/jetbrains-mono-greek-wght-normal-BW1JPxja.woff2 +0 -0
  158. llmdyno/web/static/assets/jetbrains-mono-latin-ext-wght-normal-Com2F4yK.woff2 +0 -0
  159. llmdyno/web/static/assets/jetbrains-mono-latin-wght-normal-Ajscav9D.woff2 +0 -0
  160. llmdyno/web/static/assets/jetbrains-mono-vietnamese-wght-normal-By-yputv.woff2 +0 -0
  161. llmdyno/web/static/assets/press-start-2p-cyrillic-400-normal-oElOGqNf.woff2 +0 -0
  162. llmdyno/web/static/assets/press-start-2p-cyrillic-ext-400-normal-COIQET2s.woff2 +0 -0
  163. llmdyno/web/static/assets/press-start-2p-greek-400-normal-DUIT0YPZ.woff2 +0 -0
  164. llmdyno/web/static/assets/press-start-2p-latin-400-normal-BKfmdmKK.woff +0 -0
  165. llmdyno/web/static/assets/press-start-2p-latin-400-normal-xj9lRvVs.woff2 +0 -0
  166. llmdyno/web/static/assets/press-start-2p-latin-ext-400-normal-B18ub5-k.woff +0 -0
  167. llmdyno/web/static/assets/press-start-2p-latin-ext-400-normal-DDIa4im3.woff2 +0 -0
  168. llmdyno/web/static/favicon.svg +161 -0
  169. llmdyno/web/static/index.html +17 -0
  170. llmdyno-0.3.1.dist-info/METADATA +421 -0
  171. llmdyno-0.3.1.dist-info/RECORD +175 -0
  172. llmdyno-0.3.1.dist-info/WHEEL +4 -0
  173. llmdyno-0.3.1.dist-info/entry_points.txt +2 -0
  174. llmdyno-0.3.1.dist-info/licenses/LICENSE +21 -0
  175. llmdyno-0.3.1.dist-info/licenses/NOTICE.md +33 -0
llmdyno/__init__.py ADDED
@@ -0,0 +1,50 @@
1
+ """LLMDyno: benchmark local (llama.cpp) and cloud LLMs as coding agents.
2
+
3
+ Use it as a library:
4
+
5
+ import llmdyno
6
+
7
+ gpt = llmdyno.cloud_model("openai", "gpt-6-luna", reasoning_effort="low")
8
+ qwen = llmdyno.local_model("unsloth/Qwen3.8-27B-GGUF", quant="UD-Q3_K_XL", ctx=49152)
9
+ results = llmdyno.bench([gpt, qwen], benchmarks="suite_quick", harnesses="pi")
10
+ print(results)
11
+
12
+ or as an app: `llmdyno serve --open` (or llmdyno.serve()). Both use the same data folder. See docs/LIBRARY.md.
13
+
14
+ The version follows Semantic Versioning. While it is 0.x, a minor release (0.2.0) may change the API, the data
15
+ folder layout or the database schema (with an automatic migration); patch releases (0.1.1) only fix things.
16
+ """
17
+ from typing import TYPE_CHECKING
18
+
19
+ __version__ = "0.3.1"
20
+
21
+ # The public API lives in llmdyno.lib and is loaded on first use, so `import llmdyno` stays instant and starts
22
+ # nothing (no database, no Docker, no web server) until you call something. Public names must never equal a
23
+ # submodule's name (profiles, benchmarks, judge, …): Python would set the module over the function, or the reverse.
24
+ _PUBLIC = (
25
+ "LLMDynoError", "NotFoundError", "ConflictError", "PreflightError", "Profile", "Plan", "Results", "Doctor",
26
+ "init", "data_dir", "doctor", "setup", "cloud_model", "local_model", "list_profiles", "profile", "list_benchmarks",
27
+ "list_suites", "import_swebench", "import_polyglot", "preflight", "bench", "list_plans", "plan", "results", "review",
28
+ "serve",
29
+ )
30
+ __all__ = ["__version__", *_PUBLIC]
31
+
32
+ if TYPE_CHECKING: # pragma: no cover - for IDEs and type checkers
33
+ from .lib import ( # noqa: F401
34
+ ConflictError, Doctor, LLMDynoError, NotFoundError, Plan, PreflightError, Profile, Results, bench, cloud_model,
35
+ data_dir, doctor, import_polyglot, import_swebench, init, list_benchmarks, list_plans, list_profiles, list_suites,
36
+ local_model, plan, preflight, profile, results, review, serve, setup,
37
+ )
38
+
39
+
40
+ def __getattr__(name):
41
+ if name in _PUBLIC:
42
+ from . import lib
43
+ value = getattr(lib, name)
44
+ globals()[name] = value
45
+ return value
46
+ raise AttributeError(f"module 'llmdyno' has no attribute {name!r}")
47
+
48
+
49
+ def __dir__():
50
+ return sorted(set(globals()) | set(_PUBLIC))
llmdyno/__main__.py ADDED
@@ -0,0 +1,54 @@
1
+ """Command line: `llmdyno` starts the web app; `llmdyno doctor` prints the setup checks.
2
+ From Python, `import llmdyno` gives the same features as a library (see docs/LIBRARY.md)."""
3
+ import argparse
4
+
5
+ from . import __version__
6
+ import json
7
+ import sys
8
+ import threading
9
+ import webbrowser
10
+
11
+
12
+ def main(argv=None):
13
+ ap = argparse.ArgumentParser(prog="llmdyno", description="Benchmark LLMs as coding agents.")
14
+ ap.add_argument("--version", action="version", version=f"llmdyno {__version__}")
15
+ sub = ap.add_subparsers(dest="cmd")
16
+ sv = sub.add_parser("serve", help="start the web app (default)")
17
+ sv.add_argument("--host", default="127.0.0.1")
18
+ sv.add_argument("--port", type=int, default=None)
19
+ sv.add_argument("--open", action="store_true", help="open the browser")
20
+ sub.add_parser("doctor", help="check system specs and dependencies")
21
+ sub.add_parser("specs", help="print detected hardware as JSON")
22
+ a = ap.parse_args(argv)
23
+ from . import config, db
24
+ db.init()
25
+ if a.cmd == "doctor":
26
+ from .system import deps, specs
27
+ s = specs.detect(force=True)
28
+ if not s["os"]["arch_supported"]:
29
+ print(f"Unsupported architecture {s['os']['arch']}: LLMDyno supports x86-64 and ARM64.")
30
+ return 2
31
+ for n in s["summary"]["notes"]:
32
+ print(f"[{n['level']}] {n['text']}")
33
+ print()
34
+ for d in deps.check(force=True):
35
+ mark = "OK " if d["ok"] else ("!! " if d["level"] == "required" else "-- ")
36
+ print(f"{mark}{d['name']:<22} {d['level']:<26} {d['detail']}")
37
+ return 0
38
+ if a.cmd == "specs":
39
+ from .system import specs
40
+ print(json.dumps(specs.detect(force=True), indent=2, default=str))
41
+ return 0
42
+ import uvicorn
43
+ host = getattr(a, "host", "127.0.0.1")
44
+ port = getattr(a, "port", None) or int(config.get("port"))
45
+ url = f"http://{host}:{port}"
46
+ print(f"LLMDyno on {url} (data: {config.DATA})", flush=True)
47
+ if getattr(a, "open", False):
48
+ threading.Timer(1.5, lambda: webbrowser.open(url)).start()
49
+ uvicorn.run("llmdyno.server:app", host=host, port=port, log_level="warning")
50
+ return 0
51
+
52
+
53
+ if __name__ == "__main__":
54
+ sys.exit(main())
File without changes
@@ -0,0 +1,181 @@
1
+ """External benchmarks and references: SWE-bench and Aider polyglot imports, the official SWE-bench harness, SWE-bench
2
+ predictions export, portable harness runtimes and Artificial Analysis reference numbers."""
3
+ import json
4
+ import re
5
+
6
+ from fastapi import APIRouter, Body, HTTPException
7
+ from fastapi.responses import Response
8
+
9
+ from .. import config, db, jobs, secrets
10
+ from ..benchmarks import artificial_analysis as aa
11
+ from ..benchmarks import polyglot, swebench
12
+ from ..benchmarks import registry as benches
13
+ from ..harnesses import catalog, images, runtime
14
+
15
+ router = APIRouter()
16
+
17
+
18
+ # ------------------------------------------------------------------ catalog of importable benchmarks
19
+ def _count(prefix):
20
+ return sum(1 for b in benches.all_benchmarks() if b["id"].startswith(prefix) and not b["builtin"])
21
+
22
+
23
+ @router.get("/imports")
24
+ def import_catalog():
25
+ sw = swebench.harness_status()
26
+ poly = images.image_exists(polyglot.IMAGE)
27
+ return {
28
+ "swebench": {"datasets": [{"id": k, "name": v["name"], "size": v["size"], "about": v["about"], "hf": v["hf"]}
29
+ for k, v in swebench.DATASETS.items()],
30
+ "imported": _count("swe_"), "harness": sw, "homepage": swebench.HOMEPAGE,
31
+ "harness_version": (config.get("swebench") or {}).get("harness_version")},
32
+ "polyglot": {"languages": polyglot.LANGS, "imported": _count("pg_"), "image": polyglot.IMAGE,
33
+ "image_info": poly, "context_ready": polyglot.context_ready(), "homepage": polyglot.HOMEPAGE,
34
+ "repo": f"https://github.com/{polyglot.REPO}"},
35
+ "runtime_platform": runtime.host_platform(),
36
+ }
37
+
38
+
39
+ @router.post("/imports/swebench")
40
+ def import_swebench(body: dict = Body(...)):
41
+ key = body.get("dataset", "verified_mini")
42
+ if key not in swebench.DATASETS:
43
+ raise HTTPException(400, f"unknown dataset {key}")
44
+ ids = [x for x in body.get("instance_ids") or [] if re.fullmatch(r"[\w.\-]+__[\w.\-]+-\d+", x)]
45
+ limit = int(body.get("limit") or 0)
46
+ if limit < 0 or limit > 2000:
47
+ raise HTTPException(400, "limit must be 0-2000")
48
+ name = swebench.DATASETS[key]["name"]
49
+ return {"job_id": jobs.submit("import", f"Import {name}" + (f" (first {limit})" if limit else ""),
50
+ swebench.import_job, {"dataset": key, "instance_ids": ids, "limit": limit},
51
+ ref="swebench", group="import")}
52
+
53
+
54
+ @router.post("/imports/polyglot")
55
+ def import_polyglot(body: dict = Body(default={})):
56
+ langs = [x for x in body.get("languages") or list(polyglot.LANGS) if x in polyglot.LANGS]
57
+ if not langs:
58
+ raise HTTPException(400, "choose at least one language")
59
+ return {"job_id": jobs.submit("import", "Import Aider polyglot", polyglot.import_job, {"languages": langs},
60
+ ref="polyglot", group="import")}
61
+
62
+
63
+ @router.post("/imports/swebench/harness")
64
+ def install_swebench_harness():
65
+ v = (config.get("swebench") or {}).get("harness_version")
66
+ return {"job_id": jobs.submit("install", f"Install SWE-bench harness {v}", swebench.install_harness, {"version": v},
67
+ ref="swebench_harness")}
68
+
69
+
70
+ @router.post("/imports/polyglot/image")
71
+ def build_polyglot_image():
72
+ if not polyglot.context_ready():
73
+ raise HTTPException(409, "import the polyglot benchmark first")
74
+ return {"job_id": jobs.submit("build", "Build polyglot toolchain image", polyglot.build_image, ref="polyglot_image")}
75
+
76
+
77
+ @router.delete("/imports/{kind}")
78
+ def remove_imported(kind: str):
79
+ """Delete every imported benchmark of one kind (the dataset can be imported again at any time)."""
80
+ prefix = {"swebench": "swe_", "polyglot": "pg_"}.get(kind)
81
+ if not prefix:
82
+ raise HTTPException(404, "unknown import")
83
+ n = 0
84
+ for b in benches.all_benchmarks():
85
+ if b["id"].startswith(prefix) and not b["builtin"]:
86
+ benches.delete_custom(b["id"])
87
+ n += 1
88
+ with db.Session() as s:
89
+ s.query(db.Suite).filter(db.Suite.id.like(f"suite_{kind}%")).delete(synchronize_session=False)
90
+ s.commit()
91
+ return {"removed": n}
92
+
93
+
94
+ # ------------------------------------------------------------------ SWE-bench predictions
95
+ @router.get("/plans/{pid}/swebench-predictions")
96
+ def swebench_predictions(pid: str):
97
+ with db.Session() as s:
98
+ runs = [db.row(r) for r in s.query(db.Run).filter(db.Run.plan_id == pid, db.Run.status == "done").all()]
99
+ profs = {p.id: p.name for p in s.query(db.Profile).all()}
100
+ runs = [r for r in runs if r["benchmark_id"].startswith("swe_")]
101
+ if not runs:
102
+ raise HTTPException(404, "this plan has no finished SWE-bench runs")
103
+ for r in runs:
104
+ r["label"] = re.sub(r"[^A-Za-z0-9_.-]+", "-", f"{profs.get(r['profile_id'], r['profile_id'])}__{r['harness_id']}")
105
+ lines = swebench.predictions(runs)
106
+ body = "".join(json.dumps(x) + "\n" for x in lines)
107
+ return Response(body, media_type="application/x-ndjson",
108
+ headers={"Content-Disposition": f'attachment; filename="{pid}-swebench-predictions.jsonl"'})
109
+
110
+
111
+ # ------------------------------------------------------------------ harness runtimes
112
+ @router.get("/harnesses/{hid}/runtime")
113
+ def harness_runtime(hid: str, platform: str = ""):
114
+ h = catalog.get(hid, images.custom_manifests())
115
+ pf = platform if platform in runtime.PLATFORM_ARCH else runtime.host_platform()
116
+ return {**runtime.status(h, pf), "dockerfile": runtime.dockerfile(h)}
117
+
118
+
119
+ @router.post("/harnesses/{hid}/runtime")
120
+ def build_harness_runtime(hid: str, body: dict = Body(default={})):
121
+ h = catalog.get(hid, images.custom_manifests())
122
+ pf = body.get("platform") if body.get("platform") in runtime.PLATFORM_ARCH else runtime.host_platform()
123
+
124
+ def job(ctx):
125
+ ctx.result = runtime.ensure(ctx, h, pf, ctx.log)
126
+ return {"job_id": jobs.submit("build", f"Build {h.name} runtime ({pf})", job, {"harness_id": hid, "platform": pf},
127
+ ref=f"rt_{hid}")}
128
+
129
+
130
+ # ------------------------------------------------------------------ Artificial Analysis
131
+ @router.get("/references/aa")
132
+ def aa_reference():
133
+ from .results_api import _labels
134
+ profs = _labels()[0]
135
+ ref = aa.reference(profs)
136
+ return {**aa.status(), "matches": {pid: {"score": m["score"], "manual": m["manual"], "model": m["model"]}
137
+ for pid, m in ref.items()},
138
+ "model_list": [{k: m.get(k) for k in ("slug", "name", "creator")} for m in aa.load_cache().get("models") or []]}
139
+
140
+
141
+ @router.post("/references/aa/refresh")
142
+ def aa_refresh(body: dict = Body(default={})):
143
+ try:
144
+ c = aa.refresh(force=bool(body.get("force")))
145
+ except PermissionError as e:
146
+ raise HTTPException(400, str(e)) from e
147
+ except RuntimeError as e:
148
+ raise HTTPException(429, str(e)) from e
149
+ except Exception as e: # noqa: BLE001 - network and API errors go back to the user as text
150
+ raise HTTPException(502, f"Artificial Analysis request failed: {e}") from e
151
+ return {"models": len(c.get("models") or []), "fetched_at": c.get("fetched_at")}
152
+
153
+
154
+ @router.put("/settings/aa-key")
155
+ def set_aa_key(body: dict = Body(...)):
156
+ key = (body.get("key") or "").strip()
157
+ if not key:
158
+ raise HTTPException(400, "the key is empty")
159
+ secrets.set(aa.SECRET, key)
160
+ return aa.status()
161
+
162
+
163
+ @router.delete("/settings/aa-key")
164
+ def delete_aa_key():
165
+ secrets.delete(aa.SECRET)
166
+ return aa.status()
167
+
168
+
169
+ @router.put("/references/aa/override")
170
+ def aa_override(body: dict = Body(...)):
171
+ pid = body.get("profile_id")
172
+ if not pid:
173
+ raise HTTPException(400, "profile_id is required")
174
+ ov = dict(aa.overrides())
175
+ if body.get("slug"):
176
+ ov[pid] = str(body["slug"])[:120]
177
+ else:
178
+ ov.pop(pid, None)
179
+ cur = config.get("aa_reference") or {}
180
+ config.save_settings({"aa_reference": {**cur, "overrides": ov}})
181
+ return {"overrides": ov}
@@ -0,0 +1,225 @@
1
+ """Results: scored runs and configurations for the dashboard, judge agreement, exports and the sample dataset."""
2
+ import csv
3
+ import io
4
+ import json
5
+ import statistics
6
+ import time
7
+ from pathlib import Path
8
+
9
+ from fastapi import APIRouter, Body, HTTPException
10
+ from fastapi.responses import Response
11
+
12
+ from .. import __version__, config, db, events, scoring
13
+ from ..benchmarks import registry as benches
14
+ from ..harnesses import catalog, images
15
+ from ..judge import judge as judge_mod
16
+
17
+ router = APIRouter()
18
+ SAMPLE = Path(__file__).resolve().parent.parent / "sample" / "round1.json"
19
+ SAMPLE_JUDGES = {"judge_sample_luna": "openai", "judge_sample_claude": "anthropic"}
20
+ FLAT = ("total_prompt_tokens", "prefill_tokens", "cached_prompt_tokens", "generated_tokens", "reasoning_tokens",
21
+ "cache_hit_ratio", "peak_context_tokens", "context_utilization", "decode_tps", "prefill_tps", "ttft_median_s",
22
+ "model_time_s", "requests", "turns", "n_tool_calls", "tool_errors", "gpu_energy_wh", "gpu_peak_vram_mib",
23
+ "gpu_vram_total_mib", "spill_detected", "gpu_spill_mib", "tests_passed", "tests_total", "timed_out",
24
+ "length_truncations", "http_errors", "budget_blocked")
25
+
26
+
27
+ def _labels():
28
+ with db.Session() as s:
29
+ profs = {}
30
+ for p in s.query(db.Profile).all():
31
+ m = s.get(db.Model, p.model_id) if p.model_id else None
32
+ v = s.get(db.Provider, p.provider_id) if p.provider_id else None
33
+ profs[p.id] = {"name": p.name, "kind": p.kind, "quant": m.quant if m else "", "model": m.name if m else p.remote_model,
34
+ "vendor": v.vendor if v else None, "provider": v.name if v else None,
35
+ "arch": (m.meta or {}).get("arch") if m else None, "ctx": (p.params or {}).get("ctx"),
36
+ "size_bytes": m.size_bytes if m else None}
37
+ harn = {h.id: h.name for h in catalog.BUILTIN}
38
+ for m in images.custom_manifests():
39
+ harn[m["id"]] = m["name"]
40
+ bl = {b["id"]: {"title": b["title"], "category": b["category"], "difficulty": b["difficulty"]} for b in benches.all_benchmarks()}
41
+ return profs, harn, bl
42
+
43
+
44
+ def collect(plan_ids=None, include_sample=True, benchmark_ids=None):
45
+ with db.Session() as s:
46
+ q = s.query(db.Run).filter(db.Run.status == "done")
47
+ if benchmark_ids:
48
+ q = q.filter(db.Run.benchmark_id.in_(benchmark_ids))
49
+ if plan_ids:
50
+ q = q.filter(db.Run.plan_id.in_(plan_ids))
51
+ elif not include_sample:
52
+ sample_ids = [p.id for p in s.query(db.Plan).filter(db.Plan.sample.is_(True))]
53
+ q = q.filter(db.Run.plan_id.notin_(sample_ids))
54
+ runs = [db.row(r) for r in q.all()]
55
+ ids = [r["id"] for r in runs]
56
+ reviews = [db.row(r) for r in s.query(db.Review).filter(db.Review.run_id.in_(ids)).all()] if ids else []
57
+ profs, harn, bl = _labels()
58
+ fam = judge_mod.judge_families()
59
+ flat = []
60
+ for r in runs:
61
+ m = r.get("metrics") or {}
62
+ snap = (r.get("snapshot") or {}).get("profile") or {}
63
+ p = profs.get(r["profile_id"]) or {"name": snap.get("name", r["profile_id"]), "kind": snap.get("kind", "local")}
64
+ d = {"id": r["id"], "plan_id": r["plan_id"], "profile_id": r["profile_id"], "harness_id": r["harness_id"],
65
+ "benchmark_id": r["benchmark_id"], "iteration": r["iteration"], "kind": p.get("kind", "local"),
66
+ "correctness": r["correctness"], "instruction_following": r["instruction_following"], "wall_s": r["wall_s"],
67
+ "cost_usd": r["cost_usd"], "quality_raw": r.get("quality") or {}, "started_at": r["started_at"],
68
+ "family": (r.get("snapshot") or {}).get("family") or fam.get(r["profile_id"])}
69
+ for k in FLAT:
70
+ d[k] = m.get(k)
71
+ st = (d["quality_raw"].get("static") or {})
72
+ d["sloc_total"] = st.get("sloc_total")
73
+ d["lint_per_100"] = st.get("ruff_per_100_sloc")
74
+ d["own_tests"] = st.get("own_test_functions")
75
+ flat.append(d)
76
+ for rv in reviews:
77
+ rv["judge_family"] = fam.get(rv["judge_id"]) or SAMPLE_JUDGES.get(rv["judge_id"])
78
+ return flat, reviews, profs, harn, bl
79
+
80
+
81
+ @router.get("/results")
82
+ def results(plan_ids: str = "", weights: str = "", quality_weights: str = "", pass_threshold: float = 0.8,
83
+ efficiency_ref: str = "local", include_sample: bool = True, benchmark_ids: str = ""):
84
+ ids = [x for x in plan_ids.split(",") if x]
85
+ w = json.loads(weights) if weights else config.get("weights")
86
+ qw = json.loads(quality_weights) if quality_weights else config.get("quality_weights")
87
+ bids = [x for x in benchmark_ids.split(",") if x]
88
+ flat, reviews, profs, harn, bl = collect(ids, include_sample, bids)
89
+ out = scoring.compute(flat, reviews, w, qw, efficiency_ref=efficiency_ref, pass_threshold=pass_threshold)
90
+ for r in out["runs"]:
91
+ r.pop("quality_raw", None)
92
+ r.pop("static_components", None)
93
+ out["agreement"] = scoring.agreement(reviews, flat)
94
+ dims = {}
95
+ for rv in reviews:
96
+ run = next((x for x in flat if x["id"] == rv["run_id"]), None)
97
+ if not run:
98
+ continue
99
+ key = (run["profile_id"], run["harness_id"])
100
+ for k, v in (rv.get("scores") or {}).items():
101
+ if isinstance(v, (int, float)):
102
+ dims.setdefault(key, {}).setdefault(k, []).append(v)
103
+ out["rubric"] = [{"profile_id": k[0], "harness_id": k[1], **{d: round(statistics.mean(v), 2) for d, v in ds.items()}}
104
+ for k, ds in dims.items()]
105
+ out["labels"] = {"profiles": profs, "harnesses": harn, "benchmarks": bl}
106
+ with db.Session() as s:
107
+ out["plans"] = [{"id": p.id, "name": p.name, "sample": p.sample, "status": p.status, "created_at": p.created_at}
108
+ for p in s.query(db.Plan).order_by(db.Plan.created_at.desc()).all()]
109
+ if bids:
110
+ all_flat = collect(ids, include_sample)[0]
111
+ out["benchmarks_available"] = sorted({r["benchmark_id"] for r in all_flat})
112
+ else:
113
+ out["benchmarks_available"] = sorted({r["benchmark_id"] for r in flat})
114
+ out["weights"], out["quality_weights"], out["pass_threshold"] = w, qw, pass_threshold
115
+ out["reviews"] = [{k: rv.get(k) for k in ("run_id", "judge_id", "judge_name", "overall", "summary", "issues", "extras")}
116
+ for rv in reviews]
117
+ return out
118
+
119
+
120
+ @router.get("/results/export")
121
+ def export(plan_ids: str = "", level: str = "runs", fmt: str = "csv", weights: str = "", quality_weights: str = "",
122
+ benchmark_ids: str = ""):
123
+ data = results(plan_ids=plan_ids, weights=weights, quality_weights=quality_weights, benchmark_ids=benchmark_ids)
124
+ profs, harn, bl = data["labels"]["profiles"], data["labels"]["harnesses"], data["labels"]["benchmarks"]
125
+ rows = data["configs"] if level == "configs" else data["cells"] if level == "cells" else data["runs"]
126
+ clean = []
127
+ for r in rows:
128
+ d = {"profile": (profs.get(r["profile_id"]) or {}).get("name", r["profile_id"]),
129
+ "harness": harn.get(r["harness_id"], r["harness_id"])}
130
+ if "benchmark_id" in r:
131
+ d["benchmark"] = (bl.get(r["benchmark_id"]) or {}).get("title", r["benchmark_id"])
132
+ for k, v in r.items():
133
+ if isinstance(v, (dict, list)):
134
+ if k == "points":
135
+ for pk, pv in v.items():
136
+ d[f"points_{pk}"] = pv
137
+ continue
138
+ d[k] = round(v, 5) if isinstance(v, float) else v
139
+ clean.append(d)
140
+ stamp = time.strftime("%Y%m%d-%H%M")
141
+ name = f"llmdyno-{__version__}-{level}-{stamp}" # the file name says which release computed the scores
142
+ if fmt == "json":
143
+ doc = {"llmdyno_version": __version__, "exported_at": time.strftime("%Y-%m-%dT%H:%M:%S"), "level": level,
144
+ "weights": data["weights"], "quality_weights": data["quality_weights"], "rows": clean}
145
+ return Response(json.dumps(doc, indent=2, default=str), media_type="application/json",
146
+ headers={"Content-Disposition": f'attachment; filename="{name}.json"'})
147
+ buf = io.StringIO()
148
+ keys = []
149
+ for r in clean:
150
+ keys += [k for k in r if k not in keys]
151
+ w = csv.DictWriter(buf, fieldnames=keys)
152
+ w.writeheader()
153
+ w.writerows(clean)
154
+ return Response(buf.getvalue(), media_type="text/csv",
155
+ headers={"Content-Disposition": f'attachment; filename="{name}.csv"'})
156
+
157
+
158
+ # ------------------------------------------------------------------ sample dataset
159
+ @router.get("/sample")
160
+ def sample_status():
161
+ with db.Session() as s:
162
+ p = s.query(db.Plan).filter(db.Plan.sample.is_(True)).first()
163
+ return {"available": SAMPLE.exists(), "loaded": bool(p), "plan_id": p.id if p else None}
164
+
165
+
166
+ @router.post("/sample")
167
+ def load_sample():
168
+ if not SAMPLE.exists():
169
+ raise HTTPException(404, "sample dataset is not bundled")
170
+ data = json.loads(SAMPLE.read_text(encoding="utf-8"))
171
+ with db.Session() as s:
172
+ if s.query(db.Plan).filter(db.Plan.sample.is_(True)).first():
173
+ raise HTTPException(409, "the sample is already loaded")
174
+ pid = "plan_sample_round1"
175
+ prof_ids = []
176
+ for p in data["profiles"]:
177
+ if not s.get(db.Profile, p["id"]):
178
+ s.add(db.Profile(id=p["id"], name=p["name"], kind=p["kind"], remote_model=p.get("remote_model", ""),
179
+ params=p.get("params") or {}, notes="Sample profile (round 1 results, RTX 5080)",
180
+ tags=["sample"], pricing=p.get("pricing") or {}))
181
+ prof_ids.append(p["id"])
182
+ s.add(db.Plan(id=pid, name=data["plan"]["name"], profile_ids=prof_ids, harness_ids=["pi"],
183
+ benchmark_ids=data["plan"]["benchmark_ids"], iterations=1, options={}, status="done", sample=True,
184
+ created_at=data["plan"]["created_at"], started_at=data["plan"]["created_at"],
185
+ ended_at=data["plan"]["ended_at"]))
186
+ for r in data["runs"]:
187
+ s.add(db.Run(id=r["id"], plan_id=pid, profile_id=r["profile_id"], harness_id="pi", benchmark_id=r["benchmark_id"],
188
+ iteration=1, status="done", correctness=r["correctness"],
189
+ instruction_following=r["instruction_following"], wall_s=r["wall_s"], cost_usd=r["cost_usd"],
190
+ metrics=r["metrics"], quality=r["quality"], started_at=r["started_at"], ended_at=r["ended_at"],
191
+ snapshot={"profile": {"name": next(p["name"] for p in data["profiles"] if p["id"] == r["profile_id"]),
192
+ "kind": next(p["kind"] for p in data["profiles"] if p["id"] == r["profile_id"])},
193
+ "family": r.get("family"), "sample": True}))
194
+ s.flush() # runs must exist before reviews reference them
195
+ for rv in data["reviews"]:
196
+ s.add(db.Review(id=rv["id"], run_id=rv["run_id"], judge_id=rv["judge_id"], judge_name=rv["judge_name"],
197
+ overall=rv["overall"], scores=rv["scores"], summary=rv["summary"], issues=rv["issues"],
198
+ extras=rv["extras"]))
199
+ s.commit()
200
+ events.publish("plans", "update", {"id": pid})
201
+ return {"ok": True, "plan_id": pid, "runs": len(data["runs"]), "reviews": len(data["reviews"])}
202
+
203
+
204
+ @router.delete("/sample")
205
+ def unload_sample():
206
+ with db.Session() as s:
207
+ plans = s.query(db.Plan).filter(db.Plan.sample.is_(True)).all()
208
+ for p in plans:
209
+ ids = [r.id for r in s.query(db.Run).filter(db.Run.plan_id == p.id)]
210
+ s.query(db.Review).filter(db.Review.run_id.in_(ids)).delete(synchronize_session=False)
211
+ s.query(db.Run).filter(db.Run.plan_id == p.id).delete()
212
+ s.delete(p)
213
+ s.query(db.Profile).filter(db.Profile.id.like("prof_sample_%")).delete(synchronize_session=False)
214
+ s.commit()
215
+ return {"ok": True}
216
+
217
+
218
+ @router.post("/results/weights")
219
+ def save_weights(body: dict = Body(...)):
220
+ patch = {}
221
+ if "weights" in body:
222
+ patch["weights"] = body["weights"]
223
+ if "quality_weights" in body:
224
+ patch["quality_weights"] = body["quality_weights"]
225
+ return config.save_settings(patch)