llmdyno 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llmdyno/__init__.py +50 -0
- llmdyno/__main__.py +54 -0
- llmdyno/api/__init__.py +0 -0
- llmdyno/api/external_api.py +181 -0
- llmdyno/api/results_api.py +225 -0
- llmdyno/api/routes.py +1198 -0
- llmdyno/benchmarks/__init__.py +0 -0
- llmdyno/benchmarks/artificial_analysis.py +175 -0
- llmdyno/benchmarks/builtin/t0_smoke_cli/benchmark.json +46 -0
- llmdyno/benchmarks/builtin/t0_smoke_cli/grader/spec.json +8 -0
- llmdyno/benchmarks/builtin/t0_smoke_cli/grader/tests/test_hidden_wordfreq.py +39 -0
- llmdyno/benchmarks/builtin/t0_smoke_cli/prompts/stage1.md +12 -0
- llmdyno/benchmarks/builtin/t0_smoke_cli/reference/tests/test_wordfreq.py +13 -0
- llmdyno/benchmarks/builtin/t0_smoke_cli/reference/wordfreq.py +34 -0
- llmdyno/benchmarks/builtin/t1_flask_site/benchmark.json +49 -0
- llmdyno/benchmarks/builtin/t1_flask_site/grader/grade.py +24 -0
- llmdyno/benchmarks/builtin/t1_flask_site/grader/tests/test_hidden_flask.py +170 -0
- llmdyno/benchmarks/builtin/t1_flask_site/probes/test_probes.py +75 -0
- llmdyno/benchmarks/builtin/t1_flask_site/prompts/stage1.md +20 -0
- llmdyno/benchmarks/builtin/t1_flask_site/reference/README.md +3 -0
- llmdyno/benchmarks/builtin/t1_flask_site/reference/app.py +99 -0
- llmdyno/benchmarks/builtin/t1_flask_site/reference/requirements.txt +3 -0
- llmdyno/benchmarks/builtin/t1_flask_site/reference/templates/base.html +1 -0
- llmdyno/benchmarks/builtin/t1_flask_site/reference/tests/test_app.py +17 -0
- llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/benchmark.json +49 -0
- llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/grader/grade.py +56 -0
- llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/grader/tests/test_hidden_jobs.py +139 -0
- llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/probes/test_probes.py +89 -0
- llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/prompts/stage1.md +50 -0
- llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/README.md +5 -0
- llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/backend/__init__.py +0 -0
- llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/backend/db.py +18 -0
- llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/backend/main.py +70 -0
- llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/backend/tests/test_api.py +25 -0
- llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/frontend/index.html +8 -0
- llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/frontend/src/api.ts +31 -0
- llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/frontend/src/main.ts +16 -0
- llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/frontend/tsconfig.json +11 -0
- llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/requirements.txt +4 -0
- llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/worker/__init__.py +0 -0
- llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/worker/worker.py +49 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/benchmark.json +69 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/grader/grade.py +44 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/helpers.py +11 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h1.py +57 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h1_jsonformat.py +11 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h2.py +76 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h2_jsonformat.py +14 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h3.py +68 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h4.py +85 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h5.py +102 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/probes/test_probes.py +47 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/prompts/stage1.md +21 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/prompts/stage2.md +15 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/prompts/stage3.md +12 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/prompts/stage4.md +11 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/prompts/stage5.md +19 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/reference/inventory/__init__.py +0 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/reference/inventory/__main__.py +62 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/reference/inventory/errors.py +14 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/reference/inventory/models.py +28 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/reference/inventory/store.py +114 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/reference/tests/test_own.py +24 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/stage_files/stage1/tests/test_step1.py +34 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/stage_files/stage2/tests/test_step2.py +22 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/stage_files/stage3/tests/test_step3.py +22 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/stage_files/stage4/tests/data/legacy.json +12 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/stage_files/stage4/tests/test_step4.py +27 -0
- llmdyno/benchmarks/builtin/t3_long_iterative/stage_files/stage5/tests/test_step5.py +24 -0
- llmdyno/benchmarks/builtin/t4_data_ml/benchmark.json +50 -0
- llmdyno/benchmarks/builtin/t4_data_ml/gen_data.py +203 -0
- llmdyno/benchmarks/builtin/t4_data_ml/grader/grade.py +161 -0
- llmdyno/benchmarks/builtin/t4_data_ml/grader/test_labels.csv +1201 -0
- llmdyno/benchmarks/builtin/t4_data_ml/grader/truth.json +22 -0
- llmdyno/benchmarks/builtin/t4_data_ml/grader/truth_features.parquet +0 -0
- llmdyno/benchmarks/builtin/t4_data_ml/probes/test_probes.py +87 -0
- llmdyno/benchmarks/builtin/t4_data_ml/prompts/stage1.md +43 -0
- llmdyno/benchmarks/builtin/t4_data_ml/reference/pipeline.py +77 -0
- llmdyno/benchmarks/builtin/t4_data_ml/workspace/data/customers.csv +4151 -0
- llmdyno/benchmarks/builtin/t4_data_ml/workspace/data/events.jsonl.gz +0 -0
- llmdyno/benchmarks/builtin/t4_data_ml/workspace/data/labels_train.csv +2801 -0
- llmdyno/benchmarks/builtin/t4_data_ml/workspace/data/test_ids.csv +1201 -0
- llmdyno/benchmarks/builtin/t5_bugfix_semver/benchmark.json +47 -0
- llmdyno/benchmarks/builtin/t5_bugfix_semver/grader/grade.py +31 -0
- llmdyno/benchmarks/builtin/t5_bugfix_semver/grader/tests/test_hidden_semver.py +107 -0
- llmdyno/benchmarks/builtin/t5_bugfix_semver/prompts/stage1.md +10 -0
- llmdyno/benchmarks/builtin/t5_bugfix_semver/reference/semverlite/version.py +87 -0
- llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/CHANGELOG.md +12 -0
- llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/ISSUE.md +17 -0
- llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/README.md +13 -0
- llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/semverlite/__init__.py +5 -0
- llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/semverlite/ranges.py +137 -0
- llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/semverlite/version.py +83 -0
- llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/tests/test_basic.py +60 -0
- llmdyno/benchmarks/common/grading.py +61 -0
- llmdyno/benchmarks/common/polyglot_grade.py +107 -0
- llmdyno/benchmarks/common/spec_grade.py +42 -0
- llmdyno/benchmarks/envs.py +147 -0
- llmdyno/benchmarks/polyglot.py +276 -0
- llmdyno/benchmarks/registry.py +270 -0
- llmdyno/benchmarks/swebench.py +361 -0
- llmdyno/benchmarks/swebench_shim.py +28 -0
- llmdyno/benchmarks/validate.py +87 -0
- llmdyno/config.py +107 -0
- llmdyno/db.py +220 -0
- llmdyno/docker/polyglot-test.sh +18 -0
- llmdyno/docker/polyglot.Dockerfile +44 -0
- llmdyno/docker/quality.Dockerfile +7 -0
- llmdyno/docker/sandbox.Dockerfile +33 -0
- llmdyno/events.py +59 -0
- llmdyno/harnesses/__init__.py +0 -0
- llmdyno/harnesses/catalog.py +424 -0
- llmdyno/harnesses/images.py +143 -0
- llmdyno/harnesses/runtime.py +112 -0
- llmdyno/hub/__init__.py +0 -0
- llmdyno/hub/gguf.py +270 -0
- llmdyno/hub/hf.py +238 -0
- llmdyno/inference/__init__.py +0 -0
- llmdyno/inference/llamacpp.py +450 -0
- llmdyno/jobs.py +153 -0
- llmdyno/judge/__init__.py +0 -0
- llmdyno/judge/judge.py +271 -0
- llmdyno/lib.py +995 -0
- llmdyno/llm.py +229 -0
- llmdyno/memory.py +310 -0
- llmdyno/profiles.py +103 -0
- llmdyno/providers/__init__.py +0 -0
- llmdyno/providers/registry.py +156 -0
- llmdyno/proxy/llm_proxy.py +630 -0
- llmdyno/py.typed +0 -0
- llmdyno/quality/__init__.py +0 -0
- llmdyno/quality/analyze.py +361 -0
- llmdyno/runner/__init__.py +0 -0
- llmdyno/runner/executor.py +770 -0
- llmdyno/runner/metrics.py +188 -0
- llmdyno/runner/sandbox.py +275 -0
- llmdyno/safety.py +120 -0
- llmdyno/sample/round1.json +1 -0
- llmdyno/scoring.py +202 -0
- llmdyno/secrets.py +94 -0
- llmdyno/server.py +148 -0
- llmdyno/smoke.py +107 -0
- llmdyno/system/__init__.py +0 -0
- llmdyno/system/apple.py +156 -0
- llmdyno/system/deps.py +237 -0
- llmdyno/system/specs.py +434 -0
- llmdyno/web/static/assets/index-DDIFnEmp.css +1 -0
- llmdyno/web/static/assets/index-DOnMvXno.js +612 -0
- llmdyno/web/static/assets/inter-cyrillic-ext-wght-normal-B2xhLi22.woff2 +0 -0
- llmdyno/web/static/assets/inter-cyrillic-wght-normal-CMZtQduZ.woff2 +0 -0
- llmdyno/web/static/assets/inter-greek-ext-wght-normal-CGAr0uHJ.woff2 +0 -0
- llmdyno/web/static/assets/inter-greek-wght-normal-CaVNZxsx.woff2 +0 -0
- llmdyno/web/static/assets/inter-latin-ext-wght-normal-CFHvXkgd.woff2 +0 -0
- llmdyno/web/static/assets/inter-latin-wght-normal-C2S99t-D.woff2 +0 -0
- llmdyno/web/static/assets/inter-vietnamese-wght-normal-CBcvBZtf.woff2 +0 -0
- llmdyno/web/static/assets/jetbrains-mono-cyrillic-wght-normal-BptiBwAt.woff2 +0 -0
- llmdyno/web/static/assets/jetbrains-mono-greek-wght-normal-BW1JPxja.woff2 +0 -0
- llmdyno/web/static/assets/jetbrains-mono-latin-ext-wght-normal-Com2F4yK.woff2 +0 -0
- llmdyno/web/static/assets/jetbrains-mono-latin-wght-normal-Ajscav9D.woff2 +0 -0
- llmdyno/web/static/assets/jetbrains-mono-vietnamese-wght-normal-By-yputv.woff2 +0 -0
- llmdyno/web/static/assets/press-start-2p-cyrillic-400-normal-oElOGqNf.woff2 +0 -0
- llmdyno/web/static/assets/press-start-2p-cyrillic-ext-400-normal-COIQET2s.woff2 +0 -0
- llmdyno/web/static/assets/press-start-2p-greek-400-normal-DUIT0YPZ.woff2 +0 -0
- llmdyno/web/static/assets/press-start-2p-latin-400-normal-BKfmdmKK.woff +0 -0
- llmdyno/web/static/assets/press-start-2p-latin-400-normal-xj9lRvVs.woff2 +0 -0
- llmdyno/web/static/assets/press-start-2p-latin-ext-400-normal-B18ub5-k.woff +0 -0
- llmdyno/web/static/assets/press-start-2p-latin-ext-400-normal-DDIa4im3.woff2 +0 -0
- llmdyno/web/static/favicon.svg +161 -0
- llmdyno/web/static/index.html +17 -0
- llmdyno-0.3.1.dist-info/METADATA +421 -0
- llmdyno-0.3.1.dist-info/RECORD +175 -0
- llmdyno-0.3.1.dist-info/WHEEL +4 -0
- llmdyno-0.3.1.dist-info/entry_points.txt +2 -0
- llmdyno-0.3.1.dist-info/licenses/LICENSE +21 -0
- llmdyno-0.3.1.dist-info/licenses/NOTICE.md +33 -0
llmdyno/__init__.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""LLMDyno: benchmark local (llama.cpp) and cloud LLMs as coding agents.
|
|
2
|
+
|
|
3
|
+
Use it as a library:
|
|
4
|
+
|
|
5
|
+
import llmdyno
|
|
6
|
+
|
|
7
|
+
gpt = llmdyno.cloud_model("openai", "gpt-6-luna", reasoning_effort="low")
|
|
8
|
+
qwen = llmdyno.local_model("unsloth/Qwen3.8-27B-GGUF", quant="UD-Q3_K_XL", ctx=49152)
|
|
9
|
+
results = llmdyno.bench([gpt, qwen], benchmarks="suite_quick", harnesses="pi")
|
|
10
|
+
print(results)
|
|
11
|
+
|
|
12
|
+
or as an app: `llmdyno serve --open` (or llmdyno.serve()). Both use the same data folder. See docs/LIBRARY.md.
|
|
13
|
+
|
|
14
|
+
The version follows Semantic Versioning. While it is 0.x, a minor release (0.2.0) may change the API, the data
|
|
15
|
+
folder layout or the database schema (with an automatic migration); patch releases (0.1.1) only fix things.
|
|
16
|
+
"""
|
|
17
|
+
from typing import TYPE_CHECKING
|
|
18
|
+
|
|
19
|
+
__version__ = "0.3.1"
|
|
20
|
+
|
|
21
|
+
# The public API lives in llmdyno.lib and is loaded on first use, so `import llmdyno` stays instant and starts
|
|
22
|
+
# nothing (no database, no Docker, no web server) until you call something. Public names must never equal a
|
|
23
|
+
# submodule's name (profiles, benchmarks, judge, …): Python would set the module over the function, or the reverse.
|
|
24
|
+
_PUBLIC = (
|
|
25
|
+
"LLMDynoError", "NotFoundError", "ConflictError", "PreflightError", "Profile", "Plan", "Results", "Doctor",
|
|
26
|
+
"init", "data_dir", "doctor", "setup", "cloud_model", "local_model", "list_profiles", "profile", "list_benchmarks",
|
|
27
|
+
"list_suites", "import_swebench", "import_polyglot", "preflight", "bench", "list_plans", "plan", "results", "review",
|
|
28
|
+
"serve",
|
|
29
|
+
)
|
|
30
|
+
__all__ = ["__version__", *_PUBLIC]
|
|
31
|
+
|
|
32
|
+
if TYPE_CHECKING: # pragma: no cover - for IDEs and type checkers
|
|
33
|
+
from .lib import ( # noqa: F401
|
|
34
|
+
ConflictError, Doctor, LLMDynoError, NotFoundError, Plan, PreflightError, Profile, Results, bench, cloud_model,
|
|
35
|
+
data_dir, doctor, import_polyglot, import_swebench, init, list_benchmarks, list_plans, list_profiles, list_suites,
|
|
36
|
+
local_model, plan, preflight, profile, results, review, serve, setup,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def __getattr__(name):
|
|
41
|
+
if name in _PUBLIC:
|
|
42
|
+
from . import lib
|
|
43
|
+
value = getattr(lib, name)
|
|
44
|
+
globals()[name] = value
|
|
45
|
+
return value
|
|
46
|
+
raise AttributeError(f"module 'llmdyno' has no attribute {name!r}")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def __dir__():
|
|
50
|
+
return sorted(set(globals()) | set(_PUBLIC))
|
llmdyno/__main__.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Command line: `llmdyno` starts the web app; `llmdyno doctor` prints the setup checks.
|
|
2
|
+
From Python, `import llmdyno` gives the same features as a library (see docs/LIBRARY.md)."""
|
|
3
|
+
import argparse
|
|
4
|
+
|
|
5
|
+
from . import __version__
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
import threading
|
|
9
|
+
import webbrowser
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def main(argv=None):
|
|
13
|
+
ap = argparse.ArgumentParser(prog="llmdyno", description="Benchmark LLMs as coding agents.")
|
|
14
|
+
ap.add_argument("--version", action="version", version=f"llmdyno {__version__}")
|
|
15
|
+
sub = ap.add_subparsers(dest="cmd")
|
|
16
|
+
sv = sub.add_parser("serve", help="start the web app (default)")
|
|
17
|
+
sv.add_argument("--host", default="127.0.0.1")
|
|
18
|
+
sv.add_argument("--port", type=int, default=None)
|
|
19
|
+
sv.add_argument("--open", action="store_true", help="open the browser")
|
|
20
|
+
sub.add_parser("doctor", help="check system specs and dependencies")
|
|
21
|
+
sub.add_parser("specs", help="print detected hardware as JSON")
|
|
22
|
+
a = ap.parse_args(argv)
|
|
23
|
+
from . import config, db
|
|
24
|
+
db.init()
|
|
25
|
+
if a.cmd == "doctor":
|
|
26
|
+
from .system import deps, specs
|
|
27
|
+
s = specs.detect(force=True)
|
|
28
|
+
if not s["os"]["arch_supported"]:
|
|
29
|
+
print(f"Unsupported architecture {s['os']['arch']}: LLMDyno supports x86-64 and ARM64.")
|
|
30
|
+
return 2
|
|
31
|
+
for n in s["summary"]["notes"]:
|
|
32
|
+
print(f"[{n['level']}] {n['text']}")
|
|
33
|
+
print()
|
|
34
|
+
for d in deps.check(force=True):
|
|
35
|
+
mark = "OK " if d["ok"] else ("!! " if d["level"] == "required" else "-- ")
|
|
36
|
+
print(f"{mark}{d['name']:<22} {d['level']:<26} {d['detail']}")
|
|
37
|
+
return 0
|
|
38
|
+
if a.cmd == "specs":
|
|
39
|
+
from .system import specs
|
|
40
|
+
print(json.dumps(specs.detect(force=True), indent=2, default=str))
|
|
41
|
+
return 0
|
|
42
|
+
import uvicorn
|
|
43
|
+
host = getattr(a, "host", "127.0.0.1")
|
|
44
|
+
port = getattr(a, "port", None) or int(config.get("port"))
|
|
45
|
+
url = f"http://{host}:{port}"
|
|
46
|
+
print(f"LLMDyno on {url} (data: {config.DATA})", flush=True)
|
|
47
|
+
if getattr(a, "open", False):
|
|
48
|
+
threading.Timer(1.5, lambda: webbrowser.open(url)).start()
|
|
49
|
+
uvicorn.run("llmdyno.server:app", host=host, port=port, log_level="warning")
|
|
50
|
+
return 0
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
if __name__ == "__main__":
|
|
54
|
+
sys.exit(main())
|
llmdyno/api/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""External benchmarks and references: SWE-bench and Aider polyglot imports, the official SWE-bench harness, SWE-bench
|
|
2
|
+
predictions export, portable harness runtimes and Artificial Analysis reference numbers."""
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
|
|
6
|
+
from fastapi import APIRouter, Body, HTTPException
|
|
7
|
+
from fastapi.responses import Response
|
|
8
|
+
|
|
9
|
+
from .. import config, db, jobs, secrets
|
|
10
|
+
from ..benchmarks import artificial_analysis as aa
|
|
11
|
+
from ..benchmarks import polyglot, swebench
|
|
12
|
+
from ..benchmarks import registry as benches
|
|
13
|
+
from ..harnesses import catalog, images, runtime
|
|
14
|
+
|
|
15
|
+
router = APIRouter()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# ------------------------------------------------------------------ catalog of importable benchmarks
|
|
19
|
+
def _count(prefix):
|
|
20
|
+
return sum(1 for b in benches.all_benchmarks() if b["id"].startswith(prefix) and not b["builtin"])
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@router.get("/imports")
|
|
24
|
+
def import_catalog():
|
|
25
|
+
sw = swebench.harness_status()
|
|
26
|
+
poly = images.image_exists(polyglot.IMAGE)
|
|
27
|
+
return {
|
|
28
|
+
"swebench": {"datasets": [{"id": k, "name": v["name"], "size": v["size"], "about": v["about"], "hf": v["hf"]}
|
|
29
|
+
for k, v in swebench.DATASETS.items()],
|
|
30
|
+
"imported": _count("swe_"), "harness": sw, "homepage": swebench.HOMEPAGE,
|
|
31
|
+
"harness_version": (config.get("swebench") or {}).get("harness_version")},
|
|
32
|
+
"polyglot": {"languages": polyglot.LANGS, "imported": _count("pg_"), "image": polyglot.IMAGE,
|
|
33
|
+
"image_info": poly, "context_ready": polyglot.context_ready(), "homepage": polyglot.HOMEPAGE,
|
|
34
|
+
"repo": f"https://github.com/{polyglot.REPO}"},
|
|
35
|
+
"runtime_platform": runtime.host_platform(),
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@router.post("/imports/swebench")
|
|
40
|
+
def import_swebench(body: dict = Body(...)):
|
|
41
|
+
key = body.get("dataset", "verified_mini")
|
|
42
|
+
if key not in swebench.DATASETS:
|
|
43
|
+
raise HTTPException(400, f"unknown dataset {key}")
|
|
44
|
+
ids = [x for x in body.get("instance_ids") or [] if re.fullmatch(r"[\w.\-]+__[\w.\-]+-\d+", x)]
|
|
45
|
+
limit = int(body.get("limit") or 0)
|
|
46
|
+
if limit < 0 or limit > 2000:
|
|
47
|
+
raise HTTPException(400, "limit must be 0-2000")
|
|
48
|
+
name = swebench.DATASETS[key]["name"]
|
|
49
|
+
return {"job_id": jobs.submit("import", f"Import {name}" + (f" (first {limit})" if limit else ""),
|
|
50
|
+
swebench.import_job, {"dataset": key, "instance_ids": ids, "limit": limit},
|
|
51
|
+
ref="swebench", group="import")}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@router.post("/imports/polyglot")
|
|
55
|
+
def import_polyglot(body: dict = Body(default={})):
|
|
56
|
+
langs = [x for x in body.get("languages") or list(polyglot.LANGS) if x in polyglot.LANGS]
|
|
57
|
+
if not langs:
|
|
58
|
+
raise HTTPException(400, "choose at least one language")
|
|
59
|
+
return {"job_id": jobs.submit("import", "Import Aider polyglot", polyglot.import_job, {"languages": langs},
|
|
60
|
+
ref="polyglot", group="import")}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@router.post("/imports/swebench/harness")
|
|
64
|
+
def install_swebench_harness():
|
|
65
|
+
v = (config.get("swebench") or {}).get("harness_version")
|
|
66
|
+
return {"job_id": jobs.submit("install", f"Install SWE-bench harness {v}", swebench.install_harness, {"version": v},
|
|
67
|
+
ref="swebench_harness")}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@router.post("/imports/polyglot/image")
|
|
71
|
+
def build_polyglot_image():
|
|
72
|
+
if not polyglot.context_ready():
|
|
73
|
+
raise HTTPException(409, "import the polyglot benchmark first")
|
|
74
|
+
return {"job_id": jobs.submit("build", "Build polyglot toolchain image", polyglot.build_image, ref="polyglot_image")}
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@router.delete("/imports/{kind}")
|
|
78
|
+
def remove_imported(kind: str):
|
|
79
|
+
"""Delete every imported benchmark of one kind (the dataset can be imported again at any time)."""
|
|
80
|
+
prefix = {"swebench": "swe_", "polyglot": "pg_"}.get(kind)
|
|
81
|
+
if not prefix:
|
|
82
|
+
raise HTTPException(404, "unknown import")
|
|
83
|
+
n = 0
|
|
84
|
+
for b in benches.all_benchmarks():
|
|
85
|
+
if b["id"].startswith(prefix) and not b["builtin"]:
|
|
86
|
+
benches.delete_custom(b["id"])
|
|
87
|
+
n += 1
|
|
88
|
+
with db.Session() as s:
|
|
89
|
+
s.query(db.Suite).filter(db.Suite.id.like(f"suite_{kind}%")).delete(synchronize_session=False)
|
|
90
|
+
s.commit()
|
|
91
|
+
return {"removed": n}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# ------------------------------------------------------------------ SWE-bench predictions
|
|
95
|
+
@router.get("/plans/{pid}/swebench-predictions")
|
|
96
|
+
def swebench_predictions(pid: str):
|
|
97
|
+
with db.Session() as s:
|
|
98
|
+
runs = [db.row(r) for r in s.query(db.Run).filter(db.Run.plan_id == pid, db.Run.status == "done").all()]
|
|
99
|
+
profs = {p.id: p.name for p in s.query(db.Profile).all()}
|
|
100
|
+
runs = [r for r in runs if r["benchmark_id"].startswith("swe_")]
|
|
101
|
+
if not runs:
|
|
102
|
+
raise HTTPException(404, "this plan has no finished SWE-bench runs")
|
|
103
|
+
for r in runs:
|
|
104
|
+
r["label"] = re.sub(r"[^A-Za-z0-9_.-]+", "-", f"{profs.get(r['profile_id'], r['profile_id'])}__{r['harness_id']}")
|
|
105
|
+
lines = swebench.predictions(runs)
|
|
106
|
+
body = "".join(json.dumps(x) + "\n" for x in lines)
|
|
107
|
+
return Response(body, media_type="application/x-ndjson",
|
|
108
|
+
headers={"Content-Disposition": f'attachment; filename="{pid}-swebench-predictions.jsonl"'})
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
# ------------------------------------------------------------------ harness runtimes
|
|
112
|
+
@router.get("/harnesses/{hid}/runtime")
|
|
113
|
+
def harness_runtime(hid: str, platform: str = ""):
|
|
114
|
+
h = catalog.get(hid, images.custom_manifests())
|
|
115
|
+
pf = platform if platform in runtime.PLATFORM_ARCH else runtime.host_platform()
|
|
116
|
+
return {**runtime.status(h, pf), "dockerfile": runtime.dockerfile(h)}
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@router.post("/harnesses/{hid}/runtime")
|
|
120
|
+
def build_harness_runtime(hid: str, body: dict = Body(default={})):
|
|
121
|
+
h = catalog.get(hid, images.custom_manifests())
|
|
122
|
+
pf = body.get("platform") if body.get("platform") in runtime.PLATFORM_ARCH else runtime.host_platform()
|
|
123
|
+
|
|
124
|
+
def job(ctx):
|
|
125
|
+
ctx.result = runtime.ensure(ctx, h, pf, ctx.log)
|
|
126
|
+
return {"job_id": jobs.submit("build", f"Build {h.name} runtime ({pf})", job, {"harness_id": hid, "platform": pf},
|
|
127
|
+
ref=f"rt_{hid}")}
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
# ------------------------------------------------------------------ Artificial Analysis
|
|
131
|
+
@router.get("/references/aa")
|
|
132
|
+
def aa_reference():
|
|
133
|
+
from .results_api import _labels
|
|
134
|
+
profs = _labels()[0]
|
|
135
|
+
ref = aa.reference(profs)
|
|
136
|
+
return {**aa.status(), "matches": {pid: {"score": m["score"], "manual": m["manual"], "model": m["model"]}
|
|
137
|
+
for pid, m in ref.items()},
|
|
138
|
+
"model_list": [{k: m.get(k) for k in ("slug", "name", "creator")} for m in aa.load_cache().get("models") or []]}
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@router.post("/references/aa/refresh")
|
|
142
|
+
def aa_refresh(body: dict = Body(default={})):
|
|
143
|
+
try:
|
|
144
|
+
c = aa.refresh(force=bool(body.get("force")))
|
|
145
|
+
except PermissionError as e:
|
|
146
|
+
raise HTTPException(400, str(e)) from e
|
|
147
|
+
except RuntimeError as e:
|
|
148
|
+
raise HTTPException(429, str(e)) from e
|
|
149
|
+
except Exception as e: # noqa: BLE001 - network and API errors go back to the user as text
|
|
150
|
+
raise HTTPException(502, f"Artificial Analysis request failed: {e}") from e
|
|
151
|
+
return {"models": len(c.get("models") or []), "fetched_at": c.get("fetched_at")}
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
@router.put("/settings/aa-key")
|
|
155
|
+
def set_aa_key(body: dict = Body(...)):
|
|
156
|
+
key = (body.get("key") or "").strip()
|
|
157
|
+
if not key:
|
|
158
|
+
raise HTTPException(400, "the key is empty")
|
|
159
|
+
secrets.set(aa.SECRET, key)
|
|
160
|
+
return aa.status()
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
@router.delete("/settings/aa-key")
|
|
164
|
+
def delete_aa_key():
|
|
165
|
+
secrets.delete(aa.SECRET)
|
|
166
|
+
return aa.status()
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@router.put("/references/aa/override")
|
|
170
|
+
def aa_override(body: dict = Body(...)):
|
|
171
|
+
pid = body.get("profile_id")
|
|
172
|
+
if not pid:
|
|
173
|
+
raise HTTPException(400, "profile_id is required")
|
|
174
|
+
ov = dict(aa.overrides())
|
|
175
|
+
if body.get("slug"):
|
|
176
|
+
ov[pid] = str(body["slug"])[:120]
|
|
177
|
+
else:
|
|
178
|
+
ov.pop(pid, None)
|
|
179
|
+
cur = config.get("aa_reference") or {}
|
|
180
|
+
config.save_settings({"aa_reference": {**cur, "overrides": ov}})
|
|
181
|
+
return {"overrides": ov}
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
"""Results: scored runs and configurations for the dashboard, judge agreement, exports and the sample dataset."""
|
|
2
|
+
import csv
|
|
3
|
+
import io
|
|
4
|
+
import json
|
|
5
|
+
import statistics
|
|
6
|
+
import time
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from fastapi import APIRouter, Body, HTTPException
|
|
10
|
+
from fastapi.responses import Response
|
|
11
|
+
|
|
12
|
+
from .. import __version__, config, db, events, scoring
|
|
13
|
+
from ..benchmarks import registry as benches
|
|
14
|
+
from ..harnesses import catalog, images
|
|
15
|
+
from ..judge import judge as judge_mod
|
|
16
|
+
|
|
17
|
+
router = APIRouter()
|
|
18
|
+
SAMPLE = Path(__file__).resolve().parent.parent / "sample" / "round1.json"
|
|
19
|
+
SAMPLE_JUDGES = {"judge_sample_luna": "openai", "judge_sample_claude": "anthropic"}
|
|
20
|
+
FLAT = ("total_prompt_tokens", "prefill_tokens", "cached_prompt_tokens", "generated_tokens", "reasoning_tokens",
|
|
21
|
+
"cache_hit_ratio", "peak_context_tokens", "context_utilization", "decode_tps", "prefill_tps", "ttft_median_s",
|
|
22
|
+
"model_time_s", "requests", "turns", "n_tool_calls", "tool_errors", "gpu_energy_wh", "gpu_peak_vram_mib",
|
|
23
|
+
"gpu_vram_total_mib", "spill_detected", "gpu_spill_mib", "tests_passed", "tests_total", "timed_out",
|
|
24
|
+
"length_truncations", "http_errors", "budget_blocked")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _labels():
|
|
28
|
+
with db.Session() as s:
|
|
29
|
+
profs = {}
|
|
30
|
+
for p in s.query(db.Profile).all():
|
|
31
|
+
m = s.get(db.Model, p.model_id) if p.model_id else None
|
|
32
|
+
v = s.get(db.Provider, p.provider_id) if p.provider_id else None
|
|
33
|
+
profs[p.id] = {"name": p.name, "kind": p.kind, "quant": m.quant if m else "", "model": m.name if m else p.remote_model,
|
|
34
|
+
"vendor": v.vendor if v else None, "provider": v.name if v else None,
|
|
35
|
+
"arch": (m.meta or {}).get("arch") if m else None, "ctx": (p.params or {}).get("ctx"),
|
|
36
|
+
"size_bytes": m.size_bytes if m else None}
|
|
37
|
+
harn = {h.id: h.name for h in catalog.BUILTIN}
|
|
38
|
+
for m in images.custom_manifests():
|
|
39
|
+
harn[m["id"]] = m["name"]
|
|
40
|
+
bl = {b["id"]: {"title": b["title"], "category": b["category"], "difficulty": b["difficulty"]} for b in benches.all_benchmarks()}
|
|
41
|
+
return profs, harn, bl
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def collect(plan_ids=None, include_sample=True, benchmark_ids=None):
|
|
45
|
+
with db.Session() as s:
|
|
46
|
+
q = s.query(db.Run).filter(db.Run.status == "done")
|
|
47
|
+
if benchmark_ids:
|
|
48
|
+
q = q.filter(db.Run.benchmark_id.in_(benchmark_ids))
|
|
49
|
+
if plan_ids:
|
|
50
|
+
q = q.filter(db.Run.plan_id.in_(plan_ids))
|
|
51
|
+
elif not include_sample:
|
|
52
|
+
sample_ids = [p.id for p in s.query(db.Plan).filter(db.Plan.sample.is_(True))]
|
|
53
|
+
q = q.filter(db.Run.plan_id.notin_(sample_ids))
|
|
54
|
+
runs = [db.row(r) for r in q.all()]
|
|
55
|
+
ids = [r["id"] for r in runs]
|
|
56
|
+
reviews = [db.row(r) for r in s.query(db.Review).filter(db.Review.run_id.in_(ids)).all()] if ids else []
|
|
57
|
+
profs, harn, bl = _labels()
|
|
58
|
+
fam = judge_mod.judge_families()
|
|
59
|
+
flat = []
|
|
60
|
+
for r in runs:
|
|
61
|
+
m = r.get("metrics") or {}
|
|
62
|
+
snap = (r.get("snapshot") or {}).get("profile") or {}
|
|
63
|
+
p = profs.get(r["profile_id"]) or {"name": snap.get("name", r["profile_id"]), "kind": snap.get("kind", "local")}
|
|
64
|
+
d = {"id": r["id"], "plan_id": r["plan_id"], "profile_id": r["profile_id"], "harness_id": r["harness_id"],
|
|
65
|
+
"benchmark_id": r["benchmark_id"], "iteration": r["iteration"], "kind": p.get("kind", "local"),
|
|
66
|
+
"correctness": r["correctness"], "instruction_following": r["instruction_following"], "wall_s": r["wall_s"],
|
|
67
|
+
"cost_usd": r["cost_usd"], "quality_raw": r.get("quality") or {}, "started_at": r["started_at"],
|
|
68
|
+
"family": (r.get("snapshot") or {}).get("family") or fam.get(r["profile_id"])}
|
|
69
|
+
for k in FLAT:
|
|
70
|
+
d[k] = m.get(k)
|
|
71
|
+
st = (d["quality_raw"].get("static") or {})
|
|
72
|
+
d["sloc_total"] = st.get("sloc_total")
|
|
73
|
+
d["lint_per_100"] = st.get("ruff_per_100_sloc")
|
|
74
|
+
d["own_tests"] = st.get("own_test_functions")
|
|
75
|
+
flat.append(d)
|
|
76
|
+
for rv in reviews:
|
|
77
|
+
rv["judge_family"] = fam.get(rv["judge_id"]) or SAMPLE_JUDGES.get(rv["judge_id"])
|
|
78
|
+
return flat, reviews, profs, harn, bl
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@router.get("/results")
|
|
82
|
+
def results(plan_ids: str = "", weights: str = "", quality_weights: str = "", pass_threshold: float = 0.8,
|
|
83
|
+
efficiency_ref: str = "local", include_sample: bool = True, benchmark_ids: str = ""):
|
|
84
|
+
ids = [x for x in plan_ids.split(",") if x]
|
|
85
|
+
w = json.loads(weights) if weights else config.get("weights")
|
|
86
|
+
qw = json.loads(quality_weights) if quality_weights else config.get("quality_weights")
|
|
87
|
+
bids = [x for x in benchmark_ids.split(",") if x]
|
|
88
|
+
flat, reviews, profs, harn, bl = collect(ids, include_sample, bids)
|
|
89
|
+
out = scoring.compute(flat, reviews, w, qw, efficiency_ref=efficiency_ref, pass_threshold=pass_threshold)
|
|
90
|
+
for r in out["runs"]:
|
|
91
|
+
r.pop("quality_raw", None)
|
|
92
|
+
r.pop("static_components", None)
|
|
93
|
+
out["agreement"] = scoring.agreement(reviews, flat)
|
|
94
|
+
dims = {}
|
|
95
|
+
for rv in reviews:
|
|
96
|
+
run = next((x for x in flat if x["id"] == rv["run_id"]), None)
|
|
97
|
+
if not run:
|
|
98
|
+
continue
|
|
99
|
+
key = (run["profile_id"], run["harness_id"])
|
|
100
|
+
for k, v in (rv.get("scores") or {}).items():
|
|
101
|
+
if isinstance(v, (int, float)):
|
|
102
|
+
dims.setdefault(key, {}).setdefault(k, []).append(v)
|
|
103
|
+
out["rubric"] = [{"profile_id": k[0], "harness_id": k[1], **{d: round(statistics.mean(v), 2) for d, v in ds.items()}}
|
|
104
|
+
for k, ds in dims.items()]
|
|
105
|
+
out["labels"] = {"profiles": profs, "harnesses": harn, "benchmarks": bl}
|
|
106
|
+
with db.Session() as s:
|
|
107
|
+
out["plans"] = [{"id": p.id, "name": p.name, "sample": p.sample, "status": p.status, "created_at": p.created_at}
|
|
108
|
+
for p in s.query(db.Plan).order_by(db.Plan.created_at.desc()).all()]
|
|
109
|
+
if bids:
|
|
110
|
+
all_flat = collect(ids, include_sample)[0]
|
|
111
|
+
out["benchmarks_available"] = sorted({r["benchmark_id"] for r in all_flat})
|
|
112
|
+
else:
|
|
113
|
+
out["benchmarks_available"] = sorted({r["benchmark_id"] for r in flat})
|
|
114
|
+
out["weights"], out["quality_weights"], out["pass_threshold"] = w, qw, pass_threshold
|
|
115
|
+
out["reviews"] = [{k: rv.get(k) for k in ("run_id", "judge_id", "judge_name", "overall", "summary", "issues", "extras")}
|
|
116
|
+
for rv in reviews]
|
|
117
|
+
return out
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@router.get("/results/export")
|
|
121
|
+
def export(plan_ids: str = "", level: str = "runs", fmt: str = "csv", weights: str = "", quality_weights: str = "",
|
|
122
|
+
benchmark_ids: str = ""):
|
|
123
|
+
data = results(plan_ids=plan_ids, weights=weights, quality_weights=quality_weights, benchmark_ids=benchmark_ids)
|
|
124
|
+
profs, harn, bl = data["labels"]["profiles"], data["labels"]["harnesses"], data["labels"]["benchmarks"]
|
|
125
|
+
rows = data["configs"] if level == "configs" else data["cells"] if level == "cells" else data["runs"]
|
|
126
|
+
clean = []
|
|
127
|
+
for r in rows:
|
|
128
|
+
d = {"profile": (profs.get(r["profile_id"]) or {}).get("name", r["profile_id"]),
|
|
129
|
+
"harness": harn.get(r["harness_id"], r["harness_id"])}
|
|
130
|
+
if "benchmark_id" in r:
|
|
131
|
+
d["benchmark"] = (bl.get(r["benchmark_id"]) or {}).get("title", r["benchmark_id"])
|
|
132
|
+
for k, v in r.items():
|
|
133
|
+
if isinstance(v, (dict, list)):
|
|
134
|
+
if k == "points":
|
|
135
|
+
for pk, pv in v.items():
|
|
136
|
+
d[f"points_{pk}"] = pv
|
|
137
|
+
continue
|
|
138
|
+
d[k] = round(v, 5) if isinstance(v, float) else v
|
|
139
|
+
clean.append(d)
|
|
140
|
+
stamp = time.strftime("%Y%m%d-%H%M")
|
|
141
|
+
name = f"llmdyno-{__version__}-{level}-{stamp}" # the file name says which release computed the scores
|
|
142
|
+
if fmt == "json":
|
|
143
|
+
doc = {"llmdyno_version": __version__, "exported_at": time.strftime("%Y-%m-%dT%H:%M:%S"), "level": level,
|
|
144
|
+
"weights": data["weights"], "quality_weights": data["quality_weights"], "rows": clean}
|
|
145
|
+
return Response(json.dumps(doc, indent=2, default=str), media_type="application/json",
|
|
146
|
+
headers={"Content-Disposition": f'attachment; filename="{name}.json"'})
|
|
147
|
+
buf = io.StringIO()
|
|
148
|
+
keys = []
|
|
149
|
+
for r in clean:
|
|
150
|
+
keys += [k for k in r if k not in keys]
|
|
151
|
+
w = csv.DictWriter(buf, fieldnames=keys)
|
|
152
|
+
w.writeheader()
|
|
153
|
+
w.writerows(clean)
|
|
154
|
+
return Response(buf.getvalue(), media_type="text/csv",
|
|
155
|
+
headers={"Content-Disposition": f'attachment; filename="{name}.csv"'})
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
# ------------------------------------------------------------------ sample dataset
|
|
159
|
+
@router.get("/sample")
|
|
160
|
+
def sample_status():
|
|
161
|
+
with db.Session() as s:
|
|
162
|
+
p = s.query(db.Plan).filter(db.Plan.sample.is_(True)).first()
|
|
163
|
+
return {"available": SAMPLE.exists(), "loaded": bool(p), "plan_id": p.id if p else None}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
@router.post("/sample")
|
|
167
|
+
def load_sample():
|
|
168
|
+
if not SAMPLE.exists():
|
|
169
|
+
raise HTTPException(404, "sample dataset is not bundled")
|
|
170
|
+
data = json.loads(SAMPLE.read_text(encoding="utf-8"))
|
|
171
|
+
with db.Session() as s:
|
|
172
|
+
if s.query(db.Plan).filter(db.Plan.sample.is_(True)).first():
|
|
173
|
+
raise HTTPException(409, "the sample is already loaded")
|
|
174
|
+
pid = "plan_sample_round1"
|
|
175
|
+
prof_ids = []
|
|
176
|
+
for p in data["profiles"]:
|
|
177
|
+
if not s.get(db.Profile, p["id"]):
|
|
178
|
+
s.add(db.Profile(id=p["id"], name=p["name"], kind=p["kind"], remote_model=p.get("remote_model", ""),
|
|
179
|
+
params=p.get("params") or {}, notes="Sample profile (round 1 results, RTX 5080)",
|
|
180
|
+
tags=["sample"], pricing=p.get("pricing") or {}))
|
|
181
|
+
prof_ids.append(p["id"])
|
|
182
|
+
s.add(db.Plan(id=pid, name=data["plan"]["name"], profile_ids=prof_ids, harness_ids=["pi"],
|
|
183
|
+
benchmark_ids=data["plan"]["benchmark_ids"], iterations=1, options={}, status="done", sample=True,
|
|
184
|
+
created_at=data["plan"]["created_at"], started_at=data["plan"]["created_at"],
|
|
185
|
+
ended_at=data["plan"]["ended_at"]))
|
|
186
|
+
for r in data["runs"]:
|
|
187
|
+
s.add(db.Run(id=r["id"], plan_id=pid, profile_id=r["profile_id"], harness_id="pi", benchmark_id=r["benchmark_id"],
|
|
188
|
+
iteration=1, status="done", correctness=r["correctness"],
|
|
189
|
+
instruction_following=r["instruction_following"], wall_s=r["wall_s"], cost_usd=r["cost_usd"],
|
|
190
|
+
metrics=r["metrics"], quality=r["quality"], started_at=r["started_at"], ended_at=r["ended_at"],
|
|
191
|
+
snapshot={"profile": {"name": next(p["name"] for p in data["profiles"] if p["id"] == r["profile_id"]),
|
|
192
|
+
"kind": next(p["kind"] for p in data["profiles"] if p["id"] == r["profile_id"])},
|
|
193
|
+
"family": r.get("family"), "sample": True}))
|
|
194
|
+
s.flush() # runs must exist before reviews reference them
|
|
195
|
+
for rv in data["reviews"]:
|
|
196
|
+
s.add(db.Review(id=rv["id"], run_id=rv["run_id"], judge_id=rv["judge_id"], judge_name=rv["judge_name"],
|
|
197
|
+
overall=rv["overall"], scores=rv["scores"], summary=rv["summary"], issues=rv["issues"],
|
|
198
|
+
extras=rv["extras"]))
|
|
199
|
+
s.commit()
|
|
200
|
+
events.publish("plans", "update", {"id": pid})
|
|
201
|
+
return {"ok": True, "plan_id": pid, "runs": len(data["runs"]), "reviews": len(data["reviews"])}
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
@router.delete("/sample")
|
|
205
|
+
def unload_sample():
|
|
206
|
+
with db.Session() as s:
|
|
207
|
+
plans = s.query(db.Plan).filter(db.Plan.sample.is_(True)).all()
|
|
208
|
+
for p in plans:
|
|
209
|
+
ids = [r.id for r in s.query(db.Run).filter(db.Run.plan_id == p.id)]
|
|
210
|
+
s.query(db.Review).filter(db.Review.run_id.in_(ids)).delete(synchronize_session=False)
|
|
211
|
+
s.query(db.Run).filter(db.Run.plan_id == p.id).delete()
|
|
212
|
+
s.delete(p)
|
|
213
|
+
s.query(db.Profile).filter(db.Profile.id.like("prof_sample_%")).delete(synchronize_session=False)
|
|
214
|
+
s.commit()
|
|
215
|
+
return {"ok": True}
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
@router.post("/results/weights")
|
|
219
|
+
def save_weights(body: dict = Body(...)):
|
|
220
|
+
patch = {}
|
|
221
|
+
if "weights" in body:
|
|
222
|
+
patch["weights"] = body["weights"]
|
|
223
|
+
if "quality_weights" in body:
|
|
224
|
+
patch["quality_weights"] = body["quality_weights"]
|
|
225
|
+
return config.save_settings(patch)
|