llmdyno 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llmdyno-0.3.1/.gitattributes +7 -0
- llmdyno-0.3.1/.github/ISSUE_TEMPLATE/bug_report.yml +39 -0
- llmdyno-0.3.1/.github/ISSUE_TEMPLATE/config.yml +5 -0
- llmdyno-0.3.1/.github/ISSUE_TEMPLATE/feature_request.yml +20 -0
- llmdyno-0.3.1/.github/ISSUE_TEMPLATE/new_harness_or_benchmark.yml +28 -0
- llmdyno-0.3.1/.github/dependabot.yml +24 -0
- llmdyno-0.3.1/.github/pull_request_template.md +14 -0
- llmdyno-0.3.1/.github/workflows/ci.yml +70 -0
- llmdyno-0.3.1/.github/workflows/release.yml +66 -0
- llmdyno-0.3.1/.gitignore +23 -0
- llmdyno-0.3.1/CHANGELOG.md +132 -0
- llmdyno-0.3.1/CITATION.cff +25 -0
- llmdyno-0.3.1/CODE_OF_CONDUCT.md +9 -0
- llmdyno-0.3.1/CONTRIBUTING.md +59 -0
- llmdyno-0.3.1/LICENSE +21 -0
- llmdyno-0.3.1/NOTICE.md +33 -0
- llmdyno-0.3.1/PKG-INFO +421 -0
- llmdyno-0.3.1/README.md +378 -0
- llmdyno-0.3.1/SECURITY.md +21 -0
- llmdyno-0.3.1/docs/ARCHITECTURE.md +168 -0
- llmdyno-0.3.1/docs/BENCHMARKS.md +115 -0
- llmdyno-0.3.1/docs/HARNESSES.md +91 -0
- llmdyno-0.3.1/docs/LIBRARY.md +223 -0
- llmdyno-0.3.1/docs/RELEASING.md +49 -0
- llmdyno-0.3.1/docs/images/banner.svg +170 -0
- llmdyno-0.3.1/docs/images/dyno-judging.svg +203 -0
- llmdyno-0.3.1/docs/images/dyno-running.svg +275 -0
- llmdyno-0.3.1/docs/images/judging.png +0 -0
- llmdyno-0.3.1/docs/images/logo.svg +161 -0
- llmdyno-0.3.1/docs/images/mascot.png +0 -0
- llmdyno-0.3.1/docs/images/overview.png +0 -0
- llmdyno-0.3.1/docs/images/panic-stop.png +0 -0
- llmdyno-0.3.1/docs/images/plan-wizard.png +0 -0
- llmdyno-0.3.1/docs/images/profile-editor.png +0 -0
- llmdyno-0.3.1/docs/images/results.png +0 -0
- llmdyno-0.3.1/docs/images/run-transcript.png +0 -0
- llmdyno-0.3.1/docs/images/setup.png +0 -0
- llmdyno-0.3.1/docs/images/social-preview.png +0 -0
- llmdyno-0.3.1/examples/local_settings_sweep.py +23 -0
- llmdyno-0.3.1/examples/quickstart.py +20 -0
- llmdyno-0.3.1/examples/swebench_mini.py +21 -0
- llmdyno-0.3.1/frontend/index.html +16 -0
- llmdyno-0.3.1/frontend/package-lock.json +1934 -0
- llmdyno-0.3.1/frontend/package.json +35 -0
- llmdyno-0.3.1/frontend/public/favicon.svg +161 -0
- llmdyno-0.3.1/frontend/src/App.tsx +184 -0
- llmdyno-0.3.1/frontend/src/api.ts +59 -0
- llmdyno-0.3.1/frontend/src/charts.tsx +81 -0
- llmdyno-0.3.1/frontend/src/components.tsx +396 -0
- llmdyno-0.3.1/frontend/src/dyno-sprite.ts +34 -0
- llmdyno-0.3.1/frontend/src/dyno.tsx +147 -0
- llmdyno-0.3.1/frontend/src/format.ts +56 -0
- llmdyno-0.3.1/frontend/src/logo-pixels.ts +175 -0
- llmdyno-0.3.1/frontend/src/logos.tsx +126 -0
- llmdyno-0.3.1/frontend/src/main.tsx +28 -0
- llmdyno-0.3.1/frontend/src/pages/Benchmarks.tsx +493 -0
- llmdyno-0.3.1/frontend/src/pages/Harnesses.tsx +224 -0
- llmdyno-0.3.1/frontend/src/pages/Home.tsx +151 -0
- llmdyno-0.3.1/frontend/src/pages/ImportBenchmarks.tsx +155 -0
- llmdyno-0.3.1/frontend/src/pages/Jobs.tsx +44 -0
- llmdyno-0.3.1/frontend/src/pages/Judging.tsx +206 -0
- llmdyno-0.3.1/frontend/src/pages/Models.tsx +368 -0
- llmdyno-0.3.1/frontend/src/pages/Onboarding.tsx +162 -0
- llmdyno-0.3.1/frontend/src/pages/PlanDetail.tsx +190 -0
- llmdyno-0.3.1/frontend/src/pages/PlanNew.tsx +260 -0
- llmdyno-0.3.1/frontend/src/pages/Plans.tsx +40 -0
- llmdyno-0.3.1/frontend/src/pages/ProfileEditor.tsx +531 -0
- llmdyno-0.3.1/frontend/src/pages/Profiles.tsx +137 -0
- llmdyno-0.3.1/frontend/src/pages/Providers.tsx +253 -0
- llmdyno-0.3.1/frontend/src/pages/Results.tsx +578 -0
- llmdyno-0.3.1/frontend/src/pages/ResultsExternal.tsx +103 -0
- llmdyno-0.3.1/frontend/src/pages/RunDetail.tsx +400 -0
- llmdyno-0.3.1/frontend/src/pages/Settings.tsx +192 -0
- llmdyno-0.3.1/frontend/src/pages/Setup.tsx +76 -0
- llmdyno-0.3.1/frontend/src/panic.tsx +93 -0
- llmdyno-0.3.1/frontend/src/queries.ts +131 -0
- llmdyno-0.3.1/frontend/src/styles.css +516 -0
- llmdyno-0.3.1/frontend/src/theme.tsx +50 -0
- llmdyno-0.3.1/frontend/src/transcript.tsx +373 -0
- llmdyno-0.3.1/frontend/src/ui.tsx +670 -0
- llmdyno-0.3.1/frontend/src/vite-env.d.ts +2 -0
- llmdyno-0.3.1/frontend/src/ws.ts +150 -0
- llmdyno-0.3.1/frontend/tsconfig.json +19 -0
- llmdyno-0.3.1/frontend/vite.config.ts +15 -0
- llmdyno-0.3.1/llmdyno/__init__.py +50 -0
- llmdyno-0.3.1/llmdyno/__main__.py +54 -0
- llmdyno-0.3.1/llmdyno/api/__init__.py +0 -0
- llmdyno-0.3.1/llmdyno/api/external_api.py +181 -0
- llmdyno-0.3.1/llmdyno/api/results_api.py +225 -0
- llmdyno-0.3.1/llmdyno/api/routes.py +1198 -0
- llmdyno-0.3.1/llmdyno/benchmarks/__init__.py +0 -0
- llmdyno-0.3.1/llmdyno/benchmarks/artificial_analysis.py +175 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t0_smoke_cli/benchmark.json +46 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t0_smoke_cli/grader/spec.json +8 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t0_smoke_cli/grader/tests/test_hidden_wordfreq.py +39 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t0_smoke_cli/prompts/stage1.md +12 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t0_smoke_cli/reference/tests/test_wordfreq.py +13 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t0_smoke_cli/reference/wordfreq.py +34 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t1_flask_site/benchmark.json +49 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t1_flask_site/grader/grade.py +24 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t1_flask_site/grader/tests/test_hidden_flask.py +170 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t1_flask_site/probes/test_probes.py +75 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t1_flask_site/prompts/stage1.md +20 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t1_flask_site/reference/README.md +3 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t1_flask_site/reference/app.py +99 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t1_flask_site/reference/requirements.txt +3 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t1_flask_site/reference/templates/base.html +1 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t1_flask_site/reference/tests/test_app.py +17 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/benchmark.json +49 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/grader/grade.py +56 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/grader/tests/test_hidden_jobs.py +139 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/probes/test_probes.py +89 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/prompts/stage1.md +50 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/README.md +5 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/backend/__init__.py +0 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/backend/db.py +18 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/backend/main.py +70 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/backend/tests/test_api.py +25 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/frontend/index.html +8 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/frontend/src/api.ts +31 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/frontend/src/main.ts +16 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/frontend/tsconfig.json +11 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/requirements.txt +4 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/worker/__init__.py +0 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t2_fastapi_worker_ts/reference/worker/worker.py +49 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/benchmark.json +69 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/grader/grade.py +44 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/helpers.py +11 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h1.py +57 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h1_jsonformat.py +11 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h2.py +76 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h2_jsonformat.py +14 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h3.py +68 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h4.py +85 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/grader/tests/test_h5.py +102 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/probes/test_probes.py +47 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/prompts/stage1.md +21 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/prompts/stage2.md +15 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/prompts/stage3.md +12 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/prompts/stage4.md +11 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/prompts/stage5.md +19 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/reference/inventory/__init__.py +0 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/reference/inventory/__main__.py +62 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/reference/inventory/errors.py +14 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/reference/inventory/models.py +28 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/reference/inventory/store.py +114 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/reference/tests/test_own.py +24 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/stage_files/stage1/tests/test_step1.py +34 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/stage_files/stage2/tests/test_step2.py +22 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/stage_files/stage3/tests/test_step3.py +22 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/stage_files/stage4/tests/data/legacy.json +12 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/stage_files/stage4/tests/test_step4.py +27 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t3_long_iterative/stage_files/stage5/tests/test_step5.py +24 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t4_data_ml/benchmark.json +50 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t4_data_ml/gen_data.py +203 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t4_data_ml/grader/grade.py +161 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t4_data_ml/grader/test_labels.csv +1201 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t4_data_ml/grader/truth.json +22 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t4_data_ml/grader/truth_features.parquet +0 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t4_data_ml/probes/test_probes.py +87 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t4_data_ml/prompts/stage1.md +43 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t4_data_ml/reference/pipeline.py +77 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t4_data_ml/workspace/data/customers.csv +4151 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t4_data_ml/workspace/data/events.jsonl.gz +0 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t4_data_ml/workspace/data/labels_train.csv +2801 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t4_data_ml/workspace/data/test_ids.csv +1201 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t5_bugfix_semver/benchmark.json +47 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t5_bugfix_semver/grader/grade.py +31 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t5_bugfix_semver/grader/tests/test_hidden_semver.py +107 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t5_bugfix_semver/prompts/stage1.md +10 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t5_bugfix_semver/reference/semverlite/version.py +87 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/CHANGELOG.md +12 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/ISSUE.md +17 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/README.md +13 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/semverlite/__init__.py +5 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/semverlite/ranges.py +137 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/semverlite/version.py +83 -0
- llmdyno-0.3.1/llmdyno/benchmarks/builtin/t5_bugfix_semver/workspace/tests/test_basic.py +60 -0
- llmdyno-0.3.1/llmdyno/benchmarks/common/grading.py +61 -0
- llmdyno-0.3.1/llmdyno/benchmarks/common/polyglot_grade.py +107 -0
- llmdyno-0.3.1/llmdyno/benchmarks/common/spec_grade.py +42 -0
- llmdyno-0.3.1/llmdyno/benchmarks/envs.py +147 -0
- llmdyno-0.3.1/llmdyno/benchmarks/polyglot.py +276 -0
- llmdyno-0.3.1/llmdyno/benchmarks/registry.py +270 -0
- llmdyno-0.3.1/llmdyno/benchmarks/swebench.py +361 -0
- llmdyno-0.3.1/llmdyno/benchmarks/swebench_shim.py +28 -0
- llmdyno-0.3.1/llmdyno/benchmarks/validate.py +87 -0
- llmdyno-0.3.1/llmdyno/config.py +107 -0
- llmdyno-0.3.1/llmdyno/db.py +220 -0
- llmdyno-0.3.1/llmdyno/docker/polyglot-test.sh +18 -0
- llmdyno-0.3.1/llmdyno/docker/polyglot.Dockerfile +44 -0
- llmdyno-0.3.1/llmdyno/docker/quality.Dockerfile +7 -0
- llmdyno-0.3.1/llmdyno/docker/sandbox.Dockerfile +33 -0
- llmdyno-0.3.1/llmdyno/events.py +59 -0
- llmdyno-0.3.1/llmdyno/harnesses/__init__.py +0 -0
- llmdyno-0.3.1/llmdyno/harnesses/catalog.py +424 -0
- llmdyno-0.3.1/llmdyno/harnesses/images.py +143 -0
- llmdyno-0.3.1/llmdyno/harnesses/runtime.py +112 -0
- llmdyno-0.3.1/llmdyno/hub/__init__.py +0 -0
- llmdyno-0.3.1/llmdyno/hub/gguf.py +270 -0
- llmdyno-0.3.1/llmdyno/hub/hf.py +238 -0
- llmdyno-0.3.1/llmdyno/inference/__init__.py +0 -0
- llmdyno-0.3.1/llmdyno/inference/llamacpp.py +450 -0
- llmdyno-0.3.1/llmdyno/jobs.py +153 -0
- llmdyno-0.3.1/llmdyno/judge/__init__.py +0 -0
- llmdyno-0.3.1/llmdyno/judge/judge.py +271 -0
- llmdyno-0.3.1/llmdyno/lib.py +995 -0
- llmdyno-0.3.1/llmdyno/llm.py +229 -0
- llmdyno-0.3.1/llmdyno/memory.py +310 -0
- llmdyno-0.3.1/llmdyno/profiles.py +103 -0
- llmdyno-0.3.1/llmdyno/providers/__init__.py +0 -0
- llmdyno-0.3.1/llmdyno/providers/registry.py +156 -0
- llmdyno-0.3.1/llmdyno/proxy/llm_proxy.py +630 -0
- llmdyno-0.3.1/llmdyno/py.typed +0 -0
- llmdyno-0.3.1/llmdyno/quality/__init__.py +0 -0
- llmdyno-0.3.1/llmdyno/quality/analyze.py +361 -0
- llmdyno-0.3.1/llmdyno/runner/__init__.py +0 -0
- llmdyno-0.3.1/llmdyno/runner/executor.py +770 -0
- llmdyno-0.3.1/llmdyno/runner/metrics.py +188 -0
- llmdyno-0.3.1/llmdyno/runner/sandbox.py +275 -0
- llmdyno-0.3.1/llmdyno/safety.py +120 -0
- llmdyno-0.3.1/llmdyno/sample/round1.json +1 -0
- llmdyno-0.3.1/llmdyno/scoring.py +202 -0
- llmdyno-0.3.1/llmdyno/secrets.py +94 -0
- llmdyno-0.3.1/llmdyno/server.py +148 -0
- llmdyno-0.3.1/llmdyno/smoke.py +107 -0
- llmdyno-0.3.1/llmdyno/system/__init__.py +0 -0
- llmdyno-0.3.1/llmdyno/system/apple.py +156 -0
- llmdyno-0.3.1/llmdyno/system/deps.py +237 -0
- llmdyno-0.3.1/llmdyno/system/specs.py +434 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/index-DDIFnEmp.css +1 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/index-DOnMvXno.js +612 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/inter-cyrillic-ext-wght-normal-B2xhLi22.woff2 +0 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/inter-cyrillic-wght-normal-CMZtQduZ.woff2 +0 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/inter-greek-ext-wght-normal-CGAr0uHJ.woff2 +0 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/inter-greek-wght-normal-CaVNZxsx.woff2 +0 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/inter-latin-ext-wght-normal-CFHvXkgd.woff2 +0 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/inter-latin-wght-normal-C2S99t-D.woff2 +0 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/inter-vietnamese-wght-normal-CBcvBZtf.woff2 +0 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/jetbrains-mono-cyrillic-wght-normal-BptiBwAt.woff2 +0 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/jetbrains-mono-greek-wght-normal-BW1JPxja.woff2 +0 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/jetbrains-mono-latin-ext-wght-normal-Com2F4yK.woff2 +0 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/jetbrains-mono-latin-wght-normal-Ajscav9D.woff2 +0 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/jetbrains-mono-vietnamese-wght-normal-By-yputv.woff2 +0 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/press-start-2p-cyrillic-400-normal-oElOGqNf.woff2 +0 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/press-start-2p-cyrillic-ext-400-normal-COIQET2s.woff2 +0 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/press-start-2p-greek-400-normal-DUIT0YPZ.woff2 +0 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/press-start-2p-latin-400-normal-BKfmdmKK.woff +0 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/press-start-2p-latin-400-normal-xj9lRvVs.woff2 +0 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/press-start-2p-latin-ext-400-normal-B18ub5-k.woff +0 -0
- llmdyno-0.3.1/llmdyno/web/static/assets/press-start-2p-latin-ext-400-normal-DDIa4im3.woff2 +0 -0
- llmdyno-0.3.1/llmdyno/web/static/favicon.svg +161 -0
- llmdyno-0.3.1/llmdyno/web/static/index.html +17 -0
- llmdyno-0.3.1/llms.txt +39 -0
- llmdyno-0.3.1/pyproject.toml +74 -0
- llmdyno-0.3.1/scripts/build_sample.py +61 -0
- llmdyno-0.3.1/scripts/bump_version.py +93 -0
- llmdyno-0.3.1/scripts/calibrate_memory.py +35 -0
- llmdyno-0.3.1/scripts/dev_smoke_run.py +56 -0
- llmdyno-0.3.1/scripts/make_logo.py +315 -0
- llmdyno-0.3.1/scripts/platform_probe.py +33 -0
- llmdyno-0.3.1/scripts/screenshots.py +123 -0
- llmdyno-0.3.1/setup.ps1 +63 -0
- llmdyno-0.3.1/setup.sh +61 -0
- llmdyno-0.3.1/tests/conftest.py +36 -0
- llmdyno-0.3.1/tests/fixtures/llama_release.json +157 -0
- llmdyno-0.3.1/tests/test_api.py +110 -0
- llmdyno-0.3.1/tests/test_benchmarks.py +56 -0
- llmdyno-0.3.1/tests/test_events_safety.py +48 -0
- llmdyno-0.3.1/tests/test_external.py +233 -0
- llmdyno-0.3.1/tests/test_library.py +132 -0
- llmdyno-0.3.1/tests/test_memory.py +70 -0
- llmdyno-0.3.1/tests/test_platforms.py +134 -0
- llmdyno-0.3.1/tests/test_profiles_harnesses.py +53 -0
- llmdyno-0.3.1/tests/test_proxy.py +69 -0
- llmdyno-0.3.1/tests/test_scoring.py +20 -0
- llmdyno-0.3.1/tests/test_versioning.py +79 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
name: Bug report
|
|
2
|
+
description: Something doesn't work as described
|
|
3
|
+
labels: [bug]
|
|
4
|
+
body:
|
|
5
|
+
- type: textarea
|
|
6
|
+
id: what
|
|
7
|
+
attributes:
|
|
8
|
+
label: What happened
|
|
9
|
+
description: What you did, what you expected, and what you got instead.
|
|
10
|
+
validations:
|
|
11
|
+
required: true
|
|
12
|
+
- type: textarea
|
|
13
|
+
id: steps
|
|
14
|
+
attributes:
|
|
15
|
+
label: Steps to reproduce
|
|
16
|
+
placeholder: |
|
|
17
|
+
1. Create a local profile for ...
|
|
18
|
+
2. Run plan with harness ...
|
|
19
|
+
3. ...
|
|
20
|
+
- type: input
|
|
21
|
+
id: version
|
|
22
|
+
attributes:
|
|
23
|
+
label: LLMDyno version
|
|
24
|
+
description: Shown at the bottom of the sidebar, or run `llmdyno --version`.
|
|
25
|
+
placeholder: "0.3.1"
|
|
26
|
+
validations:
|
|
27
|
+
required: true
|
|
28
|
+
- type: textarea
|
|
29
|
+
id: system
|
|
30
|
+
attributes:
|
|
31
|
+
label: System
|
|
32
|
+
description: Paste the output of `llmdyno specs`, or describe OS, GPU, driver and Docker version.
|
|
33
|
+
render: json
|
|
34
|
+
- type: textarea
|
|
35
|
+
id: logs
|
|
36
|
+
attributes:
|
|
37
|
+
label: Logs
|
|
38
|
+
description: The job log from the Activity page, or the run's Logs tab. Remove anything private.
|
|
39
|
+
render: text
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
name: Feature request
|
|
2
|
+
description: An idea that would make LLMDyno more useful
|
|
3
|
+
labels: [enhancement]
|
|
4
|
+
body:
|
|
5
|
+
- type: textarea
|
|
6
|
+
id: problem
|
|
7
|
+
attributes:
|
|
8
|
+
label: The problem
|
|
9
|
+
description: What are you trying to measure or do, and what gets in the way today?
|
|
10
|
+
validations:
|
|
11
|
+
required: true
|
|
12
|
+
- type: textarea
|
|
13
|
+
id: idea
|
|
14
|
+
attributes:
|
|
15
|
+
label: What you'd like
|
|
16
|
+
- type: dropdown
|
|
17
|
+
id: area
|
|
18
|
+
attributes:
|
|
19
|
+
label: Area
|
|
20
|
+
options: [Setup and hardware, Models and memory fit, Profiles, Cloud providers, Harnesses, Benchmarks, Runs, Judging, Results dashboard, Other]
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
name: New harness, benchmark or provider
|
|
2
|
+
description: Propose an agent harness, a benchmark task or a cloud provider preset
|
|
3
|
+
labels: [catalog]
|
|
4
|
+
body:
|
|
5
|
+
- type: dropdown
|
|
6
|
+
id: kind
|
|
7
|
+
attributes:
|
|
8
|
+
label: What to add
|
|
9
|
+
options: [Harness, Benchmark, Cloud provider]
|
|
10
|
+
validations:
|
|
11
|
+
required: true
|
|
12
|
+
- type: input
|
|
13
|
+
id: link
|
|
14
|
+
attributes:
|
|
15
|
+
label: Link
|
|
16
|
+
description: Repository, docs or API reference.
|
|
17
|
+
- type: textarea
|
|
18
|
+
id: why
|
|
19
|
+
attributes:
|
|
20
|
+
label: Why it's worth adding
|
|
21
|
+
description: For a benchmark, what skill it tests that the built-in tasks don't. For a harness, what's different about it.
|
|
22
|
+
validations:
|
|
23
|
+
required: true
|
|
24
|
+
- type: textarea
|
|
25
|
+
id: notes
|
|
26
|
+
attributes:
|
|
27
|
+
label: Notes
|
|
28
|
+
description: Licence, API styles it speaks, how it's installed, known quirks.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
version: 2
|
|
2
|
+
# one grouped pull request per ecosystem and month, instead of one per package
|
|
3
|
+
updates:
|
|
4
|
+
- package-ecosystem: pip
|
|
5
|
+
directory: /
|
|
6
|
+
schedule:
|
|
7
|
+
interval: monthly
|
|
8
|
+
groups:
|
|
9
|
+
python:
|
|
10
|
+
patterns: ["*"]
|
|
11
|
+
- package-ecosystem: npm
|
|
12
|
+
directory: /frontend
|
|
13
|
+
schedule:
|
|
14
|
+
interval: monthly
|
|
15
|
+
groups:
|
|
16
|
+
frontend:
|
|
17
|
+
patterns: ["*"]
|
|
18
|
+
- package-ecosystem: github-actions
|
|
19
|
+
directory: /
|
|
20
|
+
schedule:
|
|
21
|
+
interval: monthly
|
|
22
|
+
groups:
|
|
23
|
+
actions:
|
|
24
|
+
patterns: ["*"]
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
## What and why
|
|
2
|
+
|
|
3
|
+
<!-- What does this change, and what problem does it solve? -->
|
|
4
|
+
|
|
5
|
+
## How it was tested
|
|
6
|
+
|
|
7
|
+
<!-- Commands you ran, runs you watched, screenshots for UI changes. -->
|
|
8
|
+
|
|
9
|
+
## Checklist
|
|
10
|
+
|
|
11
|
+
- [ ] `ruff check llmdyno tests scripts` and `pytest` pass
|
|
12
|
+
- [ ] UI changes: `npm run build` in `frontend/`, and the built files are committed
|
|
13
|
+
- [ ] User-visible change: a line under `## [Unreleased]` in `CHANGELOG.md`
|
|
14
|
+
- [ ] Database schema change: `SCHEMA_VERSION` bumped and a migration added
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
concurrency:
|
|
12
|
+
group: ci-${{ github.ref }}
|
|
13
|
+
cancel-in-progress: true
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
python:
|
|
17
|
+
name: Python ${{ matrix.python }} on ${{ matrix.os }}
|
|
18
|
+
runs-on: ${{ matrix.os }}
|
|
19
|
+
# Windows on ARM is new on GitHub's runners and some wheels lag behind; report it without blocking merges
|
|
20
|
+
continue-on-error: ${{ matrix.experimental == true }}
|
|
21
|
+
strategy:
|
|
22
|
+
fail-fast: false
|
|
23
|
+
matrix:
|
|
24
|
+
os: [ubuntu-latest, windows-latest]
|
|
25
|
+
python: ["3.11", "3.12", "3.13"]
|
|
26
|
+
include:
|
|
27
|
+
- os: macos-latest # Apple Silicon (GitHub retires numbered macOS images; latest keeps up)
|
|
28
|
+
python: "3.12"
|
|
29
|
+
- os: ubuntu-24.04-arm # Linux on ARM64
|
|
30
|
+
python: "3.12"
|
|
31
|
+
- os: windows-11-arm # Windows on ARM64
|
|
32
|
+
python: "3.12"
|
|
33
|
+
experimental: true
|
|
34
|
+
steps:
|
|
35
|
+
- uses: actions/checkout@v4
|
|
36
|
+
- uses: actions/setup-python@v5
|
|
37
|
+
with:
|
|
38
|
+
python-version: ${{ matrix.python }}
|
|
39
|
+
cache: pip
|
|
40
|
+
- name: Install
|
|
41
|
+
run: python -m pip install -e ".[dev]"
|
|
42
|
+
- name: Lint
|
|
43
|
+
run: python -m ruff check llmdyno tests scripts
|
|
44
|
+
- name: Test
|
|
45
|
+
run: python -m pytest -q
|
|
46
|
+
- name: CLI smoke
|
|
47
|
+
run: llmdyno --version
|
|
48
|
+
- name: Platform detection
|
|
49
|
+
run: python scripts/platform_probe.py
|
|
50
|
+
|
|
51
|
+
frontend:
|
|
52
|
+
name: Web UI
|
|
53
|
+
runs-on: ubuntu-latest
|
|
54
|
+
defaults:
|
|
55
|
+
run:
|
|
56
|
+
working-directory: frontend
|
|
57
|
+
steps:
|
|
58
|
+
- uses: actions/checkout@v4
|
|
59
|
+
- uses: actions/setup-node@v4
|
|
60
|
+
with:
|
|
61
|
+
node-version: 22
|
|
62
|
+
cache: npm
|
|
63
|
+
cache-dependency-path: frontend/package-lock.json
|
|
64
|
+
- run: npm ci
|
|
65
|
+
- name: Type check
|
|
66
|
+
run: npx tsc --noEmit -p .
|
|
67
|
+
- name: Build
|
|
68
|
+
run: npm run build
|
|
69
|
+
- name: Built UI is committed
|
|
70
|
+
run: test -f ../llmdyno/web/static/index.html
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: write
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
release:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
- uses: actions/setup-python@v5
|
|
16
|
+
with:
|
|
17
|
+
python-version: "3.12"
|
|
18
|
+
- name: Tag matches the package version
|
|
19
|
+
run: |
|
|
20
|
+
VERSION=$(python -c "import re; print(re.search(r'__version__ = \"([^\"]+)\"', open('llmdyno/__init__.py').read()).group(1))")
|
|
21
|
+
if [ "v$VERSION" != "$GITHUB_REF_NAME" ]; then
|
|
22
|
+
echo "Tag $GITHUB_REF_NAME does not match llmdyno/__init__.py ($VERSION)"; exit 1
|
|
23
|
+
fi
|
|
24
|
+
echo "VERSION=$VERSION" >> "$GITHUB_ENV"
|
|
25
|
+
- name: Test
|
|
26
|
+
run: |
|
|
27
|
+
python -m pip install -e ".[dev]" build
|
|
28
|
+
python -m pytest -q
|
|
29
|
+
- name: Build wheel and sdist
|
|
30
|
+
run: python -m build
|
|
31
|
+
- uses: actions/upload-artifact@v4
|
|
32
|
+
with:
|
|
33
|
+
name: dist
|
|
34
|
+
path: dist/
|
|
35
|
+
- name: Release notes from the changelog
|
|
36
|
+
run: |
|
|
37
|
+
awk -v v="$VERSION" '
|
|
38
|
+
$0 ~ "^## \\[" v "\\]" {on=1; next}
|
|
39
|
+
on && /^## \[/ {exit}
|
|
40
|
+
on {print}
|
|
41
|
+
' CHANGELOG.md > notes.md
|
|
42
|
+
test -s notes.md || echo "See CHANGELOG.md" > notes.md
|
|
43
|
+
- name: Publish GitHub release
|
|
44
|
+
env:
|
|
45
|
+
GH_TOKEN: ${{ github.token }}
|
|
46
|
+
run: |
|
|
47
|
+
PRE=""
|
|
48
|
+
case "$VERSION" in *-*) PRE="--prerelease";; esac
|
|
49
|
+
gh release create "$GITHUB_REF_NAME" dist/* --title "LLMDyno $VERSION" --notes-file notes.md $PRE
|
|
50
|
+
|
|
51
|
+
# PyPI, so `pip install llmdyno` works. Off until the repository variable PUBLISH_PYPI is "true" and PyPI has a
|
|
52
|
+
# trusted publisher for this repository and workflow (pypi.org > your project or pending publisher > Publishing:
|
|
53
|
+
# owner speak2jerome, repository llmdyno, workflow release.yml, environment pypi). No token is stored anywhere.
|
|
54
|
+
pypi:
|
|
55
|
+
needs: release
|
|
56
|
+
if: vars.PUBLISH_PYPI == 'true'
|
|
57
|
+
runs-on: ubuntu-latest
|
|
58
|
+
environment: pypi
|
|
59
|
+
permissions:
|
|
60
|
+
id-token: write
|
|
61
|
+
steps:
|
|
62
|
+
- uses: actions/download-artifact@v4
|
|
63
|
+
with:
|
|
64
|
+
name: dist
|
|
65
|
+
path: dist/
|
|
66
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
llmdyno-0.3.1/.gitignore
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# local state
|
|
2
|
+
/data/
|
|
3
|
+
.venv/
|
|
4
|
+
__pycache__/
|
|
5
|
+
*.pyc
|
|
6
|
+
.pytest_cache/
|
|
7
|
+
.ruff_cache/
|
|
8
|
+
calibration.json
|
|
9
|
+
|
|
10
|
+
# packaging
|
|
11
|
+
dist/
|
|
12
|
+
build/
|
|
13
|
+
*.egg-info/
|
|
14
|
+
|
|
15
|
+
# frontend
|
|
16
|
+
frontend/node_modules/
|
|
17
|
+
frontend/dist/
|
|
18
|
+
|
|
19
|
+
# OS / editor
|
|
20
|
+
.DS_Store
|
|
21
|
+
Thumbs.db
|
|
22
|
+
.idea/
|
|
23
|
+
.vscode/
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to LLMDyno are listed here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
|
|
4
|
+
and the project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
5
|
+
|
|
6
|
+
While LLMDyno is 0.x, a minor release (0.2.0) may change the HTTP API, the data folder layout or the database schema.
|
|
7
|
+
Schema changes always ship with an automatic migration; a data folder written by a newer release is refused rather
|
|
8
|
+
than modified. Patch releases (0.1.1) only fix bugs.
|
|
9
|
+
|
|
10
|
+
## [Unreleased]
|
|
11
|
+
|
|
12
|
+
## [0.3.1] - 2026-11-22
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
|
|
16
|
+
- LLMDyno is on PyPI: `pip install llmdyno`. Releases publish there automatically through trusted publishing.
|
|
17
|
+
|
|
18
|
+
### Fixed
|
|
19
|
+
|
|
20
|
+
- The README's images and links are absolute, so they work on the PyPI project page too (a test keeps them that way).
|
|
21
|
+
|
|
22
|
+
## [0.3.0] - 2026-11-22
|
|
23
|
+
|
|
24
|
+
### Added
|
|
25
|
+
|
|
26
|
+
- **Python library**: `import llmdyno` gives the whole app as an API: `cloud_model()` and `local_model()` (download or
|
|
27
|
+
register GGUF files), `bench()` with a preflight that raises `PreflightError` before anything starts, `Results`
|
|
28
|
+
with `leaderboard()`, `runs()`, `to_pandas()`, `to_csv()`, `to_json()` and `open()`, plus `setup()`, `doctor()`,
|
|
29
|
+
SWE-bench and polyglot imports, LLM-judge `review()` and `serve()`. Long operations print progress and Ctrl+C cancels
|
|
30
|
+
them cleanly. It uses the same data folder as the app, so plans run from Python show up in the dashboard.
|
|
31
|
+
`import llmdyno` stays instant: the API loads on first use. Guide: docs/LIBRARY.md; examples in examples/.
|
|
32
|
+
- The release workflow can publish to PyPI (trusted publishing), switched on with the `PUBLISH_PYPI` repository
|
|
33
|
+
variable.
|
|
34
|
+
|
|
35
|
+
### Fixed
|
|
36
|
+
|
|
37
|
+
- `CITATION.cff` and the bug-report template said 0.3.0 in the 0.2.0 release: the version test bumped the real files
|
|
38
|
+
instead of its temporary copies. The test now redirects every file the bump script writes and fails if it touches
|
|
39
|
+
the repository.
|
|
40
|
+
|
|
41
|
+
## [0.2.0] - 2026-11-21
|
|
42
|
+
|
|
43
|
+
### Added
|
|
44
|
+
|
|
45
|
+
- **Apple Silicon and ARM64**: LLMDyno now runs on macOS (Apple Silicon), Linux on ARM64 and Windows on ARM, as well
|
|
46
|
+
as x86-64. Detection covers ARM CPU features (NEON, dot product, i8mm, SVE), performance and efficiency cores,
|
|
47
|
+
Apple GPUs (chip, GPU cores, bandwidth) and Qualcomm Adreno. The Setup page explains the Metal memory limit and
|
|
48
|
+
offers the command to raise it, and on ARM64 hosts it notes that SWE-bench images need x86-64 emulation.
|
|
49
|
+
- **llama.cpp builds for every platform**: release assets are parsed instead of hard-coded, so the installer picks
|
|
50
|
+
the right archive for the OS, CPU architecture and backend (CUDA 12/13, Metal, Vulkan, ROCm, SYCL, OpenVINO,
|
|
51
|
+
OpenCL for Adreno, CPU) and keeps working when llama.cpp moves to a new CUDA version (it just did, to 13.5).
|
|
52
|
+
- **Unified memory**: on Apple Silicon the memory check plans against the GPU's share of unified memory, reports
|
|
53
|
+
going over it as out of memory (swap) rather than a spill, and offers smaller context or quant fixes instead of
|
|
54
|
+
CPU offload. Runs record GPU memory in use and utilisation from IOAccelerator statistics.
|
|
55
|
+
- **SWE-bench**: import Verified, Lite or Verified mini from the official datasets. Agents work inside each task's
|
|
56
|
+
official image; patches are graded by the official `swebench` harness (installed into its own venv from Setup), so
|
|
57
|
+
"resolved" means what it means on the leaderboard. Validate grades a task's gold patch. Plans export their patches
|
|
58
|
+
in the SWE-bench predictions format.
|
|
59
|
+
- **Aider polyglot**: import the 225 exercises (C++, Go, Java, JavaScript, Python, Rust) with suites per language,
|
|
60
|
+
and build the toolchain image from Setup. Every test is enabled, the original tests are restored before grading,
|
|
61
|
+
and an exercise is solved only when all of them pass.
|
|
62
|
+
- **Artificial Analysis reference**: with a free API key (Settings), Results shows each profile's model with its
|
|
63
|
+
Intelligence Index, output speed, TTFT and 3:1 blended price next to what LLMDyno measured, with attribution.
|
|
64
|
+
Matches are automatic and can be corrected per profile.
|
|
65
|
+
- **Portable harness runtimes**: each harness can be built once into a self-contained folder that is mounted into
|
|
66
|
+
any benchmark image, which is how every harness runs SWE-bench and polyglot tasks. Benchmarks can name their own
|
|
67
|
+
image (`env` in `benchmark.json`), so custom tasks can use this too.
|
|
68
|
+
- Results: a Public benchmarks card (SWE-bench resolved rate, polyglot solved rate per language).
|
|
69
|
+
- **Re-grade**: a finished run can be graded again from the workspace (or SWE-bench patch) it left behind, without
|
|
70
|
+
running the agent again, for when a grader is fixed after the fact. Token, time and GPU figures are kept.
|
|
71
|
+
- CI runs on Apple Silicon, Linux ARM64 and Windows on ARM; `scripts/platform_probe.py` prints what LLMDyno detects.
|
|
72
|
+
|
|
73
|
+
### Changed
|
|
74
|
+
|
|
75
|
+
- `setup.sh` accepts macOS and ARM64. Docker Desktop on macOS can be started from Setup, with a Homebrew install
|
|
76
|
+
command when it is missing.
|
|
77
|
+
- On macOS the data folder for pip installs is `~/Library/Application Support/LLMDyno`.
|
|
78
|
+
- llama-server's bind address can be overridden with the `llama_bind_host` setting.
|
|
79
|
+
|
|
80
|
+
### Fixed
|
|
81
|
+
|
|
82
|
+
- Setup spots Docker Desktop on Windows stuck at start-up ("The file cannot be accessed by the system") because a
|
|
83
|
+
crash left socket files behind, and offers a one-click fix that moves them aside. "Reset to factory defaults",
|
|
84
|
+
which Docker suggests, would delete every image.
|
|
85
|
+
- Setup loads on Apple Silicon Macs where macOS does not report the CPU frequency (psutil raised "'pmgr' entry
|
|
86
|
+
was not found"), as seen on GitHub's macOS runners.
|
|
87
|
+
- Setup, Harnesses and the import dialog load on machines without Docker installed; before, the missing `docker`
|
|
88
|
+
command crashed them.
|
|
89
|
+
- Leaderboard bars add up to the final score when a configuration has no code-quality score (before, they were
|
|
90
|
+
scaled as if quality counted as zero while the score left it out).
|
|
91
|
+
|
|
92
|
+
## [0.1.0] - 2026-10-26
|
|
93
|
+
|
|
94
|
+
First public release.
|
|
95
|
+
|
|
96
|
+
### Added
|
|
97
|
+
|
|
98
|
+
- **Setup**: detects the OS, CPU (AVX2/AVX-512), RAM, NVIDIA/AMD/Intel GPUs, driver and CUDA version, disks and Docker
|
|
99
|
+
on x86-64 Windows and Linux; a guided first-run wizard installs or points to Docker and a matching prebuilt
|
|
100
|
+
llama.cpp release (CUDA 12/13, Vulkan, ROCm, SYCL or CPU).
|
|
101
|
+
- **Models**: Hugging Face search with a memory-fit check before download; resumable downloads with SHA-256
|
|
102
|
+
verification; import of existing GGUF folders; GGUF metadata read over HTTP range requests.
|
|
103
|
+
- **Memory projection**: per-layer KV cache (sliding-window, shared-KV, hybrid recurrent and MLA layers), compute
|
|
104
|
+
buffers and CUDA overhead, checked against `llama-fit-params`; warnings for out-of-memory, VRAM spilling into
|
|
105
|
+
shared system memory (Windows Sysmem Fallback), CPU offload, context limits and risky sampling settings.
|
|
106
|
+
- **LLM profiles** for local (llama.cpp) and cloud models, with one-click fixes, an exact memory check and a quick
|
|
107
|
+
test that measures real VRAM, speed and tool calling.
|
|
108
|
+
- **Cloud providers**: OpenAI, Anthropic, Google Gemini, DeepSeek, Moonshot (Kimi), Z.ai, Qwen, MiniMax, xAI,
|
|
109
|
+
Mistral, OpenRouter, Groq, Together, Fireworks, Cerebras and any OpenAI-compatible endpoint; keys encrypted at
|
|
110
|
+
rest (DPAPI on Windows); per-provider and per-plan spend caps enforced by a proxy.
|
|
111
|
+
- **Harnesses**: Pi, OpenCode, Codex CLI, Qwen Code, Crush, Aider and mini-SWE-agent in offline Docker sandboxes,
|
|
112
|
+
plus custom harnesses from a JSON manifest.
|
|
113
|
+
- **Benchmarks**: six built-in tasks with hidden tests, a benchmark editor, suites, zip import/export and a grader
|
|
114
|
+
validator that checks the starter fails and the reference passes.
|
|
115
|
+
- **Plans**: profiles × harnesses × benchmarks × iterations, preflight checks, live transcripts for every harness,
|
|
116
|
+
GPU telemetry, pause, resume, retry and cancel.
|
|
117
|
+
- **Judging**: blind, shuffled LLM code review with an editable rubric, any number of judges, judge agreement and
|
|
118
|
+
self-preference statistics.
|
|
119
|
+
- **Results dashboard**: leaderboard with confidence intervals, per-benchmark heatmap, radar, cost/speed scatter
|
|
120
|
+
with a Pareto front, consistency (pass@k, pass^k), token use, harness comparison, rubric heatmap, side-by-side
|
|
121
|
+
compare, adjustable weights and CSV/JSON export.
|
|
122
|
+
- **Panic stop**: one button (Ctrl+Shift+X) that cancels every job and run, removes all sandbox and proxy
|
|
123
|
+
containers and kills llama-server.
|
|
124
|
+
- **Mascot**: an animated pixel dinosaur on a treadmill that shows what the app is doing; the browser tab icon and
|
|
125
|
+
title follow it.
|
|
126
|
+
- Bundled sample results: 15 configurations × 5 tasks on an RTX 5080, judged by two models.
|
|
127
|
+
|
|
128
|
+
[Unreleased]: https://github.com/speak2jerome/llmdyno/compare/v0.3.1...HEAD
|
|
129
|
+
[0.3.1]: https://github.com/speak2jerome/llmdyno/compare/v0.3.0...v0.3.1
|
|
130
|
+
[0.3.0]: https://github.com/speak2jerome/llmdyno/compare/v0.2.0...v0.3.0
|
|
131
|
+
[0.2.0]: https://github.com/speak2jerome/llmdyno/compare/v0.1.0...v0.2.0
|
|
132
|
+
[0.1.0]: https://github.com/speak2jerome/llmdyno/releases/tag/v0.1.0
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use LLMDyno in your work, please cite it as below."
|
|
3
|
+
title: "LLMDyno: benchmark local and cloud LLMs as coding agents"
|
|
4
|
+
abstract: >-
|
|
5
|
+
LLMDyno runs open-source coding-agent harnesses against multi-file tasks with hidden tests, on local models
|
|
6
|
+
(llama.cpp, GGUF) or cloud APIs, with VRAM fit projection, sandboxed execution, spend caps, blind LLM judging and a
|
|
7
|
+
results dashboard comparing correctness, code quality, speed, memory and cost.
|
|
8
|
+
type: software
|
|
9
|
+
authors:
|
|
10
|
+
- alias: speak2jerome
|
|
11
|
+
email: speak2jerome@gmail.com
|
|
12
|
+
repository-code: "https://github.com/speak2jerome/llmdyno"
|
|
13
|
+
url: "https://github.com/speak2jerome/llmdyno"
|
|
14
|
+
license: MIT
|
|
15
|
+
version: 0.3.1
|
|
16
|
+
date-released: "2026-11-22"
|
|
17
|
+
keywords:
|
|
18
|
+
- large language models
|
|
19
|
+
- LLM benchmark
|
|
20
|
+
- coding agents
|
|
21
|
+
- llama.cpp
|
|
22
|
+
- GGUF
|
|
23
|
+
- local LLM
|
|
24
|
+
- LLM-as-a-judge
|
|
25
|
+
- VRAM
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Code of conduct
|
|
2
|
+
|
|
3
|
+
LLMDyno follows the [Contributor Covenant, version 2.1](https://www.contributor-covenant.org/version/2/1/code_of_conduct/).
|
|
4
|
+
|
|
5
|
+
In short: be respectful, assume good faith, critique code and ideas rather than people, and keep discussions on topic.
|
|
6
|
+
Harassment, personal attacks and discriminatory language are not tolerated.
|
|
7
|
+
|
|
8
|
+
Report problems privately to speak2jerome@gmail.com. Reports are handled confidentially, and the maintainer may warn,
|
|
9
|
+
remove content from, or ban anyone who breaks these rules.
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# Contributing to LLMDyno
|
|
2
|
+
|
|
3
|
+
Thanks for helping. Bug reports, benchmarks, harness adapters, provider presets and docs fixes are all welcome.
|
|
4
|
+
|
|
5
|
+
## Development setup
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
git clone https://github.com/speak2jerome/llmdyno.git
|
|
9
|
+
cd llmdyno
|
|
10
|
+
python -m venv .venv
|
|
11
|
+
.venv/bin/pip install -e ".[dev]" # Windows: .venv\Scripts\pip install -e ".[dev]"
|
|
12
|
+
cd frontend && npm ci && cd ..
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Run the backend and the UI dev server side by side:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
.venv/bin/llmdyno serve # API on http://127.0.0.1:8777
|
|
19
|
+
cd frontend && npm run dev # UI with hot reload on http://127.0.0.1:5177
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
The built UI in `llmdyno/web/static/` is committed so users don't need Node. If you change anything in `frontend/`,
|
|
23
|
+
run `npm run build` and commit the result.
|
|
24
|
+
|
|
25
|
+
## Before you open a pull request
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
.venv/bin/ruff check llmdyno tests scripts
|
|
29
|
+
.venv/bin/pytest
|
|
30
|
+
cd frontend && npx tsc --noEmit && npm run build
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
The tests need no GPU, Docker or network. CI runs the same checks on Windows and Linux.
|
|
34
|
+
|
|
35
|
+
## What goes where
|
|
36
|
+
|
|
37
|
+
| Area | Path | Guide |
|
|
38
|
+
|---|---|---|
|
|
39
|
+
| Harness adapters | `llmdyno/harnesses/catalog.py` | [docs/HARNESSES.md](docs/HARNESSES.md) |
|
|
40
|
+
| Built-in benchmarks | `llmdyno/benchmarks/builtin/` | [docs/BENCHMARKS.md](docs/BENCHMARKS.md) |
|
|
41
|
+
| Cloud provider presets | `llmdyno/providers/registry.py` | add an entry to `VENDORS` |
|
|
42
|
+
| Memory projection | `llmdyno/memory.py` | re-run `scripts/calibrate_memory.py` on real GGUFs |
|
|
43
|
+
| Mascot sprites | `scripts/make_logo.py` | edit the layers, then run the script |
|
|
44
|
+
|
|
45
|
+
## Style
|
|
46
|
+
|
|
47
|
+
- Python: ruff with a 120-character line limit. Comments explain why, not what.
|
|
48
|
+
- TypeScript: strict mode, functional React components, the shared kit in `frontend/src/ui.tsx`.
|
|
49
|
+
- UI text: plain, specific, and honest about trade-offs. Say what a button does and what it costs.
|
|
50
|
+
|
|
51
|
+
## Versioning
|
|
52
|
+
|
|
53
|
+
LLMDyno follows Semantic Versioning (0.x for now). Add a line under `## [Unreleased]` in `CHANGELOG.md` for any
|
|
54
|
+
user-visible change. Maintainers cut releases with `scripts/bump_version.py`; see [docs/RELEASING.md](docs/RELEASING.md).
|
|
55
|
+
If you change the database schema, bump `SCHEMA_VERSION` in `llmdyno/db.py` and add a migration.
|
|
56
|
+
|
|
57
|
+
## Code of conduct
|
|
58
|
+
|
|
59
|
+
Be kind and assume good faith. See [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md).
|
llmdyno-0.3.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 speak2jerome
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
llmdyno-0.3.1/NOTICE.md
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Third-party notices
|
|
2
|
+
|
|
3
|
+
LLMDyno is MIT licensed (see [LICENSE](LICENSE)). It bundles or downloads the following.
|
|
4
|
+
|
|
5
|
+
## Bundled in the web UI
|
|
6
|
+
|
|
7
|
+
| Component | License | Use |
|
|
8
|
+
|---|---|---|
|
|
9
|
+
| [@lobehub/icons-static-svg](https://github.com/lobehub/lobe-icons) | MIT | vendor and model-family logos |
|
|
10
|
+
| [Press Start 2P](https://fonts.google.com/specimen/Press+Start+2P) by CodeMan38 | SIL OFL 1.1 | pixel wordmark |
|
|
11
|
+
| [Inter](https://rsms.me/inter/) | SIL OFL 1.1 | UI text |
|
|
12
|
+
| [JetBrains Mono](https://www.jetbrains.com/lp/mono/) | SIL OFL 1.1 | code |
|
|
13
|
+
| [React](https://react.dev), [TanStack Query](https://tanstack.com/query), [React Router](https://reactrouter.com) | MIT | UI |
|
|
14
|
+
| [Apache ECharts](https://echarts.apache.org) | Apache 2.0 | charts |
|
|
15
|
+
| [Lucide](https://lucide.dev) | ISC | icons |
|
|
16
|
+
| [marked](https://marked.js.org), [DOMPurify](https://github.com/cure53/DOMPurify), [highlight.js](https://highlightjs.org), [jsdiff](https://github.com/kpdecker/jsdiff) | MIT / Apache 2.0 / BSD-3-Clause / BSD-3-Clause | transcripts and diffs |
|
|
17
|
+
|
|
18
|
+
## Downloaded at setup time (not redistributed)
|
|
19
|
+
|
|
20
|
+
- [llama.cpp](https://github.com/ggml-org/llama.cpp) prebuilt releases (MIT), from GitHub.
|
|
21
|
+
- Agent harnesses, installed into Docker images when you build them: Pi (MIT), OpenCode (MIT), Codex CLI
|
|
22
|
+
(Apache 2.0), Qwen Code (Apache 2.0), Crush (FSL-1.1-MIT), Aider (Apache 2.0) and mini-SWE-agent (MIT). Each keeps
|
|
23
|
+
its own licence.
|
|
24
|
+
- Models you choose from Hugging Face, under their own licences.
|
|
25
|
+
|
|
26
|
+
## Trademarks
|
|
27
|
+
|
|
28
|
+
Product names and logos (OpenAI, Anthropic, Claude, Google, Gemini, Gemma, Meta, Llama, Qwen, Alibaba, DeepSeek,
|
|
29
|
+
Moonshot, Kimi, Z.ai, Zhipu, MiniMax, xAI, Grok, Mistral, NVIDIA, Microsoft, IBM, OpenRouter, Groq, Together,
|
|
30
|
+
Fireworks, Cerebras, Hugging Face and others) are trademarks of their owners. LLMDyno shows them only to identify
|
|
31
|
+
which provider or model a profile uses. LLMDyno is not affiliated with or endorsed by any of them.
|
|
32
|
+
|
|
33
|
+
The mascot is original pixel art made for LLMDyno; it is not derived from any other game's sprites.
|