mjolnir-cli 0.2.1__tar.gz → 0.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/.gitignore +8 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/NOTICE +1 -1
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/PKG-INFO +39 -11
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/README.md +32 -4
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/__init__.py +1 -1
- mjolnir_cli-0.4.2/mjolnir_cli/__main__.py +10 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/accuracy.py +35 -12
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/autoresearch.py +136 -29
- mjolnir_cli-0.4.2/mjolnir_cli/availability.py +349 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/benchy.py +18 -6
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/AWQ_FLASHINFER/README.md +69 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/AWQ_FLASHINFER/config.yaml +77 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/AWQ_FLASHINFER/platforms/orin.yaml +18 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/AWQ_MJOLNIR/README.md +36 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/AWQ_MJOLNIR/config.yaml +81 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/AWQ_MJOLNIR/platforms/orin.yaml +18 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/AWQ_MJOLNIR_E2M3/.env +4 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/AWQ_MJOLNIR_E2M3/README.md +36 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/AWQ_MJOLNIR_E2M3/config.yaml +82 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/AWQ_MJOLNIR_E2M3/platforms/orin.yaml +18 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/AWQ_MJOLNIR_E2M3_VT64K/.env +6 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/AWQ_MJOLNIR_E2M3_VT64K/README.md +43 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/AWQ_MJOLNIR_E2M3_VT64K/config.yaml +90 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/AWQ_MJOLNIR_E2M3_VT64K/platforms/orin.yaml +18 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/AWQ_MJOLNIR_VT64K/.env +4 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/AWQ_MJOLNIR_VT64K/README.md +84 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/AWQ_MJOLNIR_VT64K/config.yaml +83 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/AWQ_MJOLNIR_VT64K/platforms/orin.yaml +18 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4/README.md +114 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4/platforms/orin.yaml +5 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4/platforms/spark.yaml +5 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4/platforms/thor.yaml +4 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_DFLASH2/README.md +36 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_DSPARK/README.md +36 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_FA2/README.md +36 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_FA4hd256/README.md +36 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR/README.md +82 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3/README.md +82 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_DFLASH2/README.md +36 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_DSPARK/README.md +36 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_LMH8_VT64K/README.md +102 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/README.md +87 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/kv_scales/draft_vocab-awq.json +1 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/kv_scales/e2m3-awq.json +35197 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/nvidia/Qwen3.6-35B-A3B/NVFP4_DFLASH/README.md +72 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/nvidia/Qwen3.6-35B-A3B/NVFP4_MJOLNIR/README.md +65 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/nvidia/Qwen3.6-35B-A3B/NVFP4_MJOLNIR_E2M3/README.md +102 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/nvidia/Qwen3.6-35B-A3B/NVFP4_MJOLNIR_E2M3/config.yaml +2 -2
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/nvidia/Qwen3.6-35B-A3B/NVFP4_MJOLNIR_E2M3_LMH8_VT64K/README.md +82 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/nvidia/Qwen3.6-35B-A3B/NVFP4_MJOLNIR_E2M3_VT64K/README.md +36 -0
- mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/nvidia/Qwen3.6-35B-A3B/README.md +62 -0
- mjolnir_cli-0.2.1/mjolnir_cli/bundled/scripts/hw/setup-thor-steps.sh → mjolnir_cli-0.4.2/mjolnir_cli/bundled/scripts/hw/setup-jetson-steps.sh +127 -31
- mjolnir_cli-0.2.1/mjolnir_cli/bundled/scripts/hw/setup-thor.sh → mjolnir_cli-0.4.2/mjolnir_cli/bundled/scripts/hw/setup-jetson.sh +35 -17
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/charts.py +167 -73
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/charts_base.py +54 -10
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/charts_e2e.py +184 -75
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/charts_serving.py +57 -11
- mjolnir_cli-0.4.2/mjolnir_cli/cli.py +125 -0
- mjolnir_cli-0.4.2/mjolnir_cli/cli_shared.py +38 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/clidoc.py +13 -0
- mjolnir_cli-0.4.2/mjolnir_cli/commands/__init__.py +11 -0
- mjolnir_cli-0.4.2/mjolnir_cli/commands/_common.py +58 -0
- mjolnir_cli-0.2.1/mjolnir_cli/cli_bench.py → mjolnir_cli-0.4.2/mjolnir_cli/commands/bench.py +88 -133
- mjolnir_cli-0.4.2/mjolnir_cli/commands/bench_report.py +104 -0
- mjolnir_cli-0.4.2/mjolnir_cli/commands/calibrate.py +94 -0
- mjolnir_cli-0.4.2/mjolnir_cli/commands/hw.py +102 -0
- mjolnir_cli-0.4.2/mjolnir_cli/commands/image.py +162 -0
- mjolnir_cli-0.4.2/mjolnir_cli/commands/litellm.py +91 -0
- mjolnir_cli-0.4.2/mjolnir_cli/commands/model.py +163 -0
- mjolnir_cli-0.4.2/mjolnir_cli/commands/perses.py +72 -0
- mjolnir_cli-0.4.2/mjolnir_cli/commands/platform.py +46 -0
- mjolnir_cli-0.4.2/mjolnir_cli/commands/results.py +153 -0
- mjolnir_cli-0.4.2/mjolnir_cli/commands/retention.py +35 -0
- mjolnir_cli-0.4.2/mjolnir_cli/commands/serve.py +94 -0
- mjolnir_cli-0.4.2/mjolnir_cli/commands/version.py +12 -0
- mjolnir_cli-0.4.2/mjolnir_cli/commands/window.py +54 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/config.py +81 -16
- mjolnir_cli-0.4.2/mjolnir_cli/config_bench.py +254 -0
- mjolnir_cli-0.4.2/mjolnir_cli/config_profile.py +127 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/dockerctl.py +171 -28
- mjolnir_cli-0.4.2/mjolnir_cli/drivers/fa4_driver.py +105 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/drivers/fi_driver.py +10 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/e2e.py +70 -52
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/gate.py +17 -0
- mjolnir_cli-0.4.2/mjolnir_cli/history.py +276 -0
- mjolnir_cli-0.4.2/mjolnir_cli/interactive.py +25 -0
- mjolnir_cli-0.4.2/mjolnir_cli/legs.py +235 -0
- mjolnir_cli-0.4.2/mjolnir_cli/overlays.py +145 -0
- mjolnir_cli-0.4.2/mjolnir_cli/platforms.py +542 -0
- mjolnir_cli-0.4.2/mjolnir_cli/rawprune.py +111 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/readme.py +279 -73
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/readme_tables.py +8 -2
- mjolnir_cli-0.4.2/mjolnir_cli/readme_targets.py +702 -0
- mjolnir_cli-0.4.2/mjolnir_cli/retention.py +167 -0
- mjolnir_cli-0.4.2/mjolnir_cli/site/__init__.py +9 -0
- mjolnir_cli-0.4.2/mjolnir_cli/site/app.css +136 -0
- mjolnir_cli-0.4.2/mjolnir_cli/site/app.js +662 -0
- mjolnir_cli-0.4.2/mjolnir_cli/site/data.py +161 -0
- mjolnir_cli-0.4.2/mjolnir_cli/site/page.py +123 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/stack.py +2 -1
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/tasks.py +30 -12
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/theme.py +40 -12
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/thor.py +44 -5
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/verify/cli.py +99 -24
- mjolnir_cli-0.4.2/mjolnir_cli/verify/correctness.py +263 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/verify/docker.py +121 -17
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/verify/legs.py +77 -9
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/verify/results.py +96 -8
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/verify/stages.py +188 -254
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/pyproject.toml +11 -9
- mjolnir_cli-0.2.1/mjolnir_cli/__main__.py +0 -6
- mjolnir_cli-0.2.1/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_LMH8/.env +0 -2
- mjolnir_cli-0.2.1/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_LMH8/config.yaml +0 -98
- mjolnir_cli-0.2.1/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_LMH8_VT32K/.env +0 -4
- mjolnir_cli-0.2.1/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_LMH8_VT32K/config.yaml +0 -98
- mjolnir_cli-0.2.1/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_VT32K/.env +0 -3
- mjolnir_cli-0.2.1/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_VT32K/config.yaml +0 -98
- mjolnir_cli-0.2.1/mjolnir_cli/cli.py +0 -935
- mjolnir_cli-0.2.1/mjolnir_cli/cli_shared.py +0 -42
- mjolnir_cli-0.2.1/mjolnir_cli/history.py +0 -159
- mjolnir_cli-0.2.1/mjolnir_cli/interactive.py +0 -383
- mjolnir_cli-0.2.1/mjolnir_cli/legs.py +0 -109
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/LICENSE +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/hatch_build.py +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4/config.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_DFLASH2/config.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_DSPARK/config.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_FA2/config.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_FA4hd256/config.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR/config.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3/config.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_DFLASH2/config.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_DSPARK/config.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_LMH8_VT64K/.env +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/NVFP4_MJOLNIR_E2M3_LMH8_VT64K/config.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/draft_vocab.json +0 -0
- /mjolnir_cli-0.2.1/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/kv_scales.json → /mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/Qwen/Qwen3.8-27B/kv_scales/e2m3.json +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/Qwen/Qwen3.8_chat_template.jinja +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/nvidia/Qwen3.6-35B-A3B/NVFP4_DFLASH/.env +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/nvidia/Qwen3.6-35B-A3B/NVFP4_DFLASH/config.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/nvidia/Qwen3.6-35B-A3B/NVFP4_MJOLNIR/config.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/nvidia/Qwen3.6-35B-A3B/NVFP4_MJOLNIR_E2M3/.env +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/nvidia/Qwen3.6-35B-A3B/NVFP4_MJOLNIR_E2M3_LMH8_VT64K/.env +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/nvidia/Qwen3.6-35B-A3B/NVFP4_MJOLNIR_E2M3_LMH8_VT64K/config.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/nvidia/Qwen3.6-35B-A3B/NVFP4_MJOLNIR_E2M3_VT64K/.env +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/nvidia/Qwen3.6-35B-A3B/NVFP4_MJOLNIR_E2M3_VT64K/config.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/configs/nvidia/Qwen3.6-35B-A3B/draft_vocab.json +0 -0
- /mjolnir_cli-0.2.1/mjolnir_cli/bundled/configs/nvidia/Qwen3.6-35B-A3B/kv_scales.json → /mjolnir_cli-0.4.2/mjolnir_cli/bundled/configs/nvidia/Qwen3.6-35B-A3B/kv_scales/e2m3.json +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/docker/litellm/docker-compose.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/docker/litellm/litellm.env.example +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/docker/litellm/litellm_config.template.example.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/docker/perses/README.md +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/docker/perses/dashboards/global-datasource.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/docker/perses/dashboards/performance_statistics.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/docker/perses/dashboards/project.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/docker/perses/dashboards/query_statistics.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/docker/perses/docker-compose.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/docker/perses/perses.yml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/bundled/docker/perses/prometheus.template.yaml +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/cachectl.py +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/drivers/fa2_driver.py +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/fonts/InterDisplay-Bold.ttf +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/fonts/InterDisplay-Medium.ttf +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/fonts/InterDisplay-Regular.ttf +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/fonts/InterDisplay-SemiBold.ttf +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/fonts/OFL.txt +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/litellm.py +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/perses.py +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/picker.py +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/py.typed +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/verify/__init__.py +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/verify/kvcache.py +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/verify/ncu.py +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/verify/profile.py +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/verify/report.py +0 -0
- {mjolnir_cli-0.2.1 → mjolnir_cli-0.4.2}/mjolnir_cli/verify/spec.py +0 -0
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
__pycache__/
|
|
2
2
|
*.py[cod]
|
|
3
3
|
.venv/
|
|
4
|
+
.playwright-mcp/
|
|
4
5
|
.pytest_cache/
|
|
5
6
|
.ruff_cache/
|
|
6
7
|
.env
|
|
8
|
+
# the configs' env sidecars are plain mjolnir knobs (calibration paths, feature
|
|
9
|
+
# flags) and ride with the configs; real secrets stay out
|
|
10
|
+
!configs/**/.env
|
|
7
11
|
# coverage artifacts (the CI test stage writes these; they regenerate)
|
|
8
12
|
.coverage
|
|
9
13
|
.coverage.*
|
|
@@ -24,6 +28,7 @@ benchmarks/raw/*
|
|
|
24
28
|
!benchmarks/raw/accuracy/
|
|
25
29
|
benchmarks/raw/accuracy/*/*/
|
|
26
30
|
benchmarks/logs/
|
|
31
|
+
benchmarks/staging/
|
|
27
32
|
task/
|
|
28
33
|
|
|
29
34
|
# autoresearch: the live ledger is untracked (discards are `git reset --hard`);
|
|
@@ -32,3 +37,6 @@ research/ledger.jsonl
|
|
|
32
37
|
research/ledger.tsv
|
|
33
38
|
# …and the exported runs ARE committed (the generic runs/ rule above would hide them)
|
|
34
39
|
!research/runs/
|
|
40
|
+
/cutlass/
|
|
41
|
+
/fa2/
|
|
42
|
+
/llamacpp/
|
|
@@ -8,7 +8,7 @@ works listed below; each keeps its own license and copyright notice.
|
|
|
8
8
|
-------------------------------------------------------------------------------
|
|
9
9
|
1. NVIDIA CUTLASS — CuTe DSL examples BSD-3-Clause
|
|
10
10
|
-------------------------------------------------------------------------------
|
|
11
|
-
src/mjolnir/mjolnir/
|
|
11
|
+
src/mjolnir/mjolnir/arch/thor/decode.py (the Mjolnir decode-attention kernel) is
|
|
12
12
|
derived from
|
|
13
13
|
|
|
14
14
|
examples/python/CuTeDSL/cute/blackwell/kernel/attention/mixed_input_fmha/
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: mjolnir-cli
|
|
3
|
-
Version: 0.2
|
|
3
|
+
Version: 0.4.2
|
|
4
4
|
Summary: Serve, benchmark and document LLMs on NVIDIA Jetson AGX Thor: YAML model configs, the Mjolnir vLLM image, KV-cache calibration, kernel / serving / accuracy benchmarks and auto-rendered reports.
|
|
5
|
-
Project-URL: Homepage, https://github.com/
|
|
6
|
-
Project-URL: Documentation, https://github.com/
|
|
7
|
-
Project-URL: Repository, https://github.com/
|
|
8
|
-
Project-URL: Issues, https://github.com/
|
|
9
|
-
Project-URL: Changelog, https://github.com/
|
|
10
|
-
Project-URL: CLI reference, https://github.com/
|
|
5
|
+
Project-URL: Homepage, https://github.com/GoBrrr-ai/mjolnir
|
|
6
|
+
Project-URL: Documentation, https://github.com/GoBrrr-ai/mjolnir/tree/main/docs
|
|
7
|
+
Project-URL: Repository, https://github.com/GoBrrr-ai/mjolnir.git
|
|
8
|
+
Project-URL: Issues, https://github.com/GoBrrr-ai/mjolnir/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/GoBrrr-ai/mjolnir/releases
|
|
10
|
+
Project-URL: CLI reference, https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/cli.md
|
|
11
11
|
Author-email: Mieszko Syty <mieszko@ms1design.pl>
|
|
12
12
|
License-Expression: Apache-2.0
|
|
13
13
|
License-File: LICENSE
|
|
@@ -34,7 +34,7 @@ Description-Content-Type: text/markdown
|
|
|
34
34
|
# mjolnir-cli
|
|
35
35
|
|
|
36
36
|
**Serve, benchmark and document LLMs on the NVIDIA Jetson AGX Thor** with
|
|
37
|
-
[Mjolnir](https://github.com/
|
|
37
|
+
[Mjolnir](https://github.com/GoBrrr-ai/mjolnir) — a decode-attention kernel
|
|
38
38
|
built for Thor (`sm_110`), a 6-bit KV-cache format for vLLM and a patched vLLM
|
|
39
39
|
image that runs them. This package is the `mjolnir` command that drives all of
|
|
40
40
|
it; the kernel itself ships separately as
|
|
@@ -69,17 +69,45 @@ mjolnir serve up # serve the default model on the Mjolnir image,
|
|
|
69
69
|
tests and logs kernel experiments.
|
|
70
70
|
|
|
71
71
|
Every command, with options and defaults:
|
|
72
|
-
**[the CLI reference](https://github.com/
|
|
72
|
+
**[the CLI reference](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/cli.md)**
|
|
73
73
|
(generated from the commands; `mjolnir <command> --help` prints the same).
|
|
74
74
|
|
|
75
|
+
## Library use
|
|
76
|
+
|
|
77
|
+
Two modules are declared a public, stable API — reuse them instead of
|
|
78
|
+
re-implementing the clean-window gate or the bench-history substrate:
|
|
79
|
+
|
|
80
|
+
- **`mjolnir_cli.gate`** — `wait_for_idle`, `server_load`, `server_counters`,
|
|
81
|
+
`spec_delta`, the `CleanWindow` context manager, `PreflightError` and the
|
|
82
|
+
`DEFAULT_*` tuning constants.
|
|
83
|
+
- **`mjolnir_cli.history`** — `append_record`, `load_records` and
|
|
84
|
+
`SCHEMA_VERSION`; the history-row schema is documented in the module
|
|
85
|
+
docstring. Rows carry `schema_version` — absence means the legacy (pre-0.3)
|
|
86
|
+
format — and `load_records` returns rows raw, never normalized.
|
|
87
|
+
`append_record` files one record per serving key (platform, model, image,
|
|
88
|
+
config): a re-run merges its cells into the stored record instead of
|
|
89
|
+
appending a line; records without a serving key are appended verbatim.
|
|
90
|
+
|
|
91
|
+
Anything else can change without notice; the module docstrings are the API
|
|
92
|
+
reference: [`gate.py`](https://github.com/GoBrrr-ai/mjolnir/blob/main/src/mjolnir-cli/mjolnir_cli/gate.py),
|
|
93
|
+
[`history.py`](https://github.com/GoBrrr-ai/mjolnir/blob/main/src/mjolnir-cli/mjolnir_cli/history.py).
|
|
94
|
+
|
|
95
|
+
`uv tool install mjolnir-cli` is an isolated CLI install — *not* importable
|
|
96
|
+
from another project. Library use means adding `mjolnir-cli` as a dependency
|
|
97
|
+
of your project (`uv add mjolnir-cli` / `pip install mjolnir-cli`). The
|
|
98
|
+
gate/history import chain is stdlib-only: the CLI dependency stack (typer,
|
|
99
|
+
rich, pyyaml, jinja2) is paid at install time, not import time. Python ≥ 3.12
|
|
100
|
+
is required (pip refuses older interpreters — relevant on JetPack 6-era
|
|
101
|
+
images), and type annotations ship (`py.typed`).
|
|
102
|
+
|
|
75
103
|
## Requirements
|
|
76
104
|
|
|
77
105
|
A Jetson AGX Thor (JetPack 7.x) with Docker and the NVIDIA Container Toolkit
|
|
78
|
-
for anything that touches the GPU; Python ≥ 3.
|
|
106
|
+
for anything that touches the GPU; Python ≥ 3.12. Charts need the `plot` extra
|
|
79
107
|
(`uv tool install 'mjolnir-cli[plot]'`).
|
|
80
108
|
|
|
81
109
|
## Versions
|
|
82
110
|
|
|
83
111
|
`mjolnir-cli` and `mjolnir-kernel` version independently; the Docker image is
|
|
84
112
|
tagged after the upstream vLLM it is built on (`v0.31.0`). Release notes:
|
|
85
|
-
[GitHub Releases](https://github.com/
|
|
113
|
+
[GitHub Releases](https://github.com/GoBrrr-ai/mjolnir/releases).
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# mjolnir-cli
|
|
2
2
|
|
|
3
3
|
**Serve, benchmark and document LLMs on the NVIDIA Jetson AGX Thor** with
|
|
4
|
-
[Mjolnir](https://github.com/
|
|
4
|
+
[Mjolnir](https://github.com/GoBrrr-ai/mjolnir) — a decode-attention kernel
|
|
5
5
|
built for Thor (`sm_110`), a 6-bit KV-cache format for vLLM and a patched vLLM
|
|
6
6
|
image that runs them. This package is the `mjolnir` command that drives all of
|
|
7
7
|
it; the kernel itself ships separately as
|
|
@@ -36,17 +36,45 @@ mjolnir serve up # serve the default model on the Mjolnir image,
|
|
|
36
36
|
tests and logs kernel experiments.
|
|
37
37
|
|
|
38
38
|
Every command, with options and defaults:
|
|
39
|
-
**[the CLI reference](https://github.com/
|
|
39
|
+
**[the CLI reference](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/cli.md)**
|
|
40
40
|
(generated from the commands; `mjolnir <command> --help` prints the same).
|
|
41
41
|
|
|
42
|
+
## Library use
|
|
43
|
+
|
|
44
|
+
Two modules are declared a public, stable API — reuse them instead of
|
|
45
|
+
re-implementing the clean-window gate or the bench-history substrate:
|
|
46
|
+
|
|
47
|
+
- **`mjolnir_cli.gate`** — `wait_for_idle`, `server_load`, `server_counters`,
|
|
48
|
+
`spec_delta`, the `CleanWindow` context manager, `PreflightError` and the
|
|
49
|
+
`DEFAULT_*` tuning constants.
|
|
50
|
+
- **`mjolnir_cli.history`** — `append_record`, `load_records` and
|
|
51
|
+
`SCHEMA_VERSION`; the history-row schema is documented in the module
|
|
52
|
+
docstring. Rows carry `schema_version` — absence means the legacy (pre-0.3)
|
|
53
|
+
format — and `load_records` returns rows raw, never normalized.
|
|
54
|
+
`append_record` files one record per serving key (platform, model, image,
|
|
55
|
+
config): a re-run merges its cells into the stored record instead of
|
|
56
|
+
appending a line; records without a serving key are appended verbatim.
|
|
57
|
+
|
|
58
|
+
Anything else can change without notice; the module docstrings are the API
|
|
59
|
+
reference: [`gate.py`](https://github.com/GoBrrr-ai/mjolnir/blob/main/src/mjolnir-cli/mjolnir_cli/gate.py),
|
|
60
|
+
[`history.py`](https://github.com/GoBrrr-ai/mjolnir/blob/main/src/mjolnir-cli/mjolnir_cli/history.py).
|
|
61
|
+
|
|
62
|
+
`uv tool install mjolnir-cli` is an isolated CLI install — *not* importable
|
|
63
|
+
from another project. Library use means adding `mjolnir-cli` as a dependency
|
|
64
|
+
of your project (`uv add mjolnir-cli` / `pip install mjolnir-cli`). The
|
|
65
|
+
gate/history import chain is stdlib-only: the CLI dependency stack (typer,
|
|
66
|
+
rich, pyyaml, jinja2) is paid at install time, not import time. Python ≥ 3.12
|
|
67
|
+
is required (pip refuses older interpreters — relevant on JetPack 6-era
|
|
68
|
+
images), and type annotations ship (`py.typed`).
|
|
69
|
+
|
|
42
70
|
## Requirements
|
|
43
71
|
|
|
44
72
|
A Jetson AGX Thor (JetPack 7.x) with Docker and the NVIDIA Container Toolkit
|
|
45
|
-
for anything that touches the GPU; Python ≥ 3.
|
|
73
|
+
for anything that touches the GPU; Python ≥ 3.12. Charts need the `plot` extra
|
|
46
74
|
(`uv tool install 'mjolnir-cli[plot]'`).
|
|
47
75
|
|
|
48
76
|
## Versions
|
|
49
77
|
|
|
50
78
|
`mjolnir-cli` and `mjolnir-kernel` version independently; the Docker image is
|
|
51
79
|
tagged after the upstream vLLM it is built on (`v0.31.0`). Release notes:
|
|
52
|
-
[GitHub Releases](https://github.com/
|
|
80
|
+
[GitHub Releases](https://github.com/GoBrrr-ai/mjolnir/releases).
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Run the ``mjolnir`` CLI via ``python -m mjolnir_cli`` — the same entry
|
|
2
|
+
point as the console script (``mjolnir_cli.cli:main``), UnknownPlatform
|
|
3
|
+
handler included."""
|
|
4
|
+
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
from mjolnir_cli.cli import main
|
|
8
|
+
|
|
9
|
+
if __name__ == "__main__":
|
|
10
|
+
sys.exit(main())
|
|
@@ -14,8 +14,9 @@ tool-eval-bench keeps its results in ``./data/benchmarks.sqlite`` and
|
|
|
14
14
|
under ``benchmarks/raw/accuracy/<run>/<leg>/``, so the database holds exactly
|
|
15
15
|
that leg's runs. :func:`summarize_db` reads it back into the committed
|
|
16
16
|
``accuracy.json`` (``mjolnir.accuracy/1``): per suite the score, per
|
|
17
|
-
tool-call category the share of points, per scenario the points
|
|
18
|
-
|
|
17
|
+
tool-call category the share of points, per scenario the points, and the ids
|
|
18
|
+
tool-eval-bench excluded from scoring (``excluded``) — what the docs table
|
|
19
|
+
and charts are rendered from.
|
|
19
20
|
"""
|
|
20
21
|
|
|
21
22
|
from __future__ import annotations
|
|
@@ -24,6 +25,7 @@ import json
|
|
|
24
25
|
import os
|
|
25
26
|
import shlex
|
|
26
27
|
import shutil
|
|
28
|
+
import platform as host_platform # the record's host field names the machine (platform.node)
|
|
27
29
|
import sqlite3
|
|
28
30
|
import subprocess
|
|
29
31
|
import sys
|
|
@@ -32,6 +34,7 @@ from pathlib import Path
|
|
|
32
34
|
from statistics import mean
|
|
33
35
|
|
|
34
36
|
from mjolnir_cli.config import by_model, model_of, model_short
|
|
37
|
+
from mjolnir_cli.platforms import platform_of
|
|
35
38
|
|
|
36
39
|
FORMAT = "mjolnir.accuracy/1"
|
|
37
40
|
TOOL_REPO = "https://github.com/SeraphimSerapis/tool-eval-bench"
|
|
@@ -96,7 +99,9 @@ def _pct(earned: float, mx: float) -> float | None:
|
|
|
96
99
|
def summarize_db(db: Path) -> dict:
|
|
97
100
|
"""tool-eval-bench's SQLite → ``{"suites": {suite: {...}}, "categories":
|
|
98
101
|
{...}, "scenarios": {...}}``. Several completed runs of a kind (``--trials``)
|
|
99
|
-
are averaged; each suite keeps its per-run scores.
|
|
102
|
+
are averaged; each suite keeps its per-run scores. A scenario the tool
|
|
103
|
+
excluded from scoring (an infrastructure failure: the probe failed, a
|
|
104
|
+
timeout) comes back as ``excluded``."""
|
|
100
105
|
if not db.exists():
|
|
101
106
|
return {"suites": {}, "categories": {}, "scenarios": {}, "runs": []}
|
|
102
107
|
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
|
@@ -110,6 +115,7 @@ def summarize_db(db: Path) -> dict:
|
|
|
110
115
|
per_suite: dict[str, list[dict]] = {}
|
|
111
116
|
cats: dict[str, dict] = {}
|
|
112
117
|
scen: dict[str, list[float]] = {}
|
|
118
|
+
excluded: set[str] = set()
|
|
113
119
|
runs = []
|
|
114
120
|
for run_id, created, status, run_type, scores_json, _config_json in rows:
|
|
115
121
|
runs.append({"run_id": run_id, "created_at": created, "status": status, "run_type": run_type})
|
|
@@ -117,6 +123,7 @@ def summarize_db(db: Path) -> dict:
|
|
|
117
123
|
continue
|
|
118
124
|
sc = _loads(scores_json)
|
|
119
125
|
if run_type == "tool_eval":
|
|
126
|
+
excluded |= set(sc.get("excluded_scenarios") or ()) # upstream lists them only when non-empty
|
|
120
127
|
cs = sc.get("category_scores") or []
|
|
121
128
|
std = [c for c in cs if c.get("category") != HARD_CATEGORY]
|
|
122
129
|
hard = [c for c in cs if c.get("category") == HARD_CATEGORY]
|
|
@@ -168,7 +175,7 @@ def summarize_db(db: Path) -> dict:
|
|
|
168
175
|
if comps:
|
|
169
176
|
last["completion_rate"] = min(comps) # the worst trial: an exclusion in any trial shows
|
|
170
177
|
suites[name] = {"score": round(mean(scores), 2), "scores": scores, "n": len(scores), **last}
|
|
171
|
-
|
|
178
|
+
out = {
|
|
172
179
|
"suites": suites,
|
|
173
180
|
"categories": {
|
|
174
181
|
k: {"label": v["label"], "percent": round(mean(v["percent"]), 2)}
|
|
@@ -178,6 +185,9 @@ def summarize_db(db: Path) -> dict:
|
|
|
178
185
|
"scenarios": {k: round(mean(v), 3) for k, v in sorted(scen.items())},
|
|
179
186
|
"runs": runs,
|
|
180
187
|
}
|
|
188
|
+
if excluded: # e.g. TC-45: the tool_choice=required probe failed — scored as zero on the full max
|
|
189
|
+
out["excluded"] = sorted(excluded)
|
|
190
|
+
return out
|
|
181
191
|
|
|
182
192
|
|
|
183
193
|
# ── the run ─────────────────────────────────────────────────────────────────
|
|
@@ -211,9 +221,7 @@ def leg_label(rec: dict) -> str:
|
|
|
211
221
|
if "MJOLNIR" not in cfg:
|
|
212
222
|
return f"Mjolnir image · {BACKEND_LABELS.get(cfg, cfg)} · {kv}"
|
|
213
223
|
extra = "".join(
|
|
214
|
-
f" + {name}"
|
|
215
|
-
for tag, name in (("LMH8", "fp8 lm_head"), ("VT32K", "32K draft vocab"), ("VT64K", "64K draft vocab"))
|
|
216
|
-
if tag in cfg.split("_")
|
|
224
|
+
f" + {name}" for tag, name in (("LMH8", "fp8 lm_head"), ("VT64K", "64K draft vocab")) if tag in cfg.split("_")
|
|
217
225
|
)
|
|
218
226
|
return f"Mjolnir · {kv}{extra}"
|
|
219
227
|
|
|
@@ -230,6 +238,7 @@ def run_accuracy(
|
|
|
230
238
|
tool-eval-bench against it, and write ``accuracy.json``. Returns its path."""
|
|
231
239
|
from mjolnir_cli import dockerctl
|
|
232
240
|
from mjolnir_cli.config import config_field
|
|
241
|
+
from mjolnir_cli.platforms import detect, gpu_name
|
|
233
242
|
|
|
234
243
|
tool = find_tool()
|
|
235
244
|
run_id = time.strftime("%Y%m%d-%H%M%S")
|
|
@@ -239,6 +248,10 @@ def run_accuracy(
|
|
|
239
248
|
"format": FORMAT,
|
|
240
249
|
"run_id": run_id,
|
|
241
250
|
"created": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
251
|
+
"host": host_platform.node() or "unknown",
|
|
252
|
+
"gpu": gpu_name(),
|
|
253
|
+
# the detection-first rule: the record names the *detected* platform
|
|
254
|
+
"platform": detect().platform,
|
|
242
255
|
"tool": {"repo": TOOL_REPO, "commit": TOOL_COMMIT, "command": tool},
|
|
243
256
|
"args": list(passthrough) or list(DEFAULT_ARGS),
|
|
244
257
|
"legs": [],
|
|
@@ -299,23 +312,33 @@ def load_runs(layout) -> list[dict]:
|
|
|
299
312
|
|
|
300
313
|
|
|
301
314
|
def latest_legs(docs: list[dict]) -> list[dict]:
|
|
302
|
-
"""One row per (model, image, config): every suite from the
|
|
303
|
-
that measured it (an A/B run and a later GSM8K run merge into
|
|
304
|
-
Order: first appearance.
|
|
315
|
+
"""One row per (platform, model, image side, config): every suite from the
|
|
316
|
+
newest run that measured it (an A/B run and a later GSM8K run merge into
|
|
317
|
+
one row). Order: first appearance. The platform comes from the run doc
|
|
318
|
+
(platform-less records are the repo's Thor history); a leg measured on
|
|
319
|
+
two platforms is two rows — its charts are per (platform, model). The
|
|
320
|
+
side (official image vs Mjolnir) keys the row, not the image tag, so a
|
|
321
|
+
re-measure on a newer image of the same side supersedes the older scores
|
|
322
|
+
in place (the row keeps its position and the newest image) instead of
|
|
323
|
+
duplicating a row the labels cannot tell apart."""
|
|
305
324
|
rows: dict[tuple, dict] = {}
|
|
306
325
|
for d in docs:
|
|
326
|
+
plat = platform_of(d)
|
|
307
327
|
for leg in d.get("legs", []):
|
|
308
|
-
key = (model_of(leg), leg.get("image"), leg.get("config"))
|
|
328
|
+
key = (plat, model_of(leg), _is_mjolnir(str(leg.get("image", ""))), leg.get("config"))
|
|
309
329
|
row = rows.setdefault(
|
|
310
330
|
key,
|
|
311
331
|
{k: leg.get(k) for k in ("id", "model", "image", "config", "label", "kv_cache_dtype", "env")}
|
|
312
|
-
| {"suites": {}, "categories": {}, "scenarios": {}, "runs": {}},
|
|
332
|
+
| {"platform": plat, "suites": {}, "categories": {}, "scenarios": {}, "runs": {}},
|
|
313
333
|
)
|
|
334
|
+
row["image"] = leg.get("image") or row.get("image")
|
|
314
335
|
row["kv_cache_dtype"] = leg.get("kv_cache_dtype") or row.get("kv_cache_dtype")
|
|
315
336
|
row["label"] = leg_label(row) # recomputed: labels improve after a run is stored
|
|
316
337
|
for name, suite in (leg.get("suites") or {}).items():
|
|
317
338
|
row["suites"][name] = suite
|
|
318
339
|
row["runs"][name] = {"run_id": d.get("run_id"), "args": d.get("args")}
|
|
340
|
+
if leg.get("excluded"): # an exclusion in any run of the leg shows
|
|
341
|
+
row["excluded"] = sorted({*(row.get("excluded") or ()), *leg["excluded"]})
|
|
319
342
|
if (leg.get("suites") or {}).keys() & {"tools", "hardmode"}:
|
|
320
343
|
row["categories"] = leg.get("categories") or row["categories"]
|
|
321
344
|
row["scenarios"] = leg.get("scenarios") or row["scenarios"]
|
|
@@ -4,10 +4,14 @@ After karpathy/autoresearch: an agent edits ONE file, runs a FIXED,
|
|
|
4
4
|
time-boxed evaluation, keeps the change if the metric improved and resets it
|
|
5
5
|
otherwise — forever, while every experiment lands on a ledger. Here:
|
|
6
6
|
|
|
7
|
-
* the editable
|
|
7
|
+
* the editable files — the kernel family for this board's platform
|
|
8
|
+
(``mjolnir arch/thor/*.py``: the traced decode kernel and its builder,
|
|
9
|
+
constants, plan and harness; ``arch/orin/*.py``: the mma kernel plus its
|
|
10
|
+
runner, plan, combine and harness — resolved from the active platform);
|
|
8
11
|
* the fixed harness — ``mjolnir autoresearch evaluate``: the quick
|
|
9
|
-
correctness subset (``mjolnir verify correctness --quick``) +
|
|
10
|
-
of FlashInfer vs Mjolnir
|
|
12
|
+
correctness subset (``mjolnir verify correctness --quick``) + a sweep
|
|
13
|
+
of FlashInfer vs Mjolnir at every L (ncu-timed on Thor, events-timed
|
|
14
|
+
on Orin — the platforms' headline sweep either way);
|
|
11
15
|
* the metric — per L, FlashInfer time / Mjolnir time, both measured
|
|
12
16
|
in the SAME run (drift on the shared Thor cancels out); the score is their
|
|
13
17
|
geomean over L;
|
|
@@ -17,7 +21,11 @@ otherwise — forever, while every experiment lands on a ledger. Here:
|
|
|
17
21
|
single L and no more than one L slower than the noise band; L where the
|
|
18
22
|
FlashInfer reference itself jumped are set aside; ``noisy`` when the
|
|
19
23
|
reference was unstable at too many L (re-evaluate the same commit),
|
|
20
|
-
``crash`` if a stage failed to produce data.
|
|
24
|
+
``crash`` if a stage failed to produce data. A structured sub-bar gain
|
|
25
|
+
(over half the bar, same shape) is ``parked``: the commit is tagged
|
|
26
|
+
``ar/parked/<sha>`` and survives the reset as stackable material — bricks
|
|
27
|
+
too small to certify alone clear the bar in batches — but a parked row is
|
|
28
|
+
never the reference.
|
|
21
29
|
* the ledger — ``research/ledger.jsonl`` (+ a ``ledger.tsv`` view):
|
|
22
30
|
one row per experiment with the hypothesis, the agents involved, the
|
|
23
31
|
sources (URLs / arXiv ids), the commit, the per-L numbers and the verify
|
|
@@ -29,7 +37,8 @@ The loop itself is an OpenCode session (``mjolnir autoresearch run``)
|
|
|
29
37
|
driving ``.opencode/agent/autoresearch.md`` (the orchestrator; the helpers it delegates to are user-defined OpenCode agents)
|
|
30
38
|
→ ``web-researcher`` / ``arxiv-researcher`` (ideas + sources) →
|
|
31
39
|
``python-engineer`` (the change) → ``debugger`` (crashes, correctness
|
|
32
|
-
failures). The program they follow is ``research/program.md
|
|
40
|
+
failures). The program they follow is ``research/program.md`` —
|
|
41
|
+
``research/program-orin.md`` on an Orin (resolved by the active platform).
|
|
33
42
|
"""
|
|
34
43
|
|
|
35
44
|
from __future__ import annotations
|
|
@@ -57,7 +66,33 @@ app = typer.Typer(
|
|
|
57
66
|
context_settings={"help_option_names": ["-h", "--help"]},
|
|
58
67
|
)
|
|
59
68
|
|
|
60
|
-
|
|
69
|
+
KERNEL_DIR = Path("src/mjolnir/mjolnir/arch/thor")
|
|
70
|
+
ORIN_KERNEL_DIR = Path("src/mjolnir/mjolnir/arch/orin")
|
|
71
|
+
# the kernel family an experiment may change — per family (the loop runs on
|
|
72
|
+
# the board it improves; resolved from the active platform, never a silent
|
|
73
|
+
# default). Thor: the traced decode kernel plus its builder, constants, plan
|
|
74
|
+
# and standalone harness. Orin: the mma kernel plus its runner, plan, combine
|
|
75
|
+
# and standalone harness.
|
|
76
|
+
THOR_FILES = tuple(KERNEL_DIR / f for f in ("decode.py", "call.py", "const.py", "plan.py", "harness.py"))
|
|
77
|
+
ORIN_FILES = tuple(ORIN_KERNEL_DIR / f for f in ("mma.py", "runner.py", "plan.py", "combine.py", "harness.py"))
|
|
78
|
+
# the loop's brief, per platform (what the agent reads; `run` points it there)
|
|
79
|
+
PROGRAM = {"thor": Path("research/program.md"), "orin": Path("research/program-orin.md")}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def kernel_files() -> tuple[Path, ...]:
|
|
83
|
+
"""The kernel family this board's loop may change."""
|
|
84
|
+
from mjolnir_cli.platforms import resolve_platform
|
|
85
|
+
|
|
86
|
+
return ORIN_FILES if resolve_platform(None) == "orin" else THOR_FILES
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def program_file() -> Path:
|
|
90
|
+
"""The program brief for this board's platform."""
|
|
91
|
+
from mjolnir_cli.platforms import resolve_platform
|
|
92
|
+
|
|
93
|
+
return PROGRAM.get(resolve_platform(None), PROGRAM["thor"])
|
|
94
|
+
|
|
95
|
+
|
|
61
96
|
LEDGER_DIR = Path("research")
|
|
62
97
|
TSV_COLUMNS = ["id", "ts", "commit", "status", "score", "best_before", "correctness", "agent", "hypothesis", "reason"]
|
|
63
98
|
|
|
@@ -143,6 +178,16 @@ def judge(row: dict, best: dict | None, min_gain: float = 0.02, noise: float = N
|
|
|
143
178
|
3. at most one L is slower than ``1 - noise`` (one is tolerated: 16K is
|
|
144
179
|
bimodal run to run).
|
|
145
180
|
|
|
181
|
+
A *parked* row is a sub-bar gain that still has the shape of a win:
|
|
182
|
+
geomean over ``1 + min_gain / 2``, leave-one-out over ``1 + min_gain / 4``,
|
|
183
|
+
at most one L slower than the noise band. ``evaluate`` tags the commit
|
|
184
|
+
``ar/parked/<sha>`` so it survives the reset; the orchestrator stacks
|
|
185
|
+
parked bricks — alone or mixed — and evaluates each batch as one row, and
|
|
186
|
+
bricks too small to certify alone clear the bar together. A gain resting
|
|
187
|
+
on one L, or losing at several, is a gating candidate, not a brick: plain
|
|
188
|
+
``discard``. A parked row never becomes the reference
|
|
189
|
+
(:func:`best_kept` ignores it).
|
|
190
|
+
|
|
146
191
|
An L is *suspect* — set aside — when the FlashInfer reference time moved
|
|
147
192
|
by more than ``REF_JUMP`` against the best row's run (the reference never
|
|
148
193
|
changes between experiments, so a jump there is noise in the measurement,
|
|
@@ -170,7 +215,37 @@ def judge(row: dict, best: dict | None, min_gain: float = 0.02, noise: float = N
|
|
|
170
215
|
note = f" (set aside, reference jumped: {', '.join(_L(L) for L in suspect)})" if suspect else ""
|
|
171
216
|
gain = _gm(trusted.values())
|
|
172
217
|
if gain <= 1 + min_gain:
|
|
173
|
-
|
|
218
|
+
if gain <= 1 + min_gain / 2:
|
|
219
|
+
return "discard", f"geomean {_pct(gain)} ≤ +{min_gain * 100:g}%{note}", rel
|
|
220
|
+
# Parked band (half the keep bars): a real-looking small win stays
|
|
221
|
+
# stackable instead of dying on the reset.
|
|
222
|
+
top = max(trusted, key=lambda L: trusted[L])
|
|
223
|
+
loo = _gm(v for L, v in trusted.items() if top != L)
|
|
224
|
+
slower = sorted((L for L, v in trusted.items() if v < 1 - noise), key=int)
|
|
225
|
+
if len(slower) > 1: # a gating candidate, not a brick: name the losses
|
|
226
|
+
return (
|
|
227
|
+
"discard",
|
|
228
|
+
f"geomean {_pct(gain)} under the bar but slower at "
|
|
229
|
+
+ ", ".join(f"{_L(L)} ({_pct(trusted[L])})" for L in slower)
|
|
230
|
+
+ note,
|
|
231
|
+
rel,
|
|
232
|
+
)
|
|
233
|
+
if loo <= 1 + min_gain / 4:
|
|
234
|
+
return (
|
|
235
|
+
"discard",
|
|
236
|
+
f"geomean {_pct(gain)} under the bar and rests on {_L(top)} ({_pct(trusted[top])}): "
|
|
237
|
+
f"{_pct(loo)} without it{note}",
|
|
238
|
+
rel,
|
|
239
|
+
)
|
|
240
|
+
shape = f", {_pct(loo)} without its best L ({_L(top)})"
|
|
241
|
+
if slower:
|
|
242
|
+
shape += ", slower at " + ", ".join(_L(L) for L in slower)
|
|
243
|
+
return (
|
|
244
|
+
"parked",
|
|
245
|
+
f"geomean {_pct(gain)} under the +{min_gain * 100:g}% bar but structured{shape} — "
|
|
246
|
+
f"parked for stacking{note}",
|
|
247
|
+
rel,
|
|
248
|
+
)
|
|
174
249
|
top = max(trusted, key=lambda L: trusted[L])
|
|
175
250
|
loo = _gm(v for L, v in trusted.items() if top != L)
|
|
176
251
|
if loo <= 1 + min_gain / 2:
|
|
@@ -211,8 +286,13 @@ def _init(root: Path, tag: str, base: str, evaluate_baseline: bool, image: str |
|
|
|
211
286
|
if _git(root, "branch", "--list", branch):
|
|
212
287
|
console.print(f"[red]{branch} already exists — pick a fresh tag (a run never reuses a branch)[/red]")
|
|
213
288
|
return 4
|
|
214
|
-
|
|
215
|
-
|
|
289
|
+
kfiles = kernel_files()
|
|
290
|
+
if _git(root, "status", "--porcelain", "--", *map(str, kfiles)):
|
|
291
|
+
console.print(
|
|
292
|
+
"[red]the kernel family has uncommitted changes ("
|
|
293
|
+
+ ", ".join(p.name for p in kfiles)
|
|
294
|
+
+ ") — commit or stash them first[/red]"
|
|
295
|
+
)
|
|
216
296
|
return 4
|
|
217
297
|
if load_ledger(root):
|
|
218
298
|
console.print(
|
|
@@ -255,7 +335,8 @@ def _evaluate(
|
|
|
255
335
|
rows = load_ledger(root)
|
|
256
336
|
best = best_kept(rows)
|
|
257
337
|
commit = _git(root, "rev-parse", "--short", "HEAD")
|
|
258
|
-
|
|
338
|
+
kfiles = kernel_files()
|
|
339
|
+
dirty = bool(_git(root, "status", "--porcelain", "--", *map(str, kfiles)))
|
|
259
340
|
if dirty:
|
|
260
341
|
console.print(
|
|
261
342
|
"[yellow]the kernel has uncommitted changes — commit the experiment first "
|
|
@@ -340,34 +421,52 @@ def _evaluate(
|
|
|
340
421
|
" vs best, per L: "
|
|
341
422
|
+ " ".join(f"{_L(L)} {_pct(v)}" for L, v in sorted(rel.items(), key=lambda kv: int(kv[0])))
|
|
342
423
|
)
|
|
343
|
-
if status in ("discard", "crash"):
|
|
424
|
+
if status in ("discard", "crash", "parked"):
|
|
344
425
|
console.print(f"[dim] revert: git reset --hard {best['commit'] if best else 'HEAD~1'}[/dim]")
|
|
426
|
+
if status == "parked":
|
|
427
|
+
tag = f"ar/parked/{commit}"
|
|
428
|
+
try:
|
|
429
|
+
if not _git(root, "tag", "--list", tag):
|
|
430
|
+
_git(root, "tag", "-a", tag, "-m", hypothesis, commit)
|
|
431
|
+
console.print(
|
|
432
|
+
f"[dim] parked: the brick survives as tag {tag} — cherry-pick it, alone or mixed with "
|
|
433
|
+
"other parked tags, onto the best commit and evaluate the batch as one row "
|
|
434
|
+
"(a parked row is never the reference)[/dim]"
|
|
435
|
+
)
|
|
436
|
+
except RuntimeError as exc:
|
|
437
|
+
console.print(f"[dim] parked (tagging failed: {exc!r}) — the ledger row carries the commit[/dim]")
|
|
345
438
|
elif status == "noisy":
|
|
346
439
|
console.print("[dim] the measurement was rejected, not the idea: evaluate the same commit again[/dim]")
|
|
347
|
-
|
|
440
|
+
render_progress(root)
|
|
348
441
|
return 0
|
|
349
442
|
|
|
350
443
|
|
|
351
|
-
STATUS_COLORS = {
|
|
444
|
+
STATUS_COLORS = {
|
|
445
|
+
"keep": "green",
|
|
446
|
+
"baseline": "cyan",
|
|
447
|
+
"parked": "blue",
|
|
448
|
+
"discard": "yellow",
|
|
449
|
+
"crash": "red",
|
|
450
|
+
"noisy": "magenta",
|
|
451
|
+
}
|
|
352
452
|
PROGRESS = LEDGER_DIR / "progress.png"
|
|
353
453
|
|
|
354
454
|
|
|
355
|
-
def
|
|
356
|
-
"""research/progress.png
|
|
357
|
-
|
|
358
|
-
|
|
455
|
+
def render_progress(root: Path, rows: list[dict] | None = None, out: Path | None = None) -> Path | None:
|
|
456
|
+
"""research/progress.png: every experiment on one chart, the kept
|
|
457
|
+
staircase, and the per-L split vs the baseline. Best effort — a missing
|
|
458
|
+
matplotlib never fails an evaluation."""
|
|
359
459
|
rows = rows if rows is not None else load_ledger(root)
|
|
360
460
|
try:
|
|
361
461
|
from mjolnir_cli import charts, theme
|
|
362
462
|
|
|
363
463
|
plt = charts._plt()
|
|
364
464
|
out = out or root / PROGRESS
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
plt.close(fig)
|
|
465
|
+
fig = charts.chart_autoresearch(rows, theme.DARK)
|
|
466
|
+
if fig is None:
|
|
467
|
+
return None
|
|
468
|
+
charts._save(fig, out)
|
|
469
|
+
plt.close(fig)
|
|
371
470
|
return out
|
|
372
471
|
except Exception as e: # noqa: BLE001 — a chart must never cost a ledger row
|
|
373
472
|
console.print(f"[dim]progress chart skipped: {e!r}[/dim]")
|
|
@@ -395,7 +494,7 @@ def evaluate(
|
|
|
395
494
|
image: str | None = typer.Option(None, "--image"),
|
|
396
495
|
):
|
|
397
496
|
"""Score the committed kernel: quick correctness + FlashInfer vs Mjolnir sweep;
|
|
398
|
-
append the result and the fixed keep/discard decision to the ledger."""
|
|
497
|
+
append the result and the fixed keep/park/discard decision to the ledger."""
|
|
399
498
|
raise typer.Exit(_evaluate(_root(), hypothesis, agent, sources, image, min_gain, ls, iters, gate))
|
|
400
499
|
|
|
401
500
|
|
|
@@ -457,18 +556,21 @@ def status():
|
|
|
457
556
|
def chart():
|
|
458
557
|
"""Re-render research/progress.png from the live ledger (or the newest exported run)."""
|
|
459
558
|
root = _root()
|
|
460
|
-
p =
|
|
559
|
+
p = render_progress(root, latest_rows(root))
|
|
461
560
|
console.print(f"[green]{p}[/green]" if p else "[yellow]no experiments yet[/yellow]")
|
|
462
561
|
|
|
463
562
|
|
|
464
563
|
@app.command()
|
|
465
564
|
def run(
|
|
466
565
|
tag: str | None = typer.Option(None, "--tag", help="start a new run first (init <tag>)"),
|
|
566
|
+
base: str = typer.Option(
|
|
567
|
+
None, "--base", help="branch to start from (default: main; the Orin's kernel lives on feat/orin)"
|
|
568
|
+
),
|
|
467
569
|
model: str | None = typer.Option(None, "--model", help="OpenCode model (provider/model) for the orchestrator"),
|
|
468
570
|
prompt: str = typer.Option(
|
|
469
|
-
|
|
470
|
-
"Never stop to ask; record every experiment with `mjolnir autoresearch evaluate`.",
|
|
571
|
+
None,
|
|
471
572
|
"--prompt",
|
|
573
|
+
help="the orchestrator's brief (default: read this platform's program and run the loop)",
|
|
472
574
|
),
|
|
473
575
|
):
|
|
474
576
|
"""Launch the loop: an OpenCode session with the `autoresearch` agent."""
|
|
@@ -476,8 +578,13 @@ def run(
|
|
|
476
578
|
if shutil.which("opencode") is None:
|
|
477
579
|
console.print("[red]opencode not found — install it: https://opencode.ai (npm i -g opencode-ai)[/red]")
|
|
478
580
|
raise typer.Exit(2)
|
|
581
|
+
if prompt is None:
|
|
582
|
+
prompt = (
|
|
583
|
+
f"Read {program_file()} and run the loop: setup is done when the ledger has a baseline row. "
|
|
584
|
+
"Never stop to ask; record every experiment with `mjolnir autoresearch evaluate`."
|
|
585
|
+
)
|
|
479
586
|
if tag:
|
|
480
|
-
rc = _init(root, tag, "main", True, None)
|
|
587
|
+
rc = _init(root, tag, base or ("feat/orin" if program_file() == PROGRAM["orin"] else "main"), True, None)
|
|
481
588
|
if rc != 0:
|
|
482
589
|
raise typer.Exit(rc)
|
|
483
590
|
elif not load_ledger(root):
|
|
@@ -522,7 +629,7 @@ def export(
|
|
|
522
629
|
f"- baseline: {base['score']}× vs FlashInfer at `{base['commit']}`" if base and base.get("score") else "",
|
|
523
630
|
f"- best: **{best['score']}×** at `{best['commit']}` — {best['hypothesis']}" if best else "",
|
|
524
631
|
"",
|
|
525
|
-
"" if
|
|
632
|
+
"" if render_progress(root, rows, out / "progress.png") else "",
|
|
526
633
|
"",
|
|
527
634
|
"| # | commit | status | score | why | agent | hypothesis | sources |",
|
|
528
635
|
"|--:|---|---|--:|---|---|---|---|",
|