ttsem 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ttsem-0.1.0 → ttsem-0.2.0}/CITATION.cff +2 -2
- ttsem-0.1.0/README.md → ttsem-0.2.0/PKG-INFO +69 -6
- ttsem-0.1.0/PKG-INFO → ttsem-0.2.0/README.md +34 -39
- {ttsem-0.1.0 → ttsem-0.2.0}/audits/tritonbench/device.py +2 -1
- {ttsem-0.1.0 → ttsem-0.2.0}/docs/JUDGE.md +1 -1
- ttsem-0.2.0/docs/REWARD.md +358 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/pyproject.toml +6 -2
- ttsem-0.2.0/tests/fixtures/reward/kb_shift_answer.py +30 -0
- ttsem-0.2.0/tests/fixtures/reward/kb_shift_task.py +26 -0
- ttsem-0.2.0/tests/fixtures/reward/shift_right_answer.py +22 -0
- ttsem-0.2.0/tests/fixtures/reward/shift_right_task.py +21 -0
- ttsem-0.2.0/tests/fixtures/triton39_cache_policy.py +43 -0
- ttsem-0.2.0/tests/fixtures/triton39_cache_policy.ttir.generic +25 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_audits.py +2 -1
- ttsem-0.2.0/tests/test_compat.py +38 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_judge.py +6 -2
- ttsem-0.2.0/tests/test_reward.py +635 -0
- ttsem-0.2.0/tests/test_reward_cap.py +169 -0
- ttsem-0.2.0/tests/test_reward_hygiene.py +67 -0
- ttsem-0.2.0/tests/test_reward_integration.py +187 -0
- ttsem-0.2.0/tests/test_reward_verl.py +81 -0
- ttsem-0.2.0/tests/test_triton39.py +73 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/__init__.py +1 -1
- ttsem-0.2.0/ttsem/_compat.py +51 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/_judge_runner.py +4 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/interp.py +2 -1
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/interp2.py +2 -1
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/judge.py +7 -2
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/judge_tritonbench.py +12 -2
- ttsem-0.2.0/ttsem/reward/__init__.py +15 -0
- ttsem-0.2.0/ttsem/reward/cap.py +509 -0
- ttsem-0.2.0/ttsem/reward/core.py +766 -0
- ttsem-0.2.0/ttsem/reward/verl_adapter.py +135 -0
- ttsem-0.2.0/ttsem/reward/worker.py +263 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/.gitignore +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/CONTRIBUTING.md +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/LEDGER.md +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/LICENSE +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/audits/kernelbench/README.md +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/audits/kernelbench/prepare.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/audits/kernelbench/table.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/audits/tritonbench/README.md +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/audits/tritonbench/agree.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/audits/tritonbench/device_runner.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/audits/tritonbench/manifest.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/audits/tritonbench/prepare.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/audits/tritonbench/table.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/docs/AGENTS.md +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/docs/DESIGN.md +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/docs/INDUCTOR.md +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/docs/OVERVIEW.md +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/docs/RESULTS.md +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/docs/SEMANTICS.md +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/examples/ci/test_scale.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/examples/e11601_min.mlir +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/examples/e11601_repro.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/examples/e11612_kernel_min.mlir +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/examples/e11612_repro_min.mlir +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/examples/e15_11519_main_e1f944a7a.json +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/examples/e15_11519_min.mlir +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/examples/e15_repro.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/examples/e2_repro.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/examples/inductor_models.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/examples/inductor_models2.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/examples/witness_e8m0_nan.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/examples/witness_interp_bf16_add.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/conftest.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/agents/accumulate_into_empty.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/agents/add_no_load_mask.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/agents/add_no_store_mask.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/agents/add_ok.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/agents/cuda_script.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/agents/suite_big_launch.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/agents/suite_big_tiled.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/agents/suite_no_mask.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/gluon_copy.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/inductor/channel_share_output_code.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/inductor/safe_share_output_code.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/inductor/split_scan_output_code.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/inductor/user_kernel_clone_removed_output_code.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_config_asserts.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_config_faults.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_config_pruned.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_forges.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_lookback_helper.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_nested_grid.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_no_config_fits.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_no_kernel.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_ok.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_patch_restore.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_prewrites.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_protocol_big.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_protocol_marker.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_race_unchecked.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_swallows.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_tampers.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_thread_flip.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_unsafe.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/answer_wrong.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/task_matmul.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/judge/task_relu.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/membar_11325.generic +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p1.npy +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p1.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p1.ttgir +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p1.ttgir.generic +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p1.ttir +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p1.ttir.generic +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p2.npy +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p2.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p2.ttgir +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p2.ttgir.generic +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p2.ttir +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p2.ttir.generic +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p3.npy +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p3.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p3.ttgir +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p3.ttgir.generic +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p3.ttir +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p3.ttir.generic +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p4.npy +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p4.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p4.ttgir +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p4.ttgir.generic +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p4.ttir +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/p4.ttir.generic +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_cheats.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_no_kernel.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_ok.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_peeks.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_plants.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_prewrites.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_protocol_marker.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_race_unchecked.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_raises.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_swallows.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_unsafe.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/answer_wrong.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/tritonbench/task_relu.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/fixtures/ws_desc.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_bot.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_demo.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_gluon_compile.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_harness.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_interp.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_judge_integrity.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_judge_tritonbench.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_layout_checks.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_layouts.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_lost_updates.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_minimize.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_mlir.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_nvws.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_ops.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_parser_scope.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_pidraces.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_plugin_warnings.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_races.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_sanitize.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_scaled_dot.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_scheduler.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_torch_demo.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_torch_demo_ledger.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_validate.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/tests/test_validate_corpus.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/__main__.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/_compile_runner.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/_fakedriver.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/_guard.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/_install.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/_interp_runner.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/_tritonbench_runner.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/bot.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/bot.sh +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/campaign_target.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/check_fixtures.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/defuse.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/demo.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/dump_corpus.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/harness.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/inductor_trace.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/ir_types.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/judge_io.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/judge_shapes.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/layout_checks.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/layouts.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/linear_layout.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/memory.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/minimize.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/mlir.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/ops.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/pass_stress.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/pidraces.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/py.typed +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/pytest_ttsem.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/races.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/races_corpus.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/sanitize.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/torch_demo.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/validate.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/validate_corpus.py +0 -0
- {ttsem-0.1.0 → ttsem-0.2.0}/ttsem/values.py +0 -0
|
@@ -5,8 +5,8 @@ title: "ttsem: an executable semantics of Triton's TTIR and TTGIR, with a per-pa
|
|
|
5
5
|
authors:
|
|
6
6
|
- family-names: Potenza
|
|
7
7
|
given-names: Alessandro
|
|
8
|
-
version: 0.
|
|
9
|
-
# date-released is set when 0.
|
|
8
|
+
version: 0.2.0
|
|
9
|
+
# date-released is set when 0.2.0 is tagged and published
|
|
10
10
|
license: MIT
|
|
11
11
|
repository-code: "https://github.com/alepot55/ttsem"
|
|
12
12
|
url: "https://github.com/alepot55/ttsem"
|
|
@@ -1,3 +1,38 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: ttsem
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Run Triton kernel tests without a GPU: out-of-bounds accesses and races at the kernel's line
|
|
5
|
+
Project-URL: Homepage, https://github.com/alepot55/ttsem
|
|
6
|
+
Project-URL: Source, https://github.com/alepot55/ttsem
|
|
7
|
+
Project-URL: Issues, https://github.com/alepot55/ttsem/issues
|
|
8
|
+
Author: Alessandro Potenza
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: compiler,gpu,interpreter,mlir,semantics,triton
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Software Development :: Compilers
|
|
23
|
+
Classifier: Topic :: Software Development :: Testing
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.10
|
|
26
|
+
Requires-Dist: numpy
|
|
27
|
+
Provides-Extra: test
|
|
28
|
+
Requires-Dist: pytest; extra == 'test'
|
|
29
|
+
Requires-Dist: pytest-xdist; extra == 'test'
|
|
30
|
+
Provides-Extra: triton
|
|
31
|
+
Requires-Dist: pytest; extra == 'triton'
|
|
32
|
+
Requires-Dist: torch==2.14.0; extra == 'triton'
|
|
33
|
+
Requires-Dist: triton==3.8.0; extra == 'triton'
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
|
|
1
36
|
# ttsem
|
|
2
37
|
|
|
3
38
|
[](https://github.com/alepot55/ttsem/actions/workflows/ci.yml)
|
|
@@ -11,12 +46,13 @@ the CPU.
|
|
|
11
46
|
|
|
12
47
|
```console
|
|
13
48
|
$ pip install --index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.org/simple \
|
|
14
|
-
'ttsem[triton]
|
|
49
|
+
'ttsem[triton]'
|
|
15
50
|
$ python -m ttsem.sanitize pytest tests/ -q # your suite, written for device="cuda"
|
|
16
51
|
```
|
|
17
52
|
|
|
18
|
-
The first command installs ttsem with Triton 3.8.0 and the CPU build of torch 2.14.0
|
|
19
|
-
Python 3.
|
|
53
|
+
The first command installs ttsem from PyPI with Triton 3.8.0 and the CPU build of torch 2.14.0
|
|
54
|
+
(Linux, Python 3.10 to 3.13); `pip install ttsem` alone, numpy its only dependency, runs
|
|
55
|
+
`python -m ttsem demo`. The
|
|
20
56
|
second runs a pytest suite written for `device="cuda"`, unchanged. On [suite_no_mask.py][nomask], a
|
|
21
57
|
test file whose kernel forgets the mask on its store, the size that is a multiple of the block
|
|
22
58
|
passes and the other one fails at line 15 (from a clone; the last lines of the output):
|
|
@@ -31,10 +67,10 @@ FAILED tests/fixtures/agents/suite_no_mask.py::test_add[1000] - KernelFault: ...
|
|
|
31
67
|
```
|
|
32
68
|
|
|
33
69
|
`python -m ttsem.sanitize script.py` does the same for a script. In CI, the action runs your suite
|
|
34
|
-
this way on a CPU runner
|
|
70
|
+
this way on a CPU runner:
|
|
35
71
|
|
|
36
72
|
```yaml
|
|
37
|
-
- uses: alepot55/ttsem@
|
|
73
|
+
- uses: alepot55/ttsem@v0.2.0
|
|
38
74
|
with:
|
|
39
75
|
install: pip install -e . # your project
|
|
40
76
|
args: tests/kernels -q # pytest arguments
|
|
@@ -273,11 +309,36 @@ right and holds against an adversarial answer, since the comparison rests on a r
|
|
|
273
309
|
cannot touch (`verified` means right values; a kernel launched with the work then done in PyTorch
|
|
274
310
|
still verifies). `unsafe` and `no_kernel` rest on two stop-gaps, a source scan and a run-time
|
|
275
311
|
integrity check of the whole checker, that block every attack in the test suite but are not a
|
|
276
|
-
proof: until the answer and the checker run in separate processes (planned
|
|
312
|
+
proof: until the answer and the checker run in separate processes (planned, not in 0.2), use the judge
|
|
277
313
|
as an auditor and an evaluation tool, and as an RL reward against adversarial optimisation with that
|
|
278
314
|
caveat. The rules, the batch mode, the classes, the sandbox and what each verdict guarantees are in
|
|
279
315
|
[docs/JUDGE.md][judge].
|
|
280
316
|
|
|
317
|
+
## Reward
|
|
318
|
+
|
|
319
|
+
New in 0.2, prototype-grade: `ttsem.reward` turns the judge's verdict into a reward for training
|
|
320
|
+
models that write Triton kernels, with no GPU in the reward loop. `Judge.score(reference, answer)`
|
|
321
|
+
returns a verdict: a score of 1.0, 0.0 or none (an abstention), and a paragraph for the model that
|
|
322
|
+
names the kernel, the buffer and the line. In gate mode a reward you already have is kept for a
|
|
323
|
+
verified answer and lowered to the fail value for a rejected one, never raised; in score mode the
|
|
324
|
+
score is the reward. The trainer's process imports neither torch nor Triton: the passes run
|
|
325
|
+
in a worker in the judge's own venv, every answer and every reference in its own `bwrap` sandbox,
|
|
326
|
+
and nothing runs without one. A verl adapter is included, tested against a fake worker and not yet
|
|
327
|
+
against a verl release.
|
|
328
|
+
|
|
329
|
+
```python
|
|
330
|
+
from ttsem.reward import Judge, extract_answer
|
|
331
|
+
|
|
332
|
+
judge = Judge(cache_dir=cache, python=judge_venv_python, mode="gate")
|
|
333
|
+
verdict = judge.score(reference_src, extract_answer(completion))
|
|
334
|
+
reward = judge.gate(gpu_reward, verdict)
|
|
335
|
+
```
|
|
336
|
+
|
|
337
|
+
It carries the judge's own caveat (`unsafe` and `no_kernel` are not proofs against an optimiser free
|
|
338
|
+
to write anything), prices no speed, and abstains where it cannot say whose fault an outcome is, which
|
|
339
|
+
an answer can sometimes provoke. What it does and does not catch, its API, the adapter and how the
|
|
340
|
+
worker runs are in [docs/REWARD.md][reward].
|
|
341
|
+
|
|
281
342
|
## How it was built
|
|
282
343
|
|
|
283
344
|
Built by Alessandro Potenza with Claude Code (Anthropic's coding agent) as the engineering agent
|
|
@@ -312,6 +373,7 @@ after the system was found to exploit its evaluation harness ([TechCrunch][sakan
|
|
|
312
373
|
| [docs/DESIGN.md][design] | the three levels, the interfaces between them, and why they are drawn there |
|
|
313
374
|
| [docs/RESULTS.md][results] | the corpora, the counts, and the calibration of the semantics against the device |
|
|
314
375
|
| [docs/JUDGE.md][judge] | the judge of model-written kernels: rules, verdicts, batches, the sandbox |
|
|
376
|
+
| [docs/REWARD.md][reward] | the judge as an RL reward: gate and score mode, abstentions, the API, the verl adapter, the worker, what it does not catch |
|
|
315
377
|
| [docs/INDUCTOR.md][inductor] | the Triton code of `torch.compile` under ttsem |
|
|
316
378
|
| [docs/AGENTS.md][agents] | for coding agents that write Triton kernels |
|
|
317
379
|
| [audits/](https://github.com/alepot55/ttsem/tree/main/audits) | the commands that redo the TritonBench and KernelBench audits from a clone |
|
|
@@ -326,6 +388,7 @@ this repository with the metadata in `CITATION.cff` and the commit you used. MIT
|
|
|
326
388
|
[agents]: https://github.com/alepot55/ttsem/blob/main/docs/AGENTS.md
|
|
327
389
|
[skill]: https://github.com/alepot55/ttsem/blob/main/skills/triton-verify/SKILL.md
|
|
328
390
|
[judge]: https://github.com/alepot55/ttsem/blob/main/docs/JUDGE.md
|
|
391
|
+
[reward]: https://github.com/alepot55/ttsem/blob/main/docs/REWARD.md
|
|
329
392
|
[ledger]: https://github.com/alepot55/ttsem/blob/main/LEDGER.md
|
|
330
393
|
[overview]: https://github.com/alepot55/ttsem/blob/main/docs/OVERVIEW.md
|
|
331
394
|
[semantics]: https://github.com/alepot55/ttsem/blob/main/docs/SEMANTICS.md
|
|
@@ -1,36 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.5
|
|
2
|
-
Name: ttsem
|
|
3
|
-
Version: 0.1.0
|
|
4
|
-
Summary: Run Triton kernel tests without a GPU: out-of-bounds accesses and races at the kernel's line
|
|
5
|
-
Project-URL: Homepage, https://github.com/alepot55/ttsem
|
|
6
|
-
Project-URL: Source, https://github.com/alepot55/ttsem
|
|
7
|
-
Project-URL: Issues, https://github.com/alepot55/ttsem/issues
|
|
8
|
-
Author: Alessandro Potenza
|
|
9
|
-
License-Expression: MIT
|
|
10
|
-
License-File: LICENSE
|
|
11
|
-
Keywords: compiler,gpu,interpreter,mlir,semantics,triton
|
|
12
|
-
Classifier: Development Status :: 3 - Alpha
|
|
13
|
-
Classifier: Intended Audience :: Developers
|
|
14
|
-
Classifier: Intended Audience :: Science/Research
|
|
15
|
-
Classifier: Operating System :: POSIX :: Linux
|
|
16
|
-
Classifier: Programming Language :: Python :: 3
|
|
17
|
-
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
-
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
-
Classifier: Topic :: Software Development :: Compilers
|
|
21
|
-
Classifier: Topic :: Software Development :: Testing
|
|
22
|
-
Classifier: Typing :: Typed
|
|
23
|
-
Requires-Python: >=3.12
|
|
24
|
-
Requires-Dist: numpy
|
|
25
|
-
Provides-Extra: test
|
|
26
|
-
Requires-Dist: pytest; extra == 'test'
|
|
27
|
-
Requires-Dist: pytest-xdist; extra == 'test'
|
|
28
|
-
Provides-Extra: triton
|
|
29
|
-
Requires-Dist: pytest; extra == 'triton'
|
|
30
|
-
Requires-Dist: torch==2.14.0; extra == 'triton'
|
|
31
|
-
Requires-Dist: triton==3.8.0; extra == 'triton'
|
|
32
|
-
Description-Content-Type: text/markdown
|
|
33
|
-
|
|
34
1
|
# ttsem
|
|
35
2
|
|
|
36
3
|
[](https://github.com/alepot55/ttsem/actions/workflows/ci.yml)
|
|
@@ -44,12 +11,13 @@ the CPU.
|
|
|
44
11
|
|
|
45
12
|
```console
|
|
46
13
|
$ pip install --index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.org/simple \
|
|
47
|
-
'ttsem[triton]
|
|
14
|
+
'ttsem[triton]'
|
|
48
15
|
$ python -m ttsem.sanitize pytest tests/ -q # your suite, written for device="cuda"
|
|
49
16
|
```
|
|
50
17
|
|
|
51
|
-
The first command installs ttsem with Triton 3.8.0 and the CPU build of torch 2.14.0
|
|
52
|
-
Python 3.
|
|
18
|
+
The first command installs ttsem from PyPI with Triton 3.8.0 and the CPU build of torch 2.14.0
|
|
19
|
+
(Linux, Python 3.10 to 3.13); `pip install ttsem` alone, numpy its only dependency, runs
|
|
20
|
+
`python -m ttsem demo`. The
|
|
53
21
|
second runs a pytest suite written for `device="cuda"`, unchanged. On [suite_no_mask.py][nomask], a
|
|
54
22
|
test file whose kernel forgets the mask on its store, the size that is a multiple of the block
|
|
55
23
|
passes and the other one fails at line 15 (from a clone; the last lines of the output):
|
|
@@ -64,10 +32,10 @@ FAILED tests/fixtures/agents/suite_no_mask.py::test_add[1000] - KernelFault: ...
|
|
|
64
32
|
```
|
|
65
33
|
|
|
66
34
|
`python -m ttsem.sanitize script.py` does the same for a script. In CI, the action runs your suite
|
|
67
|
-
this way on a CPU runner
|
|
35
|
+
this way on a CPU runner:
|
|
68
36
|
|
|
69
37
|
```yaml
|
|
70
|
-
- uses: alepot55/ttsem@
|
|
38
|
+
- uses: alepot55/ttsem@v0.2.0
|
|
71
39
|
with:
|
|
72
40
|
install: pip install -e . # your project
|
|
73
41
|
args: tests/kernels -q # pytest arguments
|
|
@@ -306,11 +274,36 @@ right and holds against an adversarial answer, since the comparison rests on a r
|
|
|
306
274
|
cannot touch (`verified` means right values; a kernel launched with the work then done in PyTorch
|
|
307
275
|
still verifies). `unsafe` and `no_kernel` rest on two stop-gaps, a source scan and a run-time
|
|
308
276
|
integrity check of the whole checker, that block every attack in the test suite but are not a
|
|
309
|
-
proof: until the answer and the checker run in separate processes (planned
|
|
277
|
+
proof: until the answer and the checker run in separate processes (planned, not in 0.2), use the judge
|
|
310
278
|
as an auditor and an evaluation tool, and as an RL reward against adversarial optimisation with that
|
|
311
279
|
caveat. The rules, the batch mode, the classes, the sandbox and what each verdict guarantees are in
|
|
312
280
|
[docs/JUDGE.md][judge].
|
|
313
281
|
|
|
282
|
+
## Reward
|
|
283
|
+
|
|
284
|
+
New in 0.2, prototype-grade: `ttsem.reward` turns the judge's verdict into a reward for training
|
|
285
|
+
models that write Triton kernels, with no GPU in the reward loop. `Judge.score(reference, answer)`
|
|
286
|
+
returns a verdict: a score of 1.0, 0.0 or none (an abstention), and a paragraph for the model that
|
|
287
|
+
names the kernel, the buffer and the line. In gate mode a reward you already have is kept for a
|
|
288
|
+
verified answer and lowered to the fail value for a rejected one, never raised; in score mode the
|
|
289
|
+
score is the reward. The trainer's process imports neither torch nor Triton: the passes run
|
|
290
|
+
in a worker in the judge's own venv, every answer and every reference in its own `bwrap` sandbox,
|
|
291
|
+
and nothing runs without one. A verl adapter is included, tested against a fake worker and not yet
|
|
292
|
+
against a verl release.
|
|
293
|
+
|
|
294
|
+
```python
|
|
295
|
+
from ttsem.reward import Judge, extract_answer
|
|
296
|
+
|
|
297
|
+
judge = Judge(cache_dir=cache, python=judge_venv_python, mode="gate")
|
|
298
|
+
verdict = judge.score(reference_src, extract_answer(completion))
|
|
299
|
+
reward = judge.gate(gpu_reward, verdict)
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
It carries the judge's own caveat (`unsafe` and `no_kernel` are not proofs against an optimiser free
|
|
303
|
+
to write anything), prices no speed, and abstains where it cannot say whose fault an outcome is, which
|
|
304
|
+
an answer can sometimes provoke. What it does and does not catch, its API, the adapter and how the
|
|
305
|
+
worker runs are in [docs/REWARD.md][reward].
|
|
306
|
+
|
|
314
307
|
## How it was built
|
|
315
308
|
|
|
316
309
|
Built by Alessandro Potenza with Claude Code (Anthropic's coding agent) as the engineering agent
|
|
@@ -345,6 +338,7 @@ after the system was found to exploit its evaluation harness ([TechCrunch][sakan
|
|
|
345
338
|
| [docs/DESIGN.md][design] | the three levels, the interfaces between them, and why they are drawn there |
|
|
346
339
|
| [docs/RESULTS.md][results] | the corpora, the counts, and the calibration of the semantics against the device |
|
|
347
340
|
| [docs/JUDGE.md][judge] | the judge of model-written kernels: rules, verdicts, batches, the sandbox |
|
|
341
|
+
| [docs/REWARD.md][reward] | the judge as an RL reward: gate and score mode, abstentions, the API, the verl adapter, the worker, what it does not catch |
|
|
348
342
|
| [docs/INDUCTOR.md][inductor] | the Triton code of `torch.compile` under ttsem |
|
|
349
343
|
| [docs/AGENTS.md][agents] | for coding agents that write Triton kernels |
|
|
350
344
|
| [audits/](https://github.com/alepot55/ttsem/tree/main/audits) | the commands that redo the TritonBench and KernelBench audits from a clone |
|
|
@@ -359,6 +353,7 @@ this repository with the metadata in `CITATION.cff` and the commit you used. MIT
|
|
|
359
353
|
[agents]: https://github.com/alepot55/ttsem/blob/main/docs/AGENTS.md
|
|
360
354
|
[skill]: https://github.com/alepot55/ttsem/blob/main/skills/triton-verify/SKILL.md
|
|
361
355
|
[judge]: https://github.com/alepot55/ttsem/blob/main/docs/JUDGE.md
|
|
356
|
+
[reward]: https://github.com/alepot55/ttsem/blob/main/docs/REWARD.md
|
|
362
357
|
[ledger]: https://github.com/alepot55/ttsem/blob/main/LEDGER.md
|
|
363
358
|
[overview]: https://github.com/alepot55/ttsem/blob/main/docs/OVERVIEW.md
|
|
364
359
|
[semantics]: https://github.com/alepot55/ttsem/blob/main/docs/SEMANTICS.md
|
|
@@ -36,6 +36,7 @@ from typing import Any
|
|
|
36
36
|
|
|
37
37
|
from ttsem import judge
|
|
38
38
|
from ttsem import judge_tritonbench as tb
|
|
39
|
+
from ttsem._compat import python_script
|
|
39
40
|
from ttsem.judge_io import printable, read_json, run_sandboxed
|
|
40
41
|
|
|
41
42
|
RUNNER = Path(__file__).resolve().parent / "device_runner.py"
|
|
@@ -86,7 +87,7 @@ def run_file(
|
|
|
86
87
|
env["PYTHONPATH"] = os.environ["PYTHONPATH"]
|
|
87
88
|
cmd = judge.sandbox_prefix(work, cache, [script], env, hide) + gpu_nodes() + list(extra)
|
|
88
89
|
cmd += ["--ro-bind", str(RUNNER), str(RUNNER)] # whatever /tmp or `hide` hide
|
|
89
|
-
cmd += [sys.executable,
|
|
90
|
+
cmd += [*python_script(sys.executable, str(RUNNER)), str(script.resolve())]
|
|
90
91
|
cmd += [str(work / "run.json"), str(work / "run.npz")]
|
|
91
92
|
return tb.run_report(run_sandboxed(cmd, env, work, timeout), work, script, timeout)
|
|
92
93
|
|
|
@@ -145,7 +145,7 @@ into the checker. What each verdict holds against is therefore worth stating pla
|
|
|
145
145
|
race; the 32-bit helper packs value and flag into one word updated only atomically, which is
|
|
146
146
|
never a race. None of the 7,752 published TritonBench answers calls either.
|
|
147
147
|
- Full separation of the answer from the checker (running the semantics in a process the answer's
|
|
148
|
-
code cannot reach at all) is planned
|
|
148
|
+
code cannot reach at all) is planned, and is not in 0.2. Until then the judge is fit for auditing published
|
|
149
149
|
answers and for evaluation, where the answers were not written to defeat it. Used as an RL reward
|
|
150
150
|
against an optimiser that is free to write anything, treat `unsafe` and `no_kernel` with this
|
|
151
151
|
caveat: the two stop-gaps block every attack in the test suite, but they are not a proof, and
|
|
@@ -0,0 +1,358 @@
|
|
|
1
|
+
# The judge as a reward
|
|
2
|
+
|
|
3
|
+
`ttsem.reward` turns the judge's verdict on a model-written Triton kernel into a reward for
|
|
4
|
+
training: it runs the answer on a CPU, in the judge's sandbox, against the task's reference, and
|
|
5
|
+
says whether the kernel is right, whether it reads or writes outside its buffers or races, and
|
|
6
|
+
where, in a paragraph the model can be shown. No GPU is needed in the reward loop.
|
|
7
|
+
|
|
8
|
+
**Status: new in 0.2, prototype-grade.** The API may still change. It is tested against a fake
|
|
9
|
+
worker (every verdict-to-reward rule, the tiers, the cap), against the judge's own fixtures with
|
|
10
|
+
the real worker and sandbox, and as a module loaded by path; it has not been tested inside a verl
|
|
11
|
+
release, and this release shows no training run that used it. [What it does not
|
|
12
|
+
catch](#what-it-does-not-catch) is part of the status, not a footnote.
|
|
13
|
+
|
|
14
|
+
It judges two kinds of task, the ones [docs/JUDGE.md](JUDGE.md) describes: a KernelBench-style task
|
|
15
|
+
(the reference defines `Model`, `get_inputs` and `get_init_inputs`; the answer defines `ModelNew`)
|
|
16
|
+
and, with `format="tritonbench"`, a TritonBench task (the reference, a line of 146 `#`, the test).
|
|
17
|
+
|
|
18
|
+
## A first call
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
from ttsem.reward import Judge, extract_answer
|
|
24
|
+
|
|
25
|
+
judge = Judge(
|
|
26
|
+
cache_dir=Path("~/.cache/ttsem_reward").expanduser(), # on disk, with room: see "Cache"
|
|
27
|
+
python=Path("~/venvs/judge/bin/python").expanduser(), # the judge's own venv: see "The worker"
|
|
28
|
+
workers=4, # passes judged at once
|
|
29
|
+
mode="gate",
|
|
30
|
+
)
|
|
31
|
+
verdict = judge.score(reference_src, extract_answer(completion))
|
|
32
|
+
print(verdict.score, verdict.verdict, verdict.kind)
|
|
33
|
+
print(verdict.feedback) # one paragraph, for the model's next turn
|
|
34
|
+
reward = judge.gate(gpu_reward, verdict) # never above gpu_reward
|
|
35
|
+
judge.close()
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Gate mode and score mode
|
|
39
|
+
|
|
40
|
+
`mode` says what `Verdict.score` is.
|
|
41
|
+
|
|
42
|
+
- **`mode="gate"`** (the default) is for a reward you already have, usually a GPU reward that
|
|
43
|
+
measures speed. `score` is a multiplier: 1.0 for a kernel the judge verified, 0.0 for one it
|
|
44
|
+
rejected, `None` for an abstention. `Judge.gate(base, verdict)` applies it and can only lower the
|
|
45
|
+
reward: it returns `base` for a verified answer, `min(base, fail_value)` for a rejected one
|
|
46
|
+
(`fail_value` is 0.0 unless you set it), and for an abstention `base` (`on_abstain="keep"`, the
|
|
47
|
+
default) or `min(base, fail_value)` (`on_abstain="zero"`). The worst an answer that fools the
|
|
48
|
+
judge can get is the reward it would have had without it.
|
|
49
|
+
- **`mode="cpu"`**, score mode, is for a trainer with no GPU reward: `score` is the reward, 1.0 for
|
|
50
|
+
verified, 0.0 for rejected, and `None` for an abstention, which a trainer that needs a float
|
|
51
|
+
replaces by `abstain_value` (the verl adapter uses 0.0).
|
|
52
|
+
|
|
53
|
+
The two modes differ only where a failure may be the judge's environment rather than the answer
|
|
54
|
+
(next section): score mode gives 0.0 where gate mode abstains.
|
|
55
|
+
|
|
56
|
+
## What a verdict becomes
|
|
57
|
+
|
|
58
|
+
`scores(verdict, kind)` is the whole table, `(score-mode reward, gate multiplier)`; `kind_of`
|
|
59
|
+
derives `kind`, the finer reason, from the judge's record.
|
|
60
|
+
|
|
61
|
+
| judge verdict | `kind` | score mode | gate mode |
|
|
62
|
+
|---|---|---|---|
|
|
63
|
+
| `verified` | none | 1.0 | 1.0 |
|
|
64
|
+
| `wrong` | `values`, `unwritten` (memory nobody wrote shows as NaN), `autotune` (right under the first config only), `weights`, `shape`, `not_tensor` | 0.0 | 0.0 |
|
|
65
|
+
| `unsafe` | `oob_read`, `oob_write`, `race`, `poison` | 0.0 | 0.0 |
|
|
66
|
+
| `no_kernel` | none (no Triton launch: the work was done in PyTorch) | 0.0 | 0.0 |
|
|
67
|
+
| `not_judged` | `tampered`, `forged` (the answer reached into the checker; `Verdict.hack`) | 0.0 | 0.0 |
|
|
68
|
+
| `error` | `raised` (the answer raised an exception), `no_code` (the completion holds no code) | 0.0 | 0.0 |
|
|
69
|
+
| `not_judged` | `native_code` (the answer builds C++ or CUDA of its own) | 0.0 | abstain |
|
|
70
|
+
| `error` | `version` (a compile or import error that may be drift between your Triton and the judge's), `shared_memory` | 0.0 | abstain |
|
|
71
|
+
| `error` | `crash`, `configs` (the judge's sweep forced a config the autotuner would skip), `compare_dtype` | abstain | abstain |
|
|
72
|
+
| `too_slow` | `timeout` | abstain | abstain |
|
|
73
|
+
| `not_judged` | `races_unchecked`, `not_at_task_shape`, `tool_error`, unsupported operations, others | abstain | abstain |
|
|
74
|
+
|
|
75
|
+
### Abstention
|
|
76
|
+
|
|
77
|
+
`score is None` means the judge cannot say whose fault the outcome is. It is not a pass and not a
|
|
78
|
+
fail, and `Verdict.feedback` then starts with `(the judge abstains)` and names why. An abstention
|
|
79
|
+
leaves the reward to the other terms: the base reward in gate mode (or zero with
|
|
80
|
+
`on_abstain="zero"`), `abstain_value` in score mode.
|
|
81
|
+
|
|
82
|
+
Some abstentions can be provoked by the answer itself: a launch too large for the race check
|
|
83
|
+
(`races_unchecked`), inline C++ or CUDA (`native_code`), an output dtype the judge cannot compare
|
|
84
|
+
under the v3 rule (`compare_dtype`), a run that does not finish in `timeout_s`. In gate mode with
|
|
85
|
+
`on_abstain="keep"` such an answer keeps its base reward, so a policy that learns to provoke one
|
|
86
|
+
escapes the gate. Count the abstentions per step (the verl adapter logs `ttsem_abstain`), and use
|
|
87
|
+
`on_abstain="zero"` where an abstention should not pay, at the price of zeroing the abstentions
|
|
88
|
+
that are not the answer's doing.
|
|
89
|
+
|
|
90
|
+
## The API
|
|
91
|
+
|
|
92
|
+
### `Judge`
|
|
93
|
+
|
|
94
|
+
`Judge(*, cache_dir, python=None, workers=None, mode="gate", tiers=("scale", "medium"),
|
|
95
|
+
contract="operator", rule="kernelbench", precision="fp32", gpu="", timeout_s=120, cap=16,
|
|
96
|
+
on_abstain="keep", abstain_value=None, fail_value=0.0, strict_fs=True, real_max=2**20,
|
|
97
|
+
ttsem_root=None, mem_gb=None)`
|
|
98
|
+
|
|
99
|
+
| argument | meaning |
|
|
100
|
+
|---|---|
|
|
101
|
+
| `cache_dir` | where the tasks, the answers, the reference runs and every pass's record are kept (see "Cache") |
|
|
102
|
+
| `python` | the interpreter of the judge's venv (default: this interpreter); see "The worker" |
|
|
103
|
+
| `workers` | passes the worker runs at once (default: the CPU count) |
|
|
104
|
+
| `mode` | `gate` or `cpu` |
|
|
105
|
+
| `tiers` | `scale` always, then any of `medium`, `real` |
|
|
106
|
+
| `contract` | `operator` (default) or `task`, see "Shapes" |
|
|
107
|
+
| `rule`, `precision`, `gpu` | the judge's own: `kernelbench` (`allclose` at 1e-2) or `v3` (the KernelBench-v3 release's rule); `fp32` casts every tensor input to float32 as KernelBench does, `keep` leaves the task's dtypes; `gpu` names the device whose shared-memory limit applies. `score` takes them per call too |
|
|
108
|
+
| `timeout_s` | wall seconds for each pass's run |
|
|
109
|
+
| `cap` | the block size the race cap uses; `None` turns the cap off |
|
|
110
|
+
| `on_abstain`, `abstain_value`, `fail_value` | see "Gate mode and score mode" |
|
|
111
|
+
| `strict_fs` | the sandbox shows a run only what it needs (the judge's `--strict`); on by default |
|
|
112
|
+
| `real_max` | largest tensor, in elements, the `real` tier will run |
|
|
113
|
+
| `ttsem_root` | a checkout of ttsem the worker imports ahead of the venv's own |
|
|
114
|
+
| `mem_gb` | the address-space cap of every run (`TTSEM_MEM_GB` of the judge) |
|
|
115
|
+
|
|
116
|
+
Methods:
|
|
117
|
+
|
|
118
|
+
- `score(reference_src, answer_src, **overrides) -> Verdict`: one answer against one task's
|
|
119
|
+
reference. `answer_src` may be `None` or empty: that is `error` / `no_code`, scored 0.0 without a
|
|
120
|
+
run. `overrides`: `rule`, `precision`, `gpu`, `format` (`kernelbench` or `tritonbench`),
|
|
121
|
+
`contract`. A repeated `(task, answer)` pair is judged once and read back; a failure of the
|
|
122
|
+
worker itself (it died, a request raised) is an abstention with `kind="tool_error"`, not an
|
|
123
|
+
exception. Two things do raise: an unknown option or setting (`ValueError`), and a machine that
|
|
124
|
+
cannot make the sandbox (`RuntimeError`, see "The sandbox").
|
|
125
|
+
- `score_many(items, **overrides) -> list[Verdict]` for `(reference_src, answer_src)` pairs, and
|
|
126
|
+
`await ascore(...)`, which runs `score` in a thread.
|
|
127
|
+
- `gate(base, verdict) -> float`, above.
|
|
128
|
+
- `close()` ends the worker process. `Judge.from_env(**overrides)` reads the variables listed at the
|
|
129
|
+
end of this page. `judge.info` holds what the worker reported on its first call (versions, the
|
|
130
|
+
judge's root).
|
|
131
|
+
|
|
132
|
+
### `Verdict`
|
|
133
|
+
|
|
134
|
+
A frozen dataclass.
|
|
135
|
+
|
|
136
|
+
| field | meaning |
|
|
137
|
+
|---|---|
|
|
138
|
+
| `score` | the mode's score: 1.0, 0.0 or `None` |
|
|
139
|
+
| `verdict` | `verified`, `wrong`, `unsafe`, `no_kernel`, `error`, `too_slow` or `not_judged` |
|
|
140
|
+
| `kind` | the finer reason (the table above), or `None` |
|
|
141
|
+
| `feedback` | one paragraph for the model: the judge's own message (kernel, buffer, line, source line, what to check), the sizes it was found at and why those sizes count |
|
|
142
|
+
| `tier` | the tier that decided: `scale`, `scale_cap`, `medium`, `real` (`test`, `test_cap` for TritonBench) |
|
|
143
|
+
| `seconds` | CPU seconds the answer's sandboxed runs cost, the references' excluded |
|
|
144
|
+
| `record` | the judge's full record of every pass, for logs |
|
|
145
|
+
|
|
146
|
+
`Verdict.abstain` is `score is None`; `Verdict.hack` is true for `tampered` and `forged`, which a
|
|
147
|
+
policy that learns to attack the checker produces first: log it per step.
|
|
148
|
+
|
|
149
|
+
### `extract_answer(completion) -> str | None`
|
|
150
|
+
|
|
151
|
+
The answer's code in a model's completion: the last fenced Python block that defines `ModelNew`,
|
|
152
|
+
else the last fenced Python block, else the whole completion if it is Python that defines
|
|
153
|
+
`ModelNew`; `None` when there is none.
|
|
154
|
+
|
|
155
|
+
### `scores(verdict, kind)` and `kind_of(record, fmt="kernelbench")`
|
|
156
|
+
|
|
157
|
+
The table above as functions, for code that has the judge's record and not a `Verdict`.
|
|
158
|
+
|
|
159
|
+
## Shapes: what a verdict at a small size means
|
|
160
|
+
|
|
161
|
+
The judge runs the task's program with every size constant divided by the same power of two, so
|
|
162
|
+
that an interpreter finishes, and the reward builds its tiers on that.
|
|
163
|
+
|
|
164
|
+
1. `scale`: the largest tensor is shrunk to at most 2**13 elements, one trial, small enough that
|
|
165
|
+
the race check can usually cover every launch.
|
|
166
|
+
2. `medium`, only after `scale` verified: 2**17 elements. It catches what grows with the shape.
|
|
167
|
+
3. `real`, optional (`tiers=("scale", "medium", "real")`): the task's own sizes, when its largest
|
|
168
|
+
tensor holds at most `real_max` elements.
|
|
169
|
+
|
|
170
|
+
`contract="operator"` (default) holds an answer to compute the operator at the task's shape and at
|
|
171
|
+
the scaled ones, so a kernel specialised to the benchmark's shape (no bound on a dimension that is
|
|
172
|
+
a multiple of the block there) is rejected, with feedback that says the sizes. That is a choice
|
|
173
|
+
about what to pay for and it changes the reward. `contract="task"` holds it to the task's shape
|
|
174
|
+
only: a rejection at a scaled shape stands only if the task's shape confirms it, and where that
|
|
175
|
+
shape is too large to run the verdict is an abstention (`not_at_task_shape`).
|
|
176
|
+
|
|
177
|
+
### The cap
|
|
178
|
+
|
|
179
|
+
A kernel launched as one program instance has nothing to race with. When `scale` verifies and the
|
|
180
|
+
answer has a launch whose grid is a ceiling division over a block constant (`cap.py` lists the
|
|
181
|
+
forms it recognises), the pass runs again on a copy whose blocks are capped at `cap`, so that the
|
|
182
|
+
small test launches many instances; only an `unsafe` verdict of that copy counts (`scale_cap`). A
|
|
183
|
+
launch whose autotune configs all list blocks above the cap is not capped, since no config the
|
|
184
|
+
answer's own tuner can pick launches a block that small. The cap is insurance against an optimiser
|
|
185
|
+
that learns a race, not a claim that models write them.
|
|
186
|
+
|
|
187
|
+
## The verl adapter
|
|
188
|
+
|
|
189
|
+
`ttsem.reward.verl_adapter` is a custom reward function in the shape verl documents: four
|
|
190
|
+
arguments in, a dict with a `score` key out. `ground_truth` is the task's reference module as a
|
|
191
|
+
string. The judge is configured by environment variables ([the list](#environment-variables)), so
|
|
192
|
+
the trainer needs no code of ours.
|
|
193
|
+
|
|
194
|
+
Score mode, the reward being the judge's:
|
|
195
|
+
|
|
196
|
+
```yaml
|
|
197
|
+
reward:
|
|
198
|
+
custom_reward_function:
|
|
199
|
+
path: /path/to/site-packages/ttsem/reward/verl_adapter.py # python -c "import ttsem.reward.verl_adapter as m; print(m.__file__)"
|
|
200
|
+
name: compute_score # acompute_score under verl's async reward loop
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
That is the key verl reads from 0.7.1 on (verl 0.9.1 was the latest release when this was
|
|
204
|
+
written); in 0.6 it was a top-level `custom_reward_function`, and a trainer on 0.6 needs the old
|
|
205
|
+
form.
|
|
206
|
+
|
|
207
|
+
Gate mode over a reward of your own, in a file of yours that `custom_reward_function.path` names:
|
|
208
|
+
|
|
209
|
+
```python
|
|
210
|
+
from ttsem.reward.verl_adapter import gated
|
|
211
|
+
from my_rewards import compute_score as base # the GPU reward, verl's signature
|
|
212
|
+
|
|
213
|
+
compute_score = gated(base) # gated(base, lazy=True): judge only what base accepts
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
By default the base reward and the judge run at once; with `lazy=True` the judge runs only when the
|
|
217
|
+
base is above `fail_value`, which saves CPU and adds the two latencies.
|
|
218
|
+
|
|
219
|
+
Every call returns `{"score": float, "ttsem_verdict", "ttsem_kind", "ttsem_tier", "ttsem_abstain",
|
|
220
|
+
"ttsem_hack", "ttsem_seconds", "ttsem_feedback"}`, the same keys for a judged and a skipped sample
|
|
221
|
+
(verl stacks them across a batch); gate mode adds `base_score` and keeps the base's own keys.
|
|
222
|
+
`ttsem_feedback` is the paragraph for a multi-turn rollout's next turn. The judge judges one
|
|
223
|
+
sample per call, so use a reward manager that calls the function concurrently.
|
|
224
|
+
|
|
225
|
+
The function signatures and the dict return follow the source of verl 0.9.1; its documentation
|
|
226
|
+
page gives only the signature and an example that returns a number. This repository tests the
|
|
227
|
+
adapter against a fake worker and loads it by path as verl does, not against verl itself: check
|
|
228
|
+
the reward manager and the dict convention on the verl version you pin.
|
|
229
|
+
|
|
230
|
+
Two things to plan for on a real job, read from verl's source and not yet run: verl starts several
|
|
231
|
+
reward workers (`reward.num_workers`, 8 by default), each of which builds its own `Judge`, so the
|
|
232
|
+
real ceiling on concurrent passes is that number times `TTSEM_REWARD_WORKERS`; and the workers are
|
|
233
|
+
spread over the nodes of the job, so ttsem, the judge's venv, `bwrap` and the `TTSEM_REWARD_*`
|
|
234
|
+
variables must be present on each of them.
|
|
235
|
+
|
|
236
|
+
## The worker: the judge runs in its own venv
|
|
237
|
+
|
|
238
|
+
The judge needs Triton and torch (CPU), pinned in ttsem's `triton` extra, and a trainer usually
|
|
239
|
+
has its own CUDA builds of both. So the trainer's process imports neither: `Judge` starts one
|
|
240
|
+
long-lived worker process, `python -I .../ttsem/reward/worker.py`, with the interpreter you give
|
|
241
|
+
as `python` (`TTSEM_REWARD_PYTHON`), and talks to it in JSON lines over a pipe. The worker hands
|
|
242
|
+
each pass to `ttsem.judge`, which runs the answer and the reference each in a `bwrap` sandbox of
|
|
243
|
+
its own, and sends back the judge's record. A worker that dies fails its pending requests (they
|
|
244
|
+
become abstentions, `tool_error`) and the next call starts another; a forked reward process starts
|
|
245
|
+
its own.
|
|
246
|
+
|
|
247
|
+
Make the judge's venv once, with the pinned wheels:
|
|
248
|
+
|
|
249
|
+
```console
|
|
250
|
+
$ python -m venv ~/venvs/judge
|
|
251
|
+
$ ~/venvs/judge/bin/pip install --index-url https://download.pytorch.org/whl/cpu \
|
|
252
|
+
--extra-index-url https://pypi.org/simple 'ttsem[triton]'
|
|
253
|
+
$ export TTSEM_REWARD_PYTHON=~/venvs/judge/bin/python
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
Make it from an interpreter that sits outside your home directory (the system Python, or one whose
|
|
257
|
+
real path is not under `$HOME`). The sandbox hides the home, so a venv whose `python` is a symlink
|
|
258
|
+
into a Python that a tool keeps there does not start inside it; `Judge` checks this on its first
|
|
259
|
+
call and raises instead of letting every pass abstain.
|
|
260
|
+
|
|
261
|
+
The trainer's side needs ttsem and numpy only. Keep the two ttsem installs at one version: the
|
|
262
|
+
worker file is the trainer's own and talks to the `ttsem.judge` of the venv. `ttsem_root`
|
|
263
|
+
(`TTSEM_REWARD_TTSEM_ROOT`) names a checkout the worker imports ahead of the venv's, for working on
|
|
264
|
+
ttsem itself. No new extra is needed: the worker's requirements are the `triton` extra.
|
|
265
|
+
|
|
266
|
+
Each pass costs CPU seconds, not GPU: the sandboxed run imports torch and Triton, runs the
|
|
267
|
+
reference once per task and tier (kept, see below) and runs the answer. `Verdict.seconds` is the
|
|
268
|
+
CPU time of the answer's runs, measured inside the sandbox on the process tree, so it stays honest
|
|
269
|
+
on a machine shared with other work. It is telemetry from the sandbox: an answer could forge it,
|
|
270
|
+
never the verdict.
|
|
271
|
+
|
|
272
|
+
### Cache
|
|
273
|
+
|
|
274
|
+
`cache_dir` holds the tasks and answers (named by the hash of their text), one reference run per
|
|
275
|
+
task and tier, reused across a GRPO group and across epochs, and every pass's record. It also holds
|
|
276
|
+
every model-written answer the run saw. Put it on disk, not on a RAM-backed `/tmp`, and clear it
|
|
277
|
+
when the run is done. No answer's sandbox sees it, except its own working directory.
|
|
278
|
+
|
|
279
|
+
## The sandbox
|
|
280
|
+
|
|
281
|
+
The reward never runs an answer without a sandbox and has no switch for it. It needs Linux and
|
|
282
|
+
[`bwrap`](https://github.com/containers/bubblewrap) (bubblewrap) on `PATH`, able to create its
|
|
283
|
+
namespaces: unprivileged user namespaces enabled, or root. The first call asks the worker to make
|
|
284
|
+
one, and raises `RuntimeError` ("the judge cannot run here") instead of judging if it cannot. The
|
|
285
|
+
sandbox is the judge's, described in [docs/JUDGE.md](JUDGE.md#the-sandbox): a read-only filesystem
|
|
286
|
+
(with `strict_fs`, only what the run needs), the home directory and the host's sockets hidden, no
|
|
287
|
+
network, its own pid namespace, no capabilities.
|
|
288
|
+
|
|
289
|
+
Not inside a container whose seccomp profile forbids namespaces, which is Docker's default: `bwrap`
|
|
290
|
+
cannot start there. Run the judge on the host, or start the container with a profile that allows
|
|
291
|
+
them. Switching the sandbox off is not an answer: the answer's process could then overwrite the
|
|
292
|
+
reference's outputs on disk and turn `wrong` into `verified`.
|
|
293
|
+
|
|
294
|
+
## What it does not catch
|
|
295
|
+
|
|
296
|
+
- **The judge's own caveat applies unchanged.** `wrong` versus `verified` holds against an
|
|
297
|
+
adversarial answer. `unsafe` and `no_kernel` rest on a source scan and a run-time integrity
|
|
298
|
+
check, not on a separation of the answer from the checker, which this release does not have
|
|
299
|
+
([docs/JUDGE.md](JUDGE.md#what-the-verdicts-guarantee)). Against an optimiser free to write
|
|
300
|
+
anything, they bound the damage (gate mode never raises a reward) and do not prove anything.
|
|
301
|
+
Every `tampered` and `forged` verdict is counted as a hack attempt.
|
|
302
|
+
- **Speed.** Nothing is timed. The reward says a kernel is right and safe, not fast; timing hacks
|
|
303
|
+
are the job of the GPU reward it gates.
|
|
304
|
+
- **The device.** Triton's IR is interpreted on a CPU. Below the LLVM conversion (ptxas, the
|
|
305
|
+
hardware, the order of atomic updates, fast-math, a tensor core's accumulation) nothing runs, so a
|
|
306
|
+
kernel right here can differ on a device, and the comparison tolerance of the rule decides the
|
|
307
|
+
rest.
|
|
308
|
+
- **Sizes that did not run.** A verdict holds for the inputs and sizes that ran. With the default
|
|
309
|
+
tiers the task's own shape is not run; a fault only that shape reaches is missed, and under
|
|
310
|
+
`contract="task"` a kernel wrong only at a scaled shape is let through.
|
|
311
|
+
- **Races in large launches.** Races are looked for between program instances, and past a limit on
|
|
312
|
+
the accesses of one launch they cannot be: such a launch is an abstention (`races_unchecked`),
|
|
313
|
+
not a pass.
|
|
314
|
+
- **Anything outside Triton.** Inline C++ or CUDA is not run (`native_code`), and inline PTX and the
|
|
315
|
+
operations the semantics does not model stop as `unsupported`, by name: abstentions, not
|
|
316
|
+
verdicts.
|
|
317
|
+
- **A compile error under the judge's Triton.** It may be drift between the trainer's Triton and
|
|
318
|
+
the judge's, not the answer's fault: gate mode abstains, and score mode scores 0.0, which is
|
|
319
|
+
where score mode risks rejecting a good answer.
|
|
320
|
+
- **The v3 rule and an output of another dtype.** The judge's comparison calls `torch.allclose`
|
|
321
|
+
under every rule, which raises when the answer's output dtype is not the reference's, where the
|
|
322
|
+
v3 harness casts both to float32 first. Under `rule="v3"` that is an abstention
|
|
323
|
+
(`compare_dtype`), so an answer can return another dtype to escape a gate with
|
|
324
|
+
`on_abstain="keep"`; under `rule="kernelbench"` the error stands, as KernelBench's own comparison
|
|
325
|
+
raises it.
|
|
326
|
+
- **Tasks the precision cast breaks.** `precision="fp32"` casts every tensor input to float32, which
|
|
327
|
+
breaks a reference that takes integer indices (gather, embedding): its run fails and the pass is
|
|
328
|
+
an abstention. Use `precision="keep"` for those.
|
|
329
|
+
- **What it does to training is not shown here.** The rules are tested; whether a policy trained on
|
|
330
|
+
them improves, or learns around them, is not.
|
|
331
|
+
|
|
332
|
+
## Environment variables
|
|
333
|
+
|
|
334
|
+
`Judge.from_env()` and the verl adapter read:
|
|
335
|
+
|
|
336
|
+
| variable | argument |
|
|
337
|
+
|---|---|
|
|
338
|
+
| `TTSEM_REWARD_CACHE` | `cache_dir` (default `~/.cache/ttsem_reward`) |
|
|
339
|
+
| `TTSEM_REWARD_PYTHON` | `python` |
|
|
340
|
+
| `TTSEM_REWARD_TTSEM_ROOT` | `ttsem_root` |
|
|
341
|
+
| `TTSEM_REWARD_WORKERS` | `workers` |
|
|
342
|
+
| `TTSEM_REWARD_MODE` | `mode` (the adapter sets it per function) |
|
|
343
|
+
| `TTSEM_REWARD_CONTRACT` | `contract` |
|
|
344
|
+
| `TTSEM_REWARD_CAP` | `cap`; `0` turns it off |
|
|
345
|
+
| `TTSEM_REWARD_TIERS` | `tiers`, comma-separated |
|
|
346
|
+
| `TTSEM_REWARD_TIMEOUT` | `timeout_s` |
|
|
347
|
+
| `TTSEM_REWARD_RULE`, `TTSEM_REWARD_PRECISION`, `TTSEM_REWARD_GPU` | `rule`, `precision`, `gpu` |
|
|
348
|
+
| `TTSEM_REWARD_MEM_GB` | `mem_gb` |
|
|
349
|
+
|
|
350
|
+
`on_abstain`, `abstain_value`, `fail_value`, `strict_fs` and `real_max` are not read from the
|
|
351
|
+
environment: pass them to `Judge.from_env(...)`.
|
|
352
|
+
|
|
353
|
+
## Tests
|
|
354
|
+
|
|
355
|
+
`pytest tests/test_reward*.py` runs the unit tests with no GPU and no Triton, and the integration
|
|
356
|
+
tests where the judge's interpreter has torch and Triton and `bwrap` works (they skip otherwise).
|
|
357
|
+
`tests/test_reward_hygiene.py` fails if the package or this page names a path of the machine it was
|
|
358
|
+
written on or anything that does not belong in public text.
|